Skip to content
Snippets Groups Projects
Commit 15b36cb9 authored by David Johansen's avatar David Johansen
Browse files

added recon_range functionality and chunk benchmark jobs

parent b06a4091
No related branches found
No related tags found
No related merge requests found
#!/bin/sh
#BSUB -J d1000c1000
#BSUB -q gpuv100
#BSUB -n 8
#BSUB -R "span[hosts=1]"
#BSUB -R "rusage[mem=8GB]"
#BSUB -W 0:10
#BSUB -gpu "num=1:mode=exclusive_process"
#BSUB -B
#BSUB -N
#BSUB -o logs/bjobs/chunk_benchmark/d1000c1000_%J.out
#BSUB -e logs/bjobs/chunk_benchmark/d1000c1000_%J.err
echo $HOSTNAME
nvidia-smi
module load cuda/11.6
/appl/cuda/11.6.0/samples/bin/x86_64/linux/release/deviceQuery
nvidia-smi
#### Parameters ####
ANGLE_CHUNKS=1
VOL_CHUNKS_PER_AXIS=1
####################
cd /dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/3dim-reconstruction
. env_setup.sh
VOL_CHUNKS=$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS
DATA_PATH="/work3/s214743/ct_data/casper_d1000c1000"
CONFIG_PATH="/dtu/3d-imaging-center/projects/2021_DANFIX_Casper/raw_data_3DIM/Casper_top_3_2 [2021-03-17 16.54.39]/Casper_top_3_2_recon.xtekct"
python -u ct/chunking.py \
--config-path "$CONFIG_PATH" \
--output-dir "$DATA_PATH" \
--angle-chunks $ANGLE_CHUNKS \
--vol-chunks $VOL_CHUNKS \
--read-batch-size 1
du -lh $DATA_PATH
\ No newline at end of file
#!/bin/sh
#BSUB -J d1000c500
#BSUB -q gpuv100
#BSUB -n 8
#BSUB -R "span[hosts=1]"
#BSUB -R "rusage[mem=8GB]"
#BSUB -W 0:30
#BSUB -gpu "num=1:mode=exclusive_process"
#BSUB -B
#BSUB -N
#BSUB -o logs/bjobs/chunk_benchmark/d1000c500_%J.out
#BSUB -e logs/bjobs/chunk_benchmark/d1000c500_%J.err
echo $HOSTNAME
nvidia-smi
module load cuda/11.6
/appl/cuda/11.6.0/samples/bin/x86_64/linux/release/deviceQuery
nvidia-smi
#### Parameters ####
ANGLE_CHUNKS=1
VOL_CHUNKS_PER_AXIS=2
####################
cd /dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/3dim-reconstruction
. env_setup.sh
VOL_CHUNKS=$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS
DATA_PATH="/work3/s214743/ct_data/casper_d1000c500"
CONFIG_PATH="/dtu/3d-imaging-center/projects/2021_DANFIX_Casper/raw_data_3DIM/Casper_top_3_2 [2021-03-17 16.54.39]/Casper_top_3_2_recon.xtekct"
python -u ct/chunking.py \
--config-path "$CONFIG_PATH" \
--output-dir "$DATA_PATH" \
--angle-chunks $ANGLE_CHUNKS \
--vol-chunks $VOL_CHUNKS \
--read-batch-size 1
du -lh $DATA_PATH
\ No newline at end of file
#!/bin/sh
#BSUB -J d2000c1000
#BSUB -q gpuv100
#BSUB -n 8
#BSUB -R "span[hosts=1]"
#BSUB -R "rusage[mem=8GB]"
#BSUB -W 1:00
#BSUB -gpu "num=1:mode=exclusive_process"
#BSUB -B
#BSUB -N
#BSUB -o logs/bjobs/chunk_benchmark/d2000c1000_%J.out
#BSUB -e logs/bjobs/chunk_benchmark/d2000c1000_%J.err
echo $HOSTNAME
nvidia-smi
module load cuda/11.6
/appl/cuda/11.6.0/samples/bin/x86_64/linux/release/deviceQuery
nvidia-smi
#### Parameters ####
ANGLE_CHUNKS=8 # 2**3
VOL_CHUNKS_PER_AXIS=2
####################
cd /dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/3dim-reconstruction
. env_setup.sh
VOL_CHUNKS=$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS
DATA_PATH="/work3/s214743/ct_data/casper_d2000c1000"
TIFF_DIR="/dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/data/casper_2000"
python -u ct/chunking.py \
--tiff-dir "$TIFF_DIR" \
--output-dir "$DATA_PATH" \
--angle-chunks $ANGLE_CHUNKS \
--vol-chunks $VOL_CHUNKS \
--read-batch-size 1 \
--use-pickle-geometries
du -lh $DATA_PATH
\ No newline at end of file
#!/bin/sh
#BSUB -J d2000c500
#BSUB -q gpuv100
#BSUB -n 8
#BSUB -R "span[hosts=1]"
#BSUB -R "rusage[mem=8GB]"
#BSUB -W 1:00
#BSUB -gpu "num=1:mode=exclusive_process"
#BSUB -B
#BSUB -N
#BSUB -o logs/bjobs/chunk_benchmark/d2000c500_%J.out
#BSUB -e logs/bjobs/chunk_benchmark/d2000c500_%J.err
echo $HOSTNAME
nvidia-smi
module load cuda/11.6
/appl/cuda/11.6.0/samples/bin/x86_64/linux/release/deviceQuery
nvidia-smi
#### Parameters ####
ANGLE_CHUNKS=8 # 2**3
VOL_CHUNKS_PER_AXIS=4
####################
cd /dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/3dim-reconstruction
. env_setup.sh
VOL_CHUNKS=$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS
DATA_PATH="/work3/s214743/ct_data/casper_d2000c500"
TIFF_DIR="/dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/data/casper_2000"
python -u ct/chunking.py \
--tiff-dir "$TIFF_DIR" \
--output-dir "$DATA_PATH" \
--angle-chunks $ANGLE_CHUNKS \
--vol-chunks $VOL_CHUNKS \
--read-batch-size 1 \
--use-pickle-geometries
du -lh $DATA_PATH
\ No newline at end of file
#!/bin/sh
#BSUB -J d4000c1000
#BSUB -q gpuv100
#BSUB -n 8
#BSUB -R "span[hosts=1]"
#BSUB -R "rusage[mem=8GB]"
#BSUB -W 2:00
#BSUB -gpu "num=1:mode=exclusive_process"
#BSUB -B
#BSUB -N
#BSUB -o logs/bjobs/chunk_benchmark/d4000c1000_%J.out
#BSUB -e logs/bjobs/chunk_benchmark/d4000c1000_%J.err
echo $HOSTNAME
nvidia-smi
module load cuda/11.6
/appl/cuda/11.6.0/samples/bin/x86_64/linux/release/deviceQuery
nvidia-smi
#### Parameters ####
ANGLE_CHUNKS=64 # 4**3
VOL_CHUNKS_PER_AXIS=4
####################
cd /dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/3dim-reconstruction
. env_setup.sh
VOL_CHUNKS=$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS
DATA_PATH="/work3/s214743/ct_data/casper_d4000c1000"
TIFF_DIR="/dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/data/casper_4000"
python -u ct/chunking.py \
--tiff-dir "$TIFF_DIR" \
--output-dir "$DATA_PATH" \
--angle-chunks $ANGLE_CHUNKS \
--vol-chunks $VOL_CHUNKS \
--read-batch-size 1 \
--use-pickle-geometries \
--write-projections 0 \
--recon-range "28:40"
du -lh $DATA_PATH
\ No newline at end of file
#!/bin/sh
#BSUB -J d4000c500
#BSUB -q gpuv100
#BSUB -n 8
#BSUB -R "span[hosts=1]"
#BSUB -R "rusage[mem=8GB]"
#BSUB -W 3:00
#BSUB -gpu "num=1:mode=exclusive_process"
#BSUB -B
#BSUB -N
#BSUB -o logs/bjobs/chunk_benchmark/d4000c500_%J.out
#BSUB -e logs/bjobs/chunk_benchmark/d4000c500_%J.err
echo $HOSTNAME
nvidia-smi
module load cuda/11.6
/appl/cuda/11.6.0/samples/bin/x86_64/linux/release/deviceQuery
nvidia-smi
#### Parameters ####
ANGLE_CHUNKS=64 # 4**3
VOL_CHUNKS_PER_AXIS=8
####################
cd /dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/3dim-reconstruction
. env_setup.sh
VOL_CHUNKS=$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS,$VOL_CHUNKS_PER_AXIS
DATA_PATH="/work3/s214743/ct_data/casper_d4000c500"
TIFF_DIR="/dtu/3d-imaging-center/projects/2024_QIM_platform/analysis/data/casper_4000"
python -u ct/chunking.py \
--tiff-dir "$TIFF_DIR" \
--output-dir "$DATA_PATH" \
--angle-chunks $ANGLE_CHUNKS \
--vol-chunks $VOL_CHUNKS \
--read-batch-size 1 \
--use-pickle-geometries \
--write-projections 0 \
--recon-range "500:520"
du -lh $DATA_PATH
\ No newline at end of file
......@@ -202,6 +202,7 @@ class ChunkedCT:
return ig_chunk
def reconstruct_volume_chunk(self, chunk_coords):
t = time.time()
ig_chunk = self.make_ig_chunk(chunk_coords)
vol_accumulated = np.zeros(ig_chunk.shape, dtype=np.float32)
......@@ -218,9 +219,12 @@ class ChunkedCT:
recon_zarr = zarr.open_array(store=self.recon_zarr_path, mode='a')
recon_zarr.blocks[*chunk_coords] = vol_accumulated
print(f'Finished writing volume chunk {chunk_coords}.')
print(f'Finished writing volume chunk {chunk_coords}. Took {time.time()-t} s')
def reconstruct(self):
def reconstruct(self, chunks=None):
"""
chunks: slice object specifying the ravelled indices to reconstruct.
"""
t = time.time()
# compressor = zarr.codecs.Blosc(cname='zstd')
zarr.open_array(
......@@ -231,11 +235,20 @@ class ChunkedCT:
chunks=self.vol_chunk_shape,
compressor=None
)
for chunk_coords in itertools.product(*map(range, self.vol_chunks)):
full_range = list(itertools.product(*map(range, self.vol_chunks)))
chunk_range = full_range[chunks] if chunks else full_range
for chunk_coords in chunk_range:
self.reconstruct_volume_chunk(chunk_coords)
print(f'Chunked reconstruction took {time.time()-t} s')
def parse_simple_slice(s: str) -> slice:
parts = s.split(":")
if len(parts) > 2:
raise ValueError(f"Follow the format start:end")
start, stop = (int(p) if p else None for p in parts)
return slice(start, stop)
@app.default
def main(
config_path: str=None,
......@@ -247,6 +260,8 @@ def main(
processes: str=None,
verbose: int=None,
use_pickle_geometries: bool=False,
recon_range: str=None,
write_projections: int=1,
):
"""
Parameters
......@@ -264,6 +279,10 @@ def main(
processes: str
Number of processes for multiprocessing. None uses all available.
verbose: int
recon_range: str
Volume chunks to reconstruct specified as start:end like Python indices (except no step)
write_projections: int
Whether (1) or not (0) to write_projections. Setting to 0 is useful when having multiple instances reconstruct the full volume.
"""
print('-' * 60)
print("Performing chunked CT with the configurations:")
......@@ -275,6 +294,9 @@ def main(
if len(vol_chunks) != 3:
raise ValueError
recon_range = parse_simple_slice(recon_range) if recon_range else None
write_projections = bool(write_projections)
# Currently for ease of testing
if config_path is None:
config_path = '/dtu/3d-imaging-center/projects/2021_DANFIX_Casper/raw_data_3DIM/Casper_top_3_2 [2021-03-17 16.54.39]/Casper_top_3_2_recon.xtekct'
......@@ -290,8 +312,9 @@ def main(
verbose=verbose,
use_pickle_geometries=use_pickle_geometries,
)
if write_projections:
processor.write_projection_chunks()
processor.reconstruct()
processor.reconstruct(chunks=recon_range)
if __name__ == '__main__':
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment