prasb commited on Apr 7, 2025

Commit

c80d50f

verified ·

1 Parent(s): 373aad2

Add files using upload-large-folder tool

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.DiffTumor.txt.swp +0 -0
.gitattributes +1 -0
ADiffTumor.o427024 +12 -0
ADiffTumordb.txt +95 -0
BDiffTumor.o427025 +11 -0
BDiffTumorSingle.sh +73 -0
BDiffTumordb.txt +6 -0
DiffTumor.o425447 +9 -0
DiffTumor.o425448 +9 -0
DiffTumor.o425449 +9 -0
DiffTumor.o425450 +9 -0
DiffTumor.o425451 +9 -0
DiffTumor.o425452 +9 -0
DiffTumor.o425453 +9 -0
DiffTumor.o425454 +9 -0
DiffTumor.o425455 +9 -0
DiffTumor.o425456 +9 -0
DiffTumor.o425457 +9 -0
DiffTumor.o425458 +10 -0
DiffTumor.o425459 +10 -0
DiffTumor.o425460 +10 -0
DiffTumor.o425461 +10 -0
DiffTumor.o425463 +10 -0
DiffTumor.o426418 +11 -0
DiffTumor.o426420 +12 -0
DiffTumor.o426421 +12 -0
DiffTumor.o426422 +11 -0
DiffTumor.o426423 +12 -0
DiffTumor.o426426 +11 -0
DiffTumor.o426427 +11 -0
DiffTumor.o426428 +11 -0
DiffTumor.o426604 +11 -0
DiffTumor.o426817 +11 -0
DiffTumor.o426868 +12 -0
DiffTumor.o429178 +11 -0
DiffTumor.o429215 +11 -0
DiffTumor.sh +71 -0
DiffTumor.txt +1 -0
DiffTumorHealthy.o426937 +11 -0
DiffTumorHealthy.o426938 +10 -0
DiffTumorHealthy.sh +71 -0
DiffTumorHealthy.txt +882 -0
DiffTumorNew.sh +72 -0
DiffTumorSingle.sh +73 -0
DiffTumorUCSFTest.sh +26 -0
DiffTumordb.txt +89 -0
RunDifftumor copy.sh +130 -0
logs_difftumor_atlas_test.txt +3 -0
my_container_sandbox/.exec +9 -0
my_container_sandbox/.run +23 -0

.DiffTumor.txt.swp ADDED Viewed

Binary file (16.4 kB). View file

.gitattributes CHANGED Viewed

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 difftumor_final_updated.sif filter=lfs diff=lfs merge=lfs -text

 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 difftumor_final_updated.sif filter=lfs diff=lfs merge=lfs -text
+logs_difftumor_atlas_test.txt filter=lfs diff=lfs merge=lfs -text

ADiffTumor.o427024 ADDED Viewed

	@@ -0,0 +1,12 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.
+rm: cannot remove '/fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir/case00001.nii.gz': No such file or directory

ADiffTumordb.txt ADDED Viewed

@@ -0,0 +1,95 @@
  0%|          | 0/1 [00:00<?, ?it/s]['case00001']
  0%|          | 0/1 [00:45<?, ?it/s]

+0
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-f4b41184-398a-ffd0-e84a-620cbce79589] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00001']
+2024-07-12 21:02:53,968 INFO image_writer.py:190 - writing: /workspace/outputs/case00001/predictions/case00001_liver.nii.gz
+2024-07-12 21:02:55,938 INFO image_writer.py:190 - writing: /workspace/outputs/case00001/predictions/case00001_pancreas.nii.gz
+2024-07-12 21:02:57,950 INFO image_writer.py:190 - writing: /workspace/outputs/case00001/predictions/case00001_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:45<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00001
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+Traceback (most recent call last):
+  File "tumor_mask_access/pancreas_tumor.py", line 251, in <module>
+    main()
+  File "tumor_mask_access/pancreas_tumor.py", line 212, in main
+    val_loader, post_transforms = _get_loader(args)
+  File "tumor_mask_access/pancreas_tumor.py", line 158, in _get_loader
+    test_data_list = os.listdir(args.data_root)
+FileNotFoundError: [Errno 2] No such file or directory: '/workspace/inputs'
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+Traceback (most recent call last):
+  File "tumor_mask_access/kidney_tumor.py", line 249, in <module>
+    main()
+  File "tumor_mask_access/kidney_tumor.py", line 210, in main
+    val_loader, post_transforms = _get_loader(args)
+  File "tumor_mask_access/kidney_tumor.py", line 156, in _get_loader
+    test_data_list = os.listdir(args.data_root)
+FileNotFoundError: [Errno 2] No such file or directory: '/workspace/inputs'

BDiffTumor.o427025 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

BDiffTumorSingle.sh ADDED Viewed

	@@ -0,0 +1,73 @@

+#!/bin/sh
+#PBS -l select=1:ncpus=10:ngpus=1
+#PBS -l walltime=23:59:00
+#PBS -j oe
+#PBS -N BDiffTumor
+#PBS -q gpu
+module load nvhpc-hpcx-cuda11/23.5
+module load go-1.19.4/singularity-1.1.8
+cd /work/psalvador
+output_file="BDiffTumordb.txt"
+echo "0"> "$output_file"
+#Number of processes per node to launch
+NPROC_PER_NODE=4
+#Number of process in all modes
+WORLD_SIZE=$PBS_NUM_NODES
+MASTER_ADDR=`/bin/hostname -s`
+cat $PBS_NODEFILE>nodelist
+#Make sure this node (MASTER) comes first
+SLAVES=`cat nodelist | grep -v $MASTER_ADDR | uniq`
+#We want names of master and slave nodes
+HOSTLIST="$MASTER_ADDR $SLAVES"
+#Get free port:
+BASE_PORT=16998
+INCREMENT=1
+port=$BASE_PORT
+isfree=$(netstat -taln | grep $port)
+while [[ -n "$isfree" ]]; do
+    port=$[port+INCREMENT]
+    isfree=$(netstat -taln | grep $port)
+done
+MASTER_PORT=$port
+export NCCL_DEBUG=INFO
+arr=($HOSTLIST)
+export TORCH_HOME=/fastwork/psalvador/cache
+export TORCHVISION_HOME=/fastwork/psalvador/cache
+#This command to run your pytorch script
+export inputs_data=/fastwork/psalvador/JHU/data/UCSF_remaining/SXz0J6vDWz/SXz0J6vDWz/1J07NmUKTS
+export outputs_data=/fastwork/psalvador/JHU/UCSF/DiffTumor/outputs/SXz0J6vDWz_1J07NmUKTS
+mkdir -p "$outputs_data"
+chmod 644 overlay.img
+cd /fastwork/psalvador/JHU/UCSF/DiffTumor
+echo "0"> "$output_file"
+export dir=/fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir
+# Check if the directory exists
+if [ -d "$dir" ]; then
+    # If the directory exists, remove it
+    rm -rf "$dir"
+    echo "Directory $dir has been removed."
+fi
+# Create the directory
+mkdir -p "$dir"
+for ct_file in "$inputs_data"/*.nii.gz; do
+  if [ -f "$ct_file" ]; then
+    cp "$ct_file" "$dir/$(basename "$ct_file")"
+    SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity run --fakeroot --overlay /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img --nv -B $dir:/workspace/inputs -B $outputs_data:/workspace/outputs difftumor_final.sif>> "$output_file" 2>&1
+    rm "$dir/$(basename "$ct_file")"
+  fi
+done

BDiffTumordb.txt ADDED Viewed

	@@ -0,0 +1,6 @@

+0
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-d2b0673e-c3cb-a631-b7b0-f33f88b95388] from parent process environment
+FATAL:   while loading overlay images: failed to open overlay image /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img: while locking ext3 partition from /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img: can't open /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img for writing, currently in use by another process

DiffTumor.o425447 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425448 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425449 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425450 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425451 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425452 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425453 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425454 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425455 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425456 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425457 ADDED Viewed

	@@ -0,0 +1,9 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8

DiffTumor.o425458 ADDED Viewed

	@@ -0,0 +1,10 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory

DiffTumor.o425459 ADDED Viewed

	@@ -0,0 +1,10 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory

DiffTumor.o425460 ADDED Viewed

	@@ -0,0 +1,10 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory

DiffTumor.o425461 ADDED Viewed

	@@ -0,0 +1,10 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory

DiffTumor.o425463 ADDED Viewed

	@@ -0,0 +1,10 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory

DiffTumor.o426418 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+/var/spool/pbs/mom_priv/jobs/426418.pbs01.priv.franklin.lan.SC: line 70: syntax error: unexpected end of file

DiffTumor.o426420 ADDED Viewed

	@@ -0,0 +1,12 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory tmp_dir has been removed.
+/var/spool/pbs/mom_priv/jobs/426420.pbs01.priv.franklin.lan.SC: line 70: syntax error: unexpected end of file

DiffTumor.o426421 ADDED Viewed

	@@ -0,0 +1,12 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory tmp_dir has been removed.
+/var/spool/pbs/mom_priv/jobs/426421.pbs01.priv.franklin.lan.SC: line 70: syntax error: unexpected end of file

DiffTumor.o426422 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+/var/spool/pbs/mom_priv/jobs/426422.pbs01.priv.franklin.lan.SC: line 71: syntax error: unexpected end of file

DiffTumor.o426423 ADDED Viewed

	@@ -0,0 +1,12 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.
+=>> PBS: job killed: walltime 991 exceeded limit 900

DiffTumor.o426426 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumor.o426427 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumor.o426428 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumor.o426604 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumor.o426817 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumor.o426868 ADDED Viewed

	@@ -0,0 +1,12 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.
+rm: cannot remove '/fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir/case00010.nii.gz': No such file or directory

DiffTumor.o429178 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumor.o429215 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumor.sh ADDED Viewed

	@@ -0,0 +1,71 @@

+#!/bin/sh
+#PBS -l select=1:ncpus=10:ngpus=1
+#PBS -l walltime=23:59:00
+#PBS -j oe
+#PBS -N DiffTumor
+#PBS -q gpu
+module load nvhpc-hpcx-cuda11/23.5
+module load go-1.19.4/singularity-1.1.8
+cd /work/psalvador
+output_file="DiffTumordb.txt"
+echo "0"> "$output_file"
+#Number of processes per node to launch
+NPROC_PER_NODE=4
+#Number of process in all modes
+WORLD_SIZE=$PBS_NUM_NODES
+MASTER_ADDR=`/bin/hostname -s`
+cat $PBS_NODEFILE>nodelist
+#Make sure this node (MASTER) comes first
+SLAVES=`cat nodelist | grep -v $MASTER_ADDR | uniq`
+#We want names of master and slave nodes
+HOSTLIST="$MASTER_ADDR $SLAVES"
+#Get free port:
+BASE_PORT=16998
+INCREMENT=1
+port=$BASE_PORT
+isfree=$(netstat -taln | grep $port)
+while [[ -n "$isfree" ]]; do
+    port=$[port+INCREMENT]
+    isfree=$(netstat -taln | grep $port)
+done
+MASTER_PORT=$port
+export NCCL_DEBUG=INFO
+arr=($HOSTLIST)
+export TORCH_HOME=/fastwork/psalvador/cache
+export TORCHVISION_HOME=/fastwork/psalvador/cache
+#This command to run your pytorch script
+export inputs_data=/fastwork/psalvador/JHU/data/SingleFolderUCSFRenamed
+export outputs_data=/fastwork/psalvador/JHU/UCSF/DiffTumor/outputs
+chmod 644 overlay.img
+cd /fastwork/psalvador/JHU/UCSF/DiffTumor
+echo "0"> "$output_file"
+export dir=/fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir
+# Check if the directory exists
+if [ -d "$dir" ]; then
+    # If the directory exists, remove it
+    rm -rf "$dir"
+    echo "Directory $dir has been removed."
+fi
+# Create the directory
+mkdir -p "$dir"
+for ct_file in "$inputs_data"/*.nii.gz; do
+  if [ -f "$ct_file" ]; then
+    cp "$ct_file" "$dir/$(basename "$ct_file")"
+    SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity run --fakeroot --overlay /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img --nv -B $dir:/workspace/inputs -B $outputs_data:/workspace/outputs difftumor_final.sif>> "$output_file" 2>&1
+    rm "$dir/$(basename "$ct_file")"
+  fi
+done

DiffTumor.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ 0

DiffTumorHealthy.o426937 ADDED Viewed

	@@ -0,0 +1,11 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory
+Directory /fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir has been removed.

DiffTumorHealthy.o426938 ADDED Viewed

	@@ -0,0 +1,10 @@

+Loading nvhpc-hpcx-cuda11/23.5
+  ERROR: Directory
+    '/applications/sdk/nvidia/hpc_sdk/Linux_x86_64/23.5/comm_libs/11.8/hpcx/latest/modulefiles' not found
+  ERROR: Unable to locate a modulefile for 'hpcx'
+  ERROR: Load of requirement 'hpcx' failed
+*** WARNING: Deprecated module name, singularity project has been renamed into apptainer.
+Loading go-1.19.4/singularity-1.1.8
+  Loading requirement: go-1.19.4/go go-1.19.4/apptainer-1.1.8
+chmod: cannot access 'overlay.img': No such file or directory

DiffTumorHealthy.sh ADDED Viewed

	@@ -0,0 +1,71 @@

+#!/bin/sh
+#PBS -l select=1:ncpus=10:ngpus=1
+#PBS -l walltime=23:59:00
+#PBS -j oe
+#PBS -N DiffTumorHealthy
+#PBS -q gpu
+module load nvhpc-hpcx-cuda11/23.5
+module load go-1.19.4/singularity-1.1.8
+cd /work/psalvador
+output_file="DiffTumorHealthy.txt"
+echo "0"> "$output_file"
+#Number of processes per node to launch
+NPROC_PER_NODE=4
+#Number of process in all modes
+WORLD_SIZE=$PBS_NUM_NODES
+MASTER_ADDR=`/bin/hostname -s`
+cat $PBS_NODEFILE>nodelist
+#Make sure this node (MASTER) comes first
+SLAVES=`cat nodelist | grep -v $MASTER_ADDR | uniq`
+#We want names of master and slave nodes
+HOSTLIST="$MASTER_ADDR $SLAVES"
+#Get free port:
+BASE_PORT=16998
+INCREMENT=1
+port=$BASE_PORT
+isfree=$(netstat -taln | grep $port)
+while [[ -n "$isfree" ]]; do
+    port=$[port+INCREMENT]
+    isfree=$(netstat -taln | grep $port)
+done
+MASTER_PORT=$port
+export NCCL_DEBUG=INFO
+arr=($HOSTLIST)
+export TORCH_HOME=/fastwork/psalvador/cache
+export TORCHVISION_HOME=/fastwork/psalvador/cache
+#This command to run your pytorch script
+export inputs_data=/fastwork/psalvador/JHU/data/TSHealthyRenamed
+export outputs_data=/fastwork/psalvador/JHU/UCSF/DiffTumor/outputsHealthy
+chmod 644 overlay.img
+cd /fastwork/psalvador/JHU/UCSF/DiffTumor
+echo "0"> "$output_file"
+export dir=/fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir_healthy
+# Check if the directory exists
+if [ -d "$dir" ]; then
+    # If the directory exists, remove it
+    rm -rf "$dir"
+    echo "Directory $dir has been removed."
+fi
+# Create the directory
+mkdir -p "$dir"
+for ct_file in "$inputs_data"/*.nii.gz; do
+  if [ -f "$ct_file" ]; then
+    cp "$ct_file" "$dir/$(basename "$ct_file")"
+    SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity run --fakeroot --overlay /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img --nv -B $dir:/workspace/inputs -B $outputs_data:/workspace/outputs difftumor_final.sif>> "$output_file" 2>&1
+    rm "$dir/$(basename "$ct_file")"
+  fi
+done

DiffTumorHealthy.txt ADDED Viewed

@@ -0,0 +1,882 @@
  0%|          | 0/1 [00:00<?, ?it/s]['case00001']
  0%|          | 0/1 [00:23<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00002']
  0%|          | 0/1 [00:37<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00003']
  0%|          | 0/1 [00:10<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00004']
  0%|          | 0/1 [00:09<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00005']
  0%|          | 0/1 [00:29<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00006']
  0%|          | 0/1 [00:14<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00007']
  0%|          | 0/1 [00:32<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00008']
  0%|          | 0/1 [00:22<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00009']
  0%|          | 0/1 [00:37<?, ?it/s]
  0%|          | 0/1 [00:00<?, ?it/s]['case00010']
  0%|          | 0/1 [00:22<?, ?it/s]

+0
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    Warning: Mounting unchecked fs, running e2fsck is recommended.
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00001']
+2024-07-12 16:39:06,081 INFO image_writer.py:190 - writing: /workspace/outputs/case00001/predictions/case00001_liver.nii.gz
+2024-07-12 16:39:06,299 INFO image_writer.py:190 - writing: /workspace/outputs/case00001/predictions/case00001_pancreas.nii.gz
+2024-07-12 16:39:06,506 INFO image_writer.py:190 - writing: /workspace/outputs/case00001/predictions/case00001_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:23<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00001
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00001
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00001
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00002']
+2024-07-12 16:41:57,917 INFO image_writer.py:190 - writing: /workspace/outputs/case00002/predictions/case00002_liver.nii.gz
+2024-07-12 16:41:58,310 INFO image_writer.py:190 - writing: /workspace/outputs/case00002/predictions/case00002_pancreas.nii.gz
+2024-07-12 16:41:58,712 INFO image_writer.py:190 - writing: /workspace/outputs/case00002/predictions/case00002_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:37<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00002
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00002
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00002
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00003']
+2024-07-12 16:45:24,121 INFO image_writer.py:190 - writing: /workspace/outputs/case00003/predictions/case00003_liver.nii.gz
+2024-07-12 16:45:24,225 INFO image_writer.py:190 - writing: /workspace/outputs/case00003/predictions/case00003_pancreas.nii.gz
+2024-07-12 16:45:24,321 INFO image_writer.py:190 - writing: /workspace/outputs/case00003/predictions/case00003_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:10<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00003
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00003
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00003
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00004']
+2024-07-12 16:46:49,275 INFO image_writer.py:190 - writing: /workspace/outputs/case00004/predictions/case00004_liver.nii.gz
+2024-07-12 16:46:49,361 INFO image_writer.py:190 - writing: /workspace/outputs/case00004/predictions/case00004_pancreas.nii.gz
+2024-07-12 16:46:49,442 INFO image_writer.py:190 - writing: /workspace/outputs/case00004/predictions/case00004_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:09<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00004
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00004
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00004
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00005']
+2024-07-12 16:48:27,374 INFO image_writer.py:190 - writing: /workspace/outputs/case00005/predictions/case00005_liver.nii.gz
+2024-07-12 16:48:27,609 INFO image_writer.py:190 - writing: /workspace/outputs/case00005/predictions/case00005_pancreas.nii.gz
+2024-07-12 16:48:27,828 INFO image_writer.py:190 - writing: /workspace/outputs/case00005/predictions/case00005_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:29<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00005
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00005
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00005
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00006']
+2024-07-12 16:50:57,551 INFO image_writer.py:190 - writing: /workspace/outputs/case00006/predictions/case00006_liver.nii.gz
+2024-07-12 16:50:57,642 INFO image_writer.py:190 - writing: /workspace/outputs/case00006/predictions/case00006_pancreas.nii.gz
+2024-07-12 16:50:57,726 INFO image_writer.py:190 - writing: /workspace/outputs/case00006/predictions/case00006_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:14<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00006
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00006
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00006
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00007']
+2024-07-12 16:52:45,064 INFO image_writer.py:190 - writing: /workspace/outputs/case00007/predictions/case00007_liver.nii.gz
+2024-07-12 16:52:45,409 INFO image_writer.py:190 - writing: /workspace/outputs/case00007/predictions/case00007_pancreas.nii.gz
+2024-07-12 16:52:45,770 INFO image_writer.py:190 - writing: /workspace/outputs/case00007/predictions/case00007_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:32<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00007
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00007
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00007
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00008']
+2024-07-12 16:56:26,335 INFO image_writer.py:190 - writing: /workspace/outputs/case00008/predictions/case00008_liver.nii.gz
+2024-07-12 16:56:26,507 INFO image_writer.py:190 - writing: /workspace/outputs/case00008/predictions/case00008_pancreas.nii.gz
+2024-07-12 16:56:26,678 INFO image_writer.py:190 - writing: /workspace/outputs/case00008/predictions/case00008_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:22<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00008
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00008
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00008
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00009']
+2024-07-12 16:58:50,429 INFO image_writer.py:190 - writing: /workspace/outputs/case00009/predictions/case00009_liver.nii.gz
+2024-07-12 16:58:50,779 INFO image_writer.py:190 - writing: /workspace/outputs/case00009/predictions/case00009_pancreas.nii.gz
+2024-07-12 16:58:51,124 INFO image_writer.py:190 - writing: /workspace/outputs/case00009/predictions/case00009_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:37<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00009
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00009
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00009
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-20a518b1-6085-00e6-5eab-26662b6544db] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00010']
+2024-07-12 17:02:28,247 INFO image_writer.py:190 - writing: /workspace/outputs/case00010/predictions/case00010_liver.nii.gz
+2024-07-12 17:02:28,398 INFO image_writer.py:190 - writing: /workspace/outputs/case00010/predictions/case00010_pancreas.nii.gz
+2024-07-12 17:02:28,537 INFO image_writer.py:190 - writing: /workspace/outputs/case00010/predictions/case00010_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:22<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00010
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00010
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00010

DiffTumorNew.sh ADDED Viewed

	@@ -0,0 +1,72 @@

+#!/bin/sh
+#PBS -l select=1:ncpus=10:ngpus=1
+#PBS -l walltime=00:15:00
+#PBS -j oe
+#PBS -N DiffTumor
+#PBS -q debug
+module load nvhpc-hpcx-cuda11/23.5
+module load go-1.19.4/singularity-1.1.8
+cd /work/psalvador
+output_file="DiffTumordb.txt"
+echo "0"> "$output_file"
+#Number of processes per node to launch
+NPROC_PER_NODE=4
+#Number of process in all modes
+WORLD_SIZE=$PBS_NUM_NODES
+MASTER_ADDR=`/bin/hostname -s`
+cat $PBS_NODEFILE>nodelist
+#Make sure this node (MASTER) comes first
+SLAVES=`cat nodelist | grep -v $MASTER_ADDR | uniq`
+#We want names of master and slave nodes
+HOSTLIST="$MASTER_ADDR $SLAVES"
+#Get free port:
+BASE_PORT=16998
+INCREMENT=1
+port=$BASE_PORT
+isfree=$(netstat -taln | grep $port)
+while [[ -n "$isfree" ]]; do
+    port=$[port+INCREMENT]
+    isfree=$(netstat -taln | grep $port)
+done
+MASTER_PORT=$port
+export NCCL_DEBUG=INFO
+arr=($HOSTLIST)
+export TORCH_HOME=/fastwork/psalvador/cache
+export TORCHVISION_HOME=/fastwork/psalvador/cache
+#This command to run your pytorch script
+export inputs_data=/fastwork/psalvador/JHU/data/QeTOY4UZn3_8TfcZajFaf
+export outputs_data=/fastwork/psalvador/JHU/UCSF/DiffTumor/QeTOY4UZn3_8TfcZajFaf_10
+mkdir -p $outputs_data
+chmod 644 overlay.img
+cd /fastwork/psalvador/JHU/UCSF/DiffTumor
+echo "0"> "$output_file"
+export dir=/fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir
+# Check if the directory exists
+if [ -d "$dir" ]; then
+    # If the directory exists, remove it
+    rm -rf "$dir"
+    echo "Directory $dir has been removed."
+fi
+# Create the directory
+mkdir -p "$dir"
+for ct_file in "$inputs_data"/*.nii.gz; do
+  if [ -f "$ct_file" ]; then
+    cp "$ct_file" "$dir/$(basename "$ct_file")"
+    SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity run --fakeroot --overlay /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img --nv -B $dir:/workspace/inputs -B $outputs_data:/workspace/outputs difftumor_final.sif>> "$output_file" 2>&1
+    rm "$dir/$(basename "$ct_file")"
+  fi
+done

DiffTumorSingle.sh ADDED Viewed

	@@ -0,0 +1,73 @@

+#!/bin/sh
+#PBS -l select=1:ncpus=10:ngpus=1
+#PBS -l walltime=23:59:00
+#PBS -j oe
+#PBS -N ADiffTumor
+#PBS -q gpu
+module load nvhpc-hpcx-cuda11/23.5
+module load go-1.19.4/singularity-1.1.8
+cd /work/psalvador
+output_file="ADiffTumordb.txt"
+echo "0"> "$output_file"
+#Number of processes per node to launch
+NPROC_PER_NODE=4
+#Number of process in all modes
+WORLD_SIZE=$PBS_NUM_NODES
+MASTER_ADDR=`/bin/hostname -s`
+cat $PBS_NODEFILE>nodelist
+#Make sure this node (MASTER) comes first
+SLAVES=`cat nodelist | grep -v $MASTER_ADDR | uniq`
+#We want names of master and slave nodes
+HOSTLIST="$MASTER_ADDR $SLAVES"
+#Get free port:
+BASE_PORT=16998
+INCREMENT=1
+port=$BASE_PORT
+isfree=$(netstat -taln | grep $port)
+while [[ -n "$isfree" ]]; do
+    port=$[port+INCREMENT]
+    isfree=$(netstat -taln | grep $port)
+done
+MASTER_PORT=$port
+export NCCL_DEBUG=INFO
+arr=($HOSTLIST)
+export TORCH_HOME=/fastwork/psalvador/cache
+export TORCHVISION_HOME=/fastwork/psalvador/cache
+#This command to run your pytorch script
+export inputs_data=/fastwork/psalvador/JHU/data/UCSF_remaining/SXz0J6vDWz/SXz0J6vDWz/czfrhBRDVv
+export outputs_data=/fastwork/psalvador/JHU/UCSF/DiffTumor/outputs/SXz0J6vDWz_czfrhBRDVv
+mkdir -p "$outputs_data"
+chmod 644 overlay.img
+cd /fastwork/psalvador/JHU/UCSF/DiffTumor
+echo "0"> "$output_file"
+export dir=/fastwork/psalvador/JHU/UCSF/DiffTumor/tmp_dir
+# Check if the directory exists
+if [ -d "$dir" ]; then
+    # If the directory exists, remove it
+    rm -rf "$dir"
+    echo "Directory $dir has been removed."
+fi
+# Create the directory
+mkdir -p "$dir"
+for ct_file in "$inputs_data"/*.nii.gz; do
+  if [ -f "$ct_file" ]; then
+    cp "$ct_file" "$dir/$(basename "$ct_file")"
+    SINGULARITYENV_CUDA_VISIBLE_DEVICES=0 singularity run --fakeroot --overlay /fastwork/psalvador/JHU/UCSF/DiffTumor/overlay.img --nv -B $dir:/workspace/inputs -B $outputs_data:/workspace/outputs difftumor_final.sif>> "$output_file" 2>&1
+    rm "$dir/$(basename "$ct_file")"
+  fi
+done

DiffTumorUCSFTest.sh ADDED Viewed

	@@ -0,0 +1,26 @@

+#!/bin/bash
+module load go-1.19.4/singularity-1.1.8
+parts=$1
+part=$2
+GPU=$3
+cd /fastwork/psalvador/JHU/UCSF/DiffTumor/
+CONVERTED=/fastwork/psalvador/JHU/data/Atlas_3_test/
+OUTPUTS=/fastwork/psalvador/JHU/UCSF/DiffTumor/outputs_difftumor_UCSF_test/
+export GPU
+mkdir -p $OUTPUTS
+logs_file="logs_difftumor_atlas_test.txt"
+echo 0 > $logs_file
+export SINGULARITYENV_CUDA_VISIBLE_DEVICES=$GPU
+export GPU
+echo "Arguments received:"
+echo "CONVERTED=$CONVERTED"
+echo "OUTPUTS=$OUTPUTS"
+echo "parts=$parts"
+echo "part=$part"
+bash RunDifftumor.sh $CONVERTED $OUTPUTS $parts $part #>> "$logs_file" 2>&1

DiffTumordb.txt ADDED Viewed

@@ -0,0 +1,89 @@
  0%|          | 0/1 [00:00<?, ?it/s]['case00011']
  0%|          | 0/1 [00:36<?, ?it/s]

+0
+INFO:    User not listed in /etc/subuid, trying root-mapped namespace
+INFO:    Using fakeroot command combined with root-mapped namespace
+INFO:    Environment variable SINGULARITYENV_CUDA_VISIBLE_DEVICES is set, but APPTAINERENV_CUDA_VISIBLE_DEVICES is preferred
+WARNING: Environment variable CUDA_VISIBLE_DEVICES already has value [0], will not forward new value [GPU-01992259-ec22-343e-e192-e276333d5c35] from parent process environment
+INFO:    unknown argument ignored: lazytime
+unknown argument ignored: xino=on
+/proc seems to be mounted as readonly, it can lead to unexpected failures
+Requirement already satisfied: connected-components-3d in /workspace/anaconda3/lib/python3.8/site-packages (3.17.0)
+Requirement already satisfied: fastremap in /workspace/anaconda3/lib/python3.8/site-packages (1.14.2)
+Requirement already satisfied: simpleitk in /workspace/anaconda3/lib/python3.8/site-packages (2.1.1.2)
+Requirement already satisfied: numpy in /workspace/anaconda3/lib/python3.8/site-packages (from connected-components-3d) (1.23.0)
+DEPRECATION: pytorch-lightning 1.6.4 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
+WARNING: There was an error checking the latest version of pip.
+Use pretrained weights
+test len 1
  0%|          | 0/1 [00:00<?, ?it/s]['case00011']
+2024-07-18 00:30:04,723 INFO image_writer.py:190 - writing: /workspace/outputs/case00011/predictions/case00011_liver.nii.gz
+2024-07-18 00:30:07,352 INFO image_writer.py:190 - writing: /workspace/outputs/case00011/predictions/case00011_pancreas.nii.gz
+2024-07-18 00:30:09,941 INFO image_writer.py:190 - writing: /workspace/outputs/case00011/predictions/case00011_kidney.nii.gz
+/workspace/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py:1960: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+> /workspace/difftumor/organ_mask_access/organ_test.py(30)validation()
+-> for index, batch in enumerate(tqdm(ValLoader)):
+(Pdb)
  0%|          | 0/1 [00:36<?, ?it/s]
+Traceback (most recent call last):
+  File "organ_mask_access/organ_test.py", line 135, in <module>
+    main()
+  File "organ_mask_access/organ_test.py", line 132, in main
+    validation(model, test_loader, val_transforms, args)
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "organ_mask_access/organ_test.py", line 30, in validation
+    for index, batch in enumerate(tqdm(ValLoader)):
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 88, in trace_dispatch
+    return self.dispatch_line(frame)
+  File "/workspace/anaconda3/lib/python3.8/bdb.py", line 113, in dispatch_line
+    if self.quitting: raise BdbQuit
+bdb.BdbQuit
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => liver
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00011
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => pancreas
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00011
+MAIN Argument values:
+data_root => /workspace/inputs
+organ_type => kidney
+fold => 0
+save_dir => /workspace/outputs
+feature_size => 16
+val_overlap => 0.75
+num_classes => 3
+model => unet
+swin_type => base
+-----------------
+val len 1
+unet
+Use logdir weights
+Total parameters count 4807482
+case_name case00011

RunDifftumor copy.sh ADDED Viewed

	@@ -0,0 +1,130 @@

+#!/bin/bash
+# Get the path to destination_folder as input
+destination_folder=$1
+# Set the path for the output folder
+output_folder=$2
+# Create the output folder if it doesn't exist
+mkdir -p "$output_folder"
+export NCCL_DEBUG=INFO
+# cache directories
+export TORCH_HOME=cache
+export TORCHVISION_HOME=cache
+# Temporary directories for input and output
+export temp_input_dir=tmp_inputs
+export temp_output_dir=tmp_outputs
+mkdir -p "$temp_input_dir"
+mkdir -p "$temp_output_dir"
+mkdir -p "$TORCH_HOME"
+mkdir -p "$TORCHVISION_HOME"
+# Function to copy and rename files for the segmentation model
+prepare_input_file() {
+    local input_file=$1
+    local temp_case_file="$temp_input_dir/case00001.nii.gz"
+    echo "Copying $input_file to $temp_case_file"
+    cp "$input_file" "$temp_case_file"
+    if [ $? -ne 0 ]; then
+        echo "Failed to copy $input_file to $temp_case_file"
+    else
+        echo "Successfully copied $input_file to $temp_case_file"
+    fi
+}
+# Function to run the segmentation model
+run_segmentation() {
+    echo "Running segmentation model on $temp_input_dir/case00001.nii.gz"
+    #print which gpu is being used
+    echo "Using GPU: $GPU"
+    export PYTHONBREAKPOINT=0
+    APPTAINERENV_CUDA_VISIBLE_DEVICES=$GPU singularity run --overlay overlay.img:ro --nv -B $temp_input_dir:/workspace/inputs -B $temp_output_dir:/workspace/outputs difftumor_final.sif
+    if [ $? -ne 0 ]; then
+        echo "Segmentation model failed on $temp_input_dir/case00001.nii.gz"
+        return 1
+    else
+        echo "Segmentation model succeeded on $temp_input_dir/case00001.nii.gz"
+        return 0
+    fi
+}
+# Function to move the results to the final output folder
+move_results() {
+    local series_folder=$1
+    local output_series_folder="$output_folder/${series_folder#$destination_folder}"
+    echo "Moving results to $output_series_folder"
+    mkdir -p "$output_series_folder/predictions"
+    if [ -f "$temp_output_dir/case00001/ct.nii.gz" ]; then
+        echo "Found ct.nii.gz for $series_folder"
+        cp "$temp_output_dir/case00001/ct.nii.gz" "$output_series_folder/"
+        if [ $? -ne 0 ]; then
+            echo "Failed to copy ct.nii.gz for $series_folder"
+        else
+            echo "Successfully copied ct.nii.gz for $series_folder"
+        fi
+    else
+        echo "No ct.nii.gz file found for $series_folder"
+    fi
+    if [ -d "$temp_output_dir/case00001/predictions" ]; then
+        echo "Found predictions folder for $series_folder"
+        cp "$temp_output_dir/case00001/predictions/"* "$output_series_folder/predictions/"
+        if [ $? -ne 0 ]; then
+            echo "Failed to copy predictions for $series_folder"
+        else
+            echo "Successfully copied predictions for $series_folder"
+        fi
+    else
+        echo "No predictions folder found for $series_folder"
+    fi
+}
+# Clean up temporary directories
+clean_temp_dirs() {
+    echo "Cleaning temporary directories"
+    rm -rf "$temp_input_dir"/*
+    if [ $? -ne 0 ]; then
+        echo "Failed to clean $temp_input_dir"
+    else
+        echo "Successfully cleaned $temp_input_dir"
+    fi
+    rm -rf "$temp_output_dir"/*
+    if [ $? -ne 0 ]; then
+        echo "Failed to clean $temp_output_dir"
+    else
+        echo "Successfully cleaned $temp_output_dir"
+    fi
+}
+# Find all .nii.gz files and store them in an array
+mapfile -t ct_files < <(find "$destination_folder" -type f -name "ct.nii.gz")
+# Print the number of files found
+echo "Found ${#ct_files[@]} .nii.gz files"
+#print first 10 files
+echo "First 10 files: ${ct_files[@]:0:10}"
+# Process each .nii.gz file in the array
+for ct_file in "${ct_files[@]}"; do
+    if [ -f "$ct_file" ]; then
+        series_folder=$(dirname "$ct_file")
+        echo "Processing $ct_file in $series_folder"
+        prepare_input_file "$ct_file"
+        run_segmentation
+        if [ $? -ne 0 ]; then
+            echo "Skipping $ct_file due to segmentation error"
+            continue
+        fi
+        move_results "$series_folder"
+        clean_temp_dirs
+    fi
+done
+# Final cleanup
+echo "Final cleanup"
+rm -rf "$temp_input_dir"
+rm -rf "$temp_output_dir"

logs_difftumor_atlas_test.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a6a67a2d70af5cdc10db27561cbe6762569f1a555ff32fe1378421dcf56dac4e
+size 309301

my_container_sandbox/.exec ADDED Viewed

	@@ -0,0 +1,9 @@

+#!/bin/sh
+for script in /.singularity.d/env/*.sh; do
+    if [ -f "$script" ]; then
+        . "$script"
+    fi
+done
+exec "$@"

my_container_sandbox/.run ADDED Viewed

	@@ -0,0 +1,23 @@

+#!/bin/sh
+for script in /.singularity.d/env/*.sh; do
+    if [ -f "$script" ]; then
+        . "$script"
+    fi
+done
+if test -n "${SINGULARITY_APPNAME:-}"; then
+    if test -x "/scif/apps/${SINGULARITY_APPNAME:-}/scif/runscript"; then
+        exec "/scif/apps/${SINGULARITY_APPNAME:-}/scif/runscript" "$@"
+    else
+        echo "No Singularity runscript for contained app: ${SINGULARITY_APPNAME:-}"
+        exit 1
+    fi
+elif test -x "/.singularity.d/runscript"; then
+    exec "/.singularity.d/runscript" "$@"
+else
+    echo "No Singularity runscript found, executing /bin/sh"
+    exec /bin/sh "$@"
+fi