0802-230641

diffusion.py

@@ -257,12 +257,12 @@ class TrainConfig:
 
     # dim = 2
     dim = 2
-    stride = (2,2) if dim == 2 else (2,2,2)
-    num_image = 3000#6000#30#60#6000#1000#2000#20000#15000#7000#25600#3000#10000#1000#10000#5000#2560#800#2560
-    batch_size = 30#50#10#50#20#50#1#2#50#20#2#100 # 10
-    n_epoch = 50#5#50#100#50#100#30#120#5#4# 10#50#20#20#2#5#25 # 120
+    stride = (2,4) if dim == 2 else (2,2,2)
+    num_image = 30#3000#6000#30#60#6000#1000#2000#20000#15000#7000#25600#3000#10000#1000#10000#5000#2560#800#2560
+    batch_size = 10#50#10#50#20#50#1#2#50#20#2#100 # 10
+    n_epoch = 5#50#5#50#100#50#100#30#120#5#4# 10#50#20#20#2#5#25 # 120
     HII_DIM = 64
-    num_redshift = 64#512#64#512#64#256CUDAoom#128#64#512#128#64#512#256#256#64#512#128
+    num_redshift = 512#64#512#64#512#64#256CUDAoom#128#64#512#128#64#512#256#256#64#512#128
     channel = 1
     img_shape = (channel, HII_DIM, num_redshift) if dim == 2 else (channel, HII_DIM, HII_DIM, num_redshift)
 

phoenix_diffusion.sbatch

@@ -2,10 +2,10 @@
 #SBATCH -J diffusion # Job name
 #SBATCH -A gts-jw254-coda20 
 #SBATCH -qembers
-#SBATCH -N6 --gpus-per-node=RTX_6000:1 # -C A100-80GB              # Number of nodes and cores per node required
+#SBATCH -N1 --gpus-per-node=RTX_6000:1 # -C A100-80GB              # Number of nodes and cores per node required
 #SBATCH --ntasks-per-node=1
-#SBATCH --mem-per-gpu=8G                        # Memory per core
-#SBATCH -t 01:00:00                                    # Duration of the job (Ex: 15 mins)
+#SBATCH --mem-per-gpu=16G                        # Memory per core
+#SBATCH -t 00:05:00                                    # Duration of the job (Ex: 15 mins)
 #SBATCH -oReport-%j                         # Combined output and error messages file
 #SBATCH --error=error-%j
 #SBATCH --mail-type=BEGIN,END,FAIL              # Mail preferences
@@ -31,7 +31,7 @@ export MASTER_PORT=$MASTER_PORT
 srun python diffusion.py \
     --train 1 \
     --sample 0 \
-    --resume outputs/model_state-N3000-device_count2-node2-epoch49-172.27.149.189 \
+    --resume outputs/model_state-N3000-device_count3-node2-epoch49-172.27.145.67 \
     --num_new_img_per_gpu 200 \
     --max_num_img_per_gpu 40 \