32353789

diffusion.py

@@ -278,7 +278,7 @@ class TrainConfig:
     # seed = 0
     # save_dir = './outputs/'
 
-    save_period = 5 #np.infty #n_epoch // 2 #np.infty#.1 # the period of sampling
+    save_period = 10 #np.infty #n_epoch // 2 #np.infty#.1 # the period of sampling
     # general parameters for the name and logger    
     # device = "cuda" if torch.cuda.is_available() else "cpu"
     lrate = 1e-4

perlmutter_diffusion.sbatch

@@ -5,7 +5,7 @@
 #SBATCH -q regular #shared 
 #SBATCH -N1
 #SBATCH --gpus-per-node=4
-#SBATCH -t 16:00
+#SBATCH -t 3:00:00
 #SBATCH --ntasks-per-node=1
 #SBATCH -oReport-%j 
 #SBATCH --mail-type=BEGIN,END,FAIL
@@ -39,7 +39,7 @@ srun python diffusion.py \
     --use_checkpoint 1 \
     --dropout 0.2 \
     --lrate 2e-5 \
-    --train "$SCRATCH/LEN128-DIM64-CUB16-Tvir[4, 6]-zeta[10, 250]-0809-123640.h5" \
-    #--resume ./outputs/model-N1600-device_count4-node4-epoch19-32257242 \
+    --resume ./outputs/model-N1600-device_count4-node4-epoch14-32353762 \
+    #--train "$SCRATCH/LEN128-DIM64-CUB16-Tvir[4, 6]-zeta[10, 250]-0809-123640.h5" \
 
 date