20051216

diffusion_test.py

@@ -495,7 +495,7 @@ class DDPM21CM:
             pbar_train = tqdm(total=len(self.dataloader), file=sys.stderr, disable=True)#, mininterval=self.config.pbar_update_step)#, disable=True)#not self.accelerator.is_local_main_process)
             pbar_train.set_description(f"{socket.gethostbyname(socket.gethostname())} cuda:{torch.cuda.current_device()}/{self.config.global_rank} Epoch {ep}")
             epoch_start = time()
-            print(f"epoch = {ep}")
+            #print(f"epoch = {ep}")
             for i, (x, c) in enumerate(self.dataloader):
                 x = self.transform(x)
                 x = x.to(self.config.device)#.to(self.config.dtype)

perlmutter_diffusion.sbatch

@@ -35,10 +35,10 @@ srun python diffusion.py \
     --num_new_img_per_gpu 4 \
     --max_num_img_per_gpu 2 \
     --gradient_accumulation_steps 1 \
-    --autocast 1 \
+    --autocast 0 \
     --use_checkpoint 1 \
     --dropout 0.1 \
-    --resume ./outputs/model-N1600-device_count4-node4-epoch9-20051216 \
+    --resume ./outputs/model-N1600-device_count4-node4-epoch29-20051216 \
     #--train "$SCRATCH/LEN128-DIM64-CUB16-Tvir[4, 6]-zeta[10, 250]-0809-123640.h5" \
 
 date

quantify_results.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c4959c36d99047fc29ed5ded2317a3f74e5fbca497e8848778ed4e86c41e5f56
-size 12046825
+oid sha256:a0c13ff1f79531d58b80a67cf6ae9141685d85483eac6006d7fc90f33d55283e
+size 28539850