05140800

diffusion.py

@@ -513,11 +513,16 @@ class DDPM21CM:
         #print(f"cuda:{torch.cuda.current_device()}/{self.config.global_rank} lr_scheduler: {self.lr_scheduler.optimizer is self.optimizer}", f"{time()-lr_start:.3f}s")
         #print(f"cuda:{torch.cuda.current_device()}/{self.config.global_rank} print costs {print_end-print_start:.3f}s")
 
+        print(f"cuda:{torch.cuda.current_device()}/{self.config.global_rank}")
         acc_prep_start = time()
-        self.nn_model, self.optimizer, self.dataloader, self.lr_scheduler = \
-            self.accelerator.prepare(
-            self.nn_model, self.optimizer, self.dataloader, self.lr_scheduler
-            )
+        #self.nn_model, self.optimizer, self.dataloader, self.lr_scheduler = \
+        #    self.accelerator.prepare(
+        #    self.nn_model, self.optimizer, self.dataloader, self.lr_scheduler
+        #    )
+        self.nn_model = self.accelerator.prepare(self.nn_model)
+        self.optimizer = self.accelerator.prepare(self.optimizer)
+        self.dataloader = self.accelerator.prepare(self.dataloader)
+        self.lr_scheduler = self.accelerator.prepare(self.lr_scheduler)
         acc_prep_end = time()
         print(f"cuda:{torch.cuda.current_device()}/{self.config.global_rank} accelerate.prepare cost {acc_prep_end-acc_prep_start:.3f}s")
         # self.nn_model, self.optimizer, self.lr_scheduler = \

perlmutter_diffusion.sbatch

@@ -3,7 +3,7 @@
 #SBATCH -J diffusion
 #SBATCH -C gpu
 #SBATCH -q debug
-#SBATCH -N2 
+#SBATCH -N1 
 #SBATCH --gpus-per-node=2
 #SBATCH -t 0:05:00
 #SBATCH --ntasks-per-node=1
@@ -14,11 +14,11 @@
 date
 #module load anaconda3/2022.05 # Load module dependencies
 module load pytorch #/2.0.1
-#conda activate diffusion #diffusers 
+#conda activate diffusers 
 conda env list
 module list
 which python
-srun python -c "import torch; print(torch.cuda.device_count(), torch.__path__, torch.version.cuda); import accelerate; print(accelerate.__version__, accelerate.__path__)"
+srun python -c "import torch; print('device_count', torch.cuda.device_count(), 'torch.__path__', torch.__path__, 'cuda version', torch.version.cuda); import accelerate; print('accelerate.__version', accelerate.__version__, 'accelerate.__path__', accelerate.__path__)"
 cat $0
 
 MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)
@@ -29,6 +29,10 @@ export MASTER_PORT=$MASTER_PORT
 #export SLURM_CPU_BIND="cores"
 echo $MASTER_ADDR
 echo $MASTER_PORT
+nc -zv $MASTER_ADDR $MASTER_PORT
+
+export NCCL_DEBUG=INFO
+export NCCL_DEBUG_SUBSYS=ALL
 
 srun python diffusion.py \
     --train "$SCRATCH/LEN128-DIM64-CUB16-Tvir[4, 6]-zeta[10, 250]-0809-123640.h5" \