0728-1034

diffusion.py

@@ -651,12 +651,12 @@ class DDPM21CM:
 
 #num_train_image_list = [6000]#[60]#[8000]#[1000]#[100]#
 def train(rank, world_size, local_world_size, master_addr, master_port):
-    #print("before ddp_setup")
+    print("before ddp_setup")
     ddp_setup(rank, world_size, local_world_size, master_addr, master_port)
-    #print("after ddp_setup")
+    print("after ddp_setup")
     local_rank = rank % local_world_size
     torch.cuda.set_device(local_rank)
-    #print("after set device")
+    print("after set device")
     print(f"rank = {rank}, local_rank = {local_rank}, world_size = {world_size}, local_world_size = {local_world_size}")
 
     config = TrainConfig()

phoenix_diffusion.sbatch

@@ -2,7 +2,7 @@
 #SBATCH -J diffusion # Job name
 #SBATCH -A gts-jw254-coda20 
 #SBATCH -qembers
-#SBATCH -N1 --gpus-per-node=RTX_6000:4 # -C A100-80GB              # Number of nodes and cores per node required
+#SBATCH -N1 --gpus-per-node=RTX_6000:3 # -C A100-80GB              # Number of nodes and cores per node required
 #SBATCH --ntasks-per-node=1
 #SBATCH --mem-per-gpu=32G                        # Memory per core
 #SBATCH -t 10:00                                    # Duration of the job (Ex: 15 mins)