13133235

perlmutter_diffusion.sbatch

@@ -2,10 +2,10 @@
 #SBATCH -A m4717 
 #SBATCH -J diffusion
 #SBATCH -C gpu&hbm80g
-#SBATCH -q regular #shared 
-#SBATCH -N5
-#SBATCH --gpus-per-node=4
-#SBATCH -t 48:00:00
+#SBATCH -q shared #regular 
+#SBATCH -N1
+#SBATCH --gpus-per-node=1
+#SBATCH -t 6:00:00
 #SBATCH --ntasks-per-node=1
 #SBATCH -oReport-%j 
 #SBATCH --mail-type=BEGIN,END,FAIL
@@ -40,12 +40,12 @@ srun python diffusion.py \
     --batch_size 2 \
     --n_epoch 80 \
     --channel_mult 0.5 1 2 4 4 8 \
-    --num_new_img_per_gpu 800 \
-    --max_num_img_per_gpu 100 \
+    --num_new_img_per_gpu 9 \
+    --max_num_img_per_gpu 3 \
     --gradient_accumulation_steps 1 \
     --autocast 1 \
     --use_checkpoint 1 \
-    --train "$SCRATCH/LEN128-DIM64-CUB16-Tvir[4, 6]-zeta[10, 250]-0809-123640.h5" \
-    #--resume ./outputs/model-N6400-device_count4-node1-epoch199-05185634 \
+    --resume ./outputs/model-N1280-device_count4-node5-epoch34-13133235 \
+    #--train "$SCRATCH/LEN128-DIM64-CUB16-Tvir[4, 6]-zeta[10, 250]-0809-123640.h5" \
 
 date

tensorboard.ipynb

@@ -23,13 +23,13 @@
      "data": {
       "text/html": [
        "\n",
-       "      <iframe id=\"tensorboard-frame-8bbb5cb424abc4b5\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
+       "      <iframe id=\"tensorboard-frame-c11f24d3c7445b04\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
        "      </iframe>\n",
        "      <script>\n",
        "        (function() {\n",
-       "          const frame = document.getElementById(\"tensorboard-frame-8bbb5cb424abc4b5\");\n",
+       "          const frame = document.getElementById(\"tensorboard-frame-c11f24d3c7445b04\");\n",
        "          const url = new URL(\"/\", window.location);\n",
-       "          const port = 34693;\n",
+       "          const port = 46861;\n",
        "          if (port) {\n",
        "            url.port = port;\n",
        "          }\n",
@@ -59,7 +59,7 @@
     {
      "data": {
       "text/html": [
-       "<a href=\"https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/34693/\">https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/34693/</a>"
+       "<a href=\"https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/46861/\">https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/46861/</a>"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"