32353762

diffusion.py

@@ -445,6 +445,9 @@ class DDPM21CM:
         # flip diagonally 
         if getrandbits(1):
             img[idx] = img[idx].clone().transpose(1,2)
+            #print(f"transform: img.shape={img.shape}, idx={idx}, flip_xy={flip_xy}, w/ transpose")
+        #else:
+            #print(f"transform: img.shape={img.shape}, idx={idx}, flip_xy={flip_xy}, w/o tranpose")
         return img
 
     def train(self):

perlmutter_diffusion.sbatch

@@ -5,7 +5,7 @@
 #SBATCH -q regular #shared 
 #SBATCH -N4
 #SBATCH --gpus-per-node=4
-#SBATCH -t 48:00:00
+#SBATCH -t 16:00:00
 #SBATCH --ntasks-per-node=1
 #SBATCH -oReport-%j 
 #SBATCH --mail-type=BEGIN,END,FAIL
@@ -30,7 +30,7 @@ cat $0
 srun python diffusion.py \
     --num_image 1600 \
     --batch_size 2 \
-    --n_epoch 60 \
+    --n_epoch 20 \
     --channel_mult 0.5 1 2 4 4 8 \
     --num_new_img_per_gpu 4 \
     --max_num_img_per_gpu 2 \
@@ -40,6 +40,6 @@ srun python diffusion.py \
     --dropout 0.2 \
     --lrate 2e-5 \
     --train "$SCRATCH/LEN128-DIM64-CUB16-Tvir[4, 6]-zeta[10, 250]-0809-123640.h5" \
-    #--resume ./outputs/model-N1600-device_count4-node4-epoch19-32185426 \
+    #--resume ./outputs/model-N1600-device_count4-node4-epoch19-32257242 \
 
 date

quantify_results.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6691f0135f3bb373506e6090511c5527f23e0b4dc780f031bf80ca6d141e32ca
-size 25396988
+oid sha256:6893bbfd60b952c7a56b8eea8506da7df6e47513a4cd301d0980db2e0dafd5cc
+size 24436267

tensorboard.ipynb

@@ -2,12 +2,21 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 4,
    "id": "ae45e44e-a11c-43ef-b830-c7a58a72f51e",
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The tensorboard extension is already loaded. To reload it, use:\n",
+      "  %reload_ext tensorboard\n"
+     ]
+    }
+   ],
    "source": [
     "import nersc_tensorboard_helper\n",
     "%load_ext tensorboard"
@@ -15,21 +24,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 5,
    "id": "a5c088b8-5051-402f-b4ec-2b684ad5a952",
    "metadata": {},
    "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Reusing TensorBoard on port 33249 (pid 774528), started 4:01:53 ago. (Use '!kill 774528' to kill it.)"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
     {
      "data": {
       "text/html": [
        "\n",
-       "      <iframe id=\"tensorboard-frame-b3fe77206bcde3f5\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
+       "      <iframe id=\"tensorboard-frame-a8fadb66b88e501a\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
        "      </iframe>\n",
        "      <script>\n",
        "        (function() {\n",
-       "          const frame = document.getElementById(\"tensorboard-frame-b3fe77206bcde3f5\");\n",
+       "          const frame = document.getElementById(\"tensorboard-frame-a8fadb66b88e501a\");\n",
        "          const url = new URL(\"/\", window.location);\n",
-       "          const port = 33553;\n",
+       "          const port = 33249;\n",
        "          if (port) {\n",
        "            url.port = port;\n",
        "          }\n",
@@ -52,14 +70,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 6,
    "id": "2f76c0a9-2218-4073-86aa-f4f655d7642f",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<a href=\"https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/33553/\">https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/33553/</a>"
+       "<a href=\"https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/33249/\">https://jupyter.nersc.gov/user/binxia/perlmutter-login-node-base/proxy/33249/</a>"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"