fix: with cell outputs

notebooks/03_fine_tuning.ipynb

@@ -561,7 +561,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 19,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -589,9 +589,29 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 20,
       "metadata": {},
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Gradient accumulation steps: 2\n",
+            "Effective batch size: 4\n",
+            "Effective tokens per update: 4096\n",
+            "\n",
+            "Training constants:\n",
+            "  Total training steps: 32\n",
+            "  Log training metrics every: 2 steps\n",
+            "  Run validation every: 4 steps\n",
+            "  Warmup steps: 3\n",
+            "\n",
+            "Optimizer setup:\n",
+            "  Initial LR: 1e-05\n",
+            "  Peak LR: 5e-05\n"
+          ]
+        }
+      ],
       "source": [
         "# Calculate gradient accumulation steps and effective batch size\n",
         "num_devices = 1  # Single device for now\n",
@@ -663,7 +683,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 21,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -753,7 +773,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 22,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -771,9 +791,17 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 23,
       "metadata": {},
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Scaling functions created\n"
+          ]
+        }
+      ],
       "source": [
         "def get_track_means(bigwig_file_ids: List[str]) -> np.ndarray:\n",
         "    \"\"\"\n",
@@ -899,7 +927,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 24,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -975,7 +1003,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 25,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -1061,9 +1089,98 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 26,
       "metadata": {},
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Starting training...\n",
+            "Training for 32 steps with 2 gradient accumulation steps\n",
+            "\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/home/y-bornachot/venvs/ntv3-env/lib/python3.12/site-packages/torch/amp/autocast_mode.py:287: UserWarning: In CPU autocast, but the target dtype is not supported. Disabling autocast.\n",
+            "CPU Autocast only supports dtype of torch.bfloat16, torch.float16 currently.\n",
+            "  warnings.warn(error_message)\n",
+            "/home/y-bornachot/venvs/ntv3-env/lib/python3.12/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: The variance of predictions or target is close to zero. This can cause instability in Pearson correlationcoefficient, leading to wrong results. Consider re-scaling the input if possible or computing using alarger dtype (currently using torch.float32). Setting the correlation coefficient to nan.\n",
+            "  warnings.warn(*args, **kwargs)\n",
+            "/tmp/ipykernel_1758159/1960846655.py:68: RuntimeWarning: Mean of empty slice\n",
+            "  metrics_dict[\"metrics_scaled/mean/pearson\"] = np.nanmean(correlations_scaled)\n",
+            "/tmp/ipykernel_1758159/1960846655.py:77: RuntimeWarning: Mean of empty slice\n",
+            "  metrics_dict[\"metrics_raw/mean/pearson\"] = np.nanmean(correlations_raw)\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Step 1/32 | Loss: 0.8378 | Mean Pearson: nan | LR: 1.17e-09 | Tokens: 4,096\n",
+            "\n",
+            "Running validation at step 0...\n",
+            "  Validation Loss: 0.5279\n",
+            "  Validation Mean Pearson: -0.0192\n",
+            "    ENCFF884LDL/pearson: -0.0192\n",
+            "Step 3/32 | Loss: 0.4650 | Mean Pearson: -0.0149 | LR: 2.50e-09 | Tokens: 12,288\n",
+            "Step 5/32 | Loss: 0.3369 | Mean Pearson: -0.1350 | LR: 2.41e-09 | Tokens: 20,480\n",
+            "\n",
+            "Running validation at step 4...\n",
+            "  Validation Loss: 0.3878\n",
+            "  Validation Mean Pearson: -0.1298\n",
+            "    ENCFF884LDL/pearson: -0.1298\n",
+            "Step 7/32 | Loss: 0.3609 | Mean Pearson: -0.0102 | LR: 2.32e-09 | Tokens: 28,672\n",
+            "Step 9/32 | Loss: 0.3301 | Mean Pearson: -0.0902 | LR: 2.23e-09 | Tokens: 36,864\n",
+            "\n",
+            "Running validation at step 8...\n",
+            "  Validation Loss: 0.4743\n",
+            "  Validation Mean Pearson: -0.0739\n",
+            "    ENCFF884LDL/pearson: -0.0739\n",
+            "Step 11/32 | Loss: 0.3905 | Mean Pearson: -0.0113 | LR: 2.13e-09 | Tokens: 45,056\n",
+            "Step 13/32 | Loss: 0.3181 | Mean Pearson: -0.1564 | LR: 2.02e-09 | Tokens: 53,248\n",
+            "\n",
+            "Running validation at step 12...\n",
+            "  Validation Loss: 0.3337\n",
+            "  Validation Mean Pearson: -0.0650\n",
+            "    ENCFF884LDL/pearson: -0.0650\n",
+            "Step 15/32 | Loss: 0.3638 | Mean Pearson: 0.0295 | LR: 1.91e-09 | Tokens: 61,440\n",
+            "Step 17/32 | Loss: 0.4170 | Mean Pearson: -0.0442 | LR: 1.80e-09 | Tokens: 69,632\n",
+            "\n",
+            "Running validation at step 16...\n",
+            "  Validation Loss: 0.7969\n",
+            "  Validation Mean Pearson: -0.0304\n",
+            "    ENCFF884LDL/pearson: -0.0304\n",
+            "Step 19/32 | Loss: 0.5033 | Mean Pearson: -0.0173 | LR: 1.67e-09 | Tokens: 77,824\n",
+            "Step 21/32 | Loss: 0.4084 | Mean Pearson: -0.0516 | LR: 1.54e-09 | Tokens: 86,016\n",
+            "\n",
+            "Running validation at step 20...\n",
+            "  Validation Loss: 0.3475\n",
+            "  Validation Mean Pearson: -0.3040\n",
+            "    ENCFF884LDL/pearson: -0.3040\n",
+            "Step 23/32 | Loss: 0.4915 | Mean Pearson: -0.1727 | LR: 1.39e-09 | Tokens: 94,208\n",
+            "Step 25/32 | Loss: 0.3654 | Mean Pearson: -0.3257 | LR: 1.23e-09 | Tokens: 102,400\n",
+            "\n",
+            "Running validation at step 24...\n",
+            "  Validation Loss: 0.4069\n",
+            "  Validation Mean Pearson: -0.0551\n",
+            "    ENCFF884LDL/pearson: -0.0551\n",
+            "Step 27/32 | Loss: 0.5344 | Mean Pearson: -0.0604 | LR: 1.04e-09 | Tokens: 110,592\n",
+            "Step 29/32 | Loss: 0.3671 | Mean Pearson: -0.0290 | LR: 8.04e-10 | Tokens: 118,784\n",
+            "\n",
+            "Running validation at step 28...\n",
+            "  Validation Loss: 0.3162\n",
+            "  Validation Mean Pearson: -0.1008\n",
+            "    ENCFF884LDL/pearson: -0.1008\n",
+            "Step 31/32 | Loss: 0.5994 | Mean Pearson: -0.0107 | LR: 4.64e-10 | Tokens: 126,976\n",
+            "\n",
+            "Training completed after 32 steps!\n"
+          ]
+        }
+      ],
       "source": [
         "# Training loop (step-based with gradient accumulation)\n",
         "print(\"Starting training...\")\n",
@@ -1174,7 +1291,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 27,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -1216,9 +1333,34 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 28,
       "metadata": {},
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "\n",
+            "==================================================\n",
+            "Test Set Evaluation\n",
+            "==================================================\n",
+            "Running test evaluation with 5 steps (10 samples)\n",
+            "\n",
+            "==================================================\n",
+            "Test Set Results\n",
+            "==================================================\n",
+            "\n",
+            "Scaled Metrics (scaled predictions vs scaled targets):\n",
+            "  Mean Pearson (scaled): -0.0020\n",
+            "    ENCFF884LDL/pearson: -0.0020\n",
+            "\n",
+            "Raw Metrics (raw predictions vs raw targets):\n",
+            "  Mean Pearson (raw): -0.0020\n",
+            "    ENCFF884LDL/pearson: -0.0020\n",
+            "==================================================\n"
+          ]
+        }
+      ],
       "source": [
         "print(\"\\n\" + \"=\"*50)\n",
         "print(\"Test Set Evaluation\")\n",
@@ -1265,13 +1407,6 @@
         "    print(f\"    {track_name}/pearson: {test_metrics_dict[f'metrics_raw/{track_name}/pearson']:.4f}\")\n",
         "print(\"=\"*50)"
       ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": []
     }
   ],
   "metadata": {