fix: ran cells

notebooks/03_fine_tuning.ipynb

@@ -1032,7 +1032,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 42,
    "metadata": {},
    "outputs": [
     {
@@ -1041,84 +1041,63 @@
      "text": [
       "Starting training...\n",
       "Training for 32 steps with 2 gradient accumulation steps\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/y-bornachot/venvs/ntv3-env/lib/python3.12/site-packages/torch/amp/autocast_mode.py:287: UserWarning: In CPU autocast, but the target dtype is not supported. Disabling autocast.\n",
-      "CPU Autocast only supports dtype of torch.bfloat16, torch.float16 currently.\n",
-      "  warnings.warn(error_message)\n",
-      "/home/y-bornachot/venvs/ntv3-env/lib/python3.12/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: The variance of predictions or target is close to zero. This can cause instability in Pearson correlationcoefficient, leading to wrong results. Consider re-scaling the input if possible or computing using alarger dtype (currently using torch.float32). Setting the correlation coefficient to nan.\n",
-      "  warnings.warn(*args, **kwargs)\n",
-      "/tmp/ipykernel_1758159/1960846655.py:68: RuntimeWarning: Mean of empty slice\n",
-      "  metrics_dict[\"metrics_scaled/mean/pearson\"] = np.nanmean(correlations_scaled)\n",
-      "/tmp/ipykernel_1758159/1960846655.py:77: RuntimeWarning: Mean of empty slice\n",
-      "  metrics_dict[\"metrics_raw/mean/pearson\"] = np.nanmean(correlations_raw)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 1/32 | Loss: 0.8378 | Mean Pearson: nan | LR: 1.17e-09 | Tokens: 4,096\n",
+      "\n",
+      "Step 1/32 | Loss: 0.5661 | Mean Pearson: -0.0525 | Tokens: 4,096\n",
       "\n",
       "Running validation at step 0...\n",
-      "  Validation Loss: 0.5279\n",
-      "  Validation Mean Pearson: -0.0192\n",
-      "    ENCFF884LDL/pearson: -0.0192\n",
-      "Step 3/32 | Loss: 0.4650 | Mean Pearson: -0.0149 | LR: 2.50e-09 | Tokens: 12,288\n",
-      "Step 5/32 | Loss: 0.3369 | Mean Pearson: -0.1350 | LR: 2.41e-09 | Tokens: 20,480\n",
+      "  Validation Loss: 0.3987\n",
+      "  Validation Mean Pearson: -0.0426\n",
+      "    ENCFF884LDL/pearson: -0.0426\n",
+      "Step 3/32 | Loss: 0.3825 | Mean Pearson: -0.0112 | Tokens: 12,288\n",
+      "Step 5/32 | Loss: 1.1384 | Mean Pearson: -0.0777 | Tokens: 20,480\n",
       "\n",
       "Running validation at step 4...\n",
-      "  Validation Loss: 0.3878\n",
-      "  Validation Mean Pearson: -0.1298\n",
-      "    ENCFF884LDL/pearson: -0.1298\n",
-      "Step 7/32 | Loss: 0.3609 | Mean Pearson: -0.0102 | LR: 2.32e-09 | Tokens: 28,672\n",
-      "Step 9/32 | Loss: 0.3301 | Mean Pearson: -0.0902 | LR: 2.23e-09 | Tokens: 36,864\n",
+      "  Validation Loss: 0.4381\n",
+      "  Validation Mean Pearson: -0.0017\n",
+      "    ENCFF884LDL/pearson: -0.0017\n",
+      "Step 7/32 | Loss: 0.4961 | Mean Pearson: -0.0188 | Tokens: 28,672\n",
+      "Step 9/32 | Loss: 0.4903 | Mean Pearson: -0.1522 | Tokens: 36,864\n",
       "\n",
       "Running validation at step 8...\n",
-      "  Validation Loss: 0.4743\n",
-      "  Validation Mean Pearson: -0.0739\n",
-      "    ENCFF884LDL/pearson: -0.0739\n",
-      "Step 11/32 | Loss: 0.3905 | Mean Pearson: -0.0113 | LR: 2.13e-09 | Tokens: 45,056\n",
-      "Step 13/32 | Loss: 0.3181 | Mean Pearson: -0.1564 | LR: 2.02e-09 | Tokens: 53,248\n",
+      "  Validation Loss: 0.3429\n",
+      "  Validation Mean Pearson: -0.0997\n",
+      "    ENCFF884LDL/pearson: -0.0997\n",
+      "Step 11/32 | Loss: 0.4597 | Mean Pearson: -0.0199 | Tokens: 45,056\n",
+      "Step 13/32 | Loss: 0.6507 | Mean Pearson: -0.0256 | Tokens: 53,248\n",
       "\n",
       "Running validation at step 12...\n",
-      "  Validation Loss: 0.3337\n",
-      "  Validation Mean Pearson: -0.0650\n",
-      "    ENCFF884LDL/pearson: -0.0650\n",
-      "Step 15/32 | Loss: 0.3638 | Mean Pearson: 0.0295 | LR: 1.91e-09 | Tokens: 61,440\n",
-      "Step 17/32 | Loss: 0.4170 | Mean Pearson: -0.0442 | LR: 1.80e-09 | Tokens: 69,632\n",
+      "  Validation Loss: 0.3901\n",
+      "  Validation Mean Pearson: -0.0786\n",
+      "    ENCFF884LDL/pearson: -0.0786\n",
+      "Step 15/32 | Loss: 0.3911 | Mean Pearson: -0.0419 | Tokens: 61,440\n",
+      "Step 17/32 | Loss: 0.4202 | Mean Pearson: -0.0883 | Tokens: 69,632\n",
       "\n",
       "Running validation at step 16...\n",
-      "  Validation Loss: 0.7969\n",
-      "  Validation Mean Pearson: -0.0304\n",
-      "    ENCFF884LDL/pearson: -0.0304\n",
-      "Step 19/32 | Loss: 0.5033 | Mean Pearson: -0.0173 | LR: 1.67e-09 | Tokens: 77,824\n",
-      "Step 21/32 | Loss: 0.4084 | Mean Pearson: -0.0516 | LR: 1.54e-09 | Tokens: 86,016\n",
+      "  Validation Loss: 0.3626\n",
+      "  Validation Mean Pearson: -0.0840\n",
+      "    ENCFF884LDL/pearson: -0.0840\n",
+      "Step 19/32 | Loss: 0.3608 | Mean Pearson: -0.1057 | Tokens: 77,824\n",
+      "Step 21/32 | Loss: 0.3942 | Mean Pearson: 0.1459 | Tokens: 86,016\n",
       "\n",
       "Running validation at step 20...\n",
-      "  Validation Loss: 0.3475\n",
-      "  Validation Mean Pearson: -0.3040\n",
-      "    ENCFF884LDL/pearson: -0.3040\n",
-      "Step 23/32 | Loss: 0.4915 | Mean Pearson: -0.1727 | LR: 1.39e-09 | Tokens: 94,208\n",
-      "Step 25/32 | Loss: 0.3654 | Mean Pearson: -0.3257 | LR: 1.23e-09 | Tokens: 102,400\n",
+      "  Validation Loss: 0.3281\n",
+      "  Validation Mean Pearson: -0.0667\n",
+      "    ENCFF884LDL/pearson: -0.0667\n",
+      "Step 23/32 | Loss: 0.4090 | Mean Pearson: 0.0540 | Tokens: 94,208\n",
+      "Step 25/32 | Loss: 0.5151 | Mean Pearson: -0.0076 | Tokens: 102,400\n",
       "\n",
       "Running validation at step 24...\n",
-      "  Validation Loss: 0.4069\n",
-      "  Validation Mean Pearson: -0.0551\n",
-      "    ENCFF884LDL/pearson: -0.0551\n",
-      "Step 27/32 | Loss: 0.5344 | Mean Pearson: -0.0604 | LR: 1.04e-09 | Tokens: 110,592\n",
-      "Step 29/32 | Loss: 0.3671 | Mean Pearson: -0.0290 | LR: 8.04e-10 | Tokens: 118,784\n",
+      "  Validation Loss: 0.2927\n",
+      "  Validation Mean Pearson: -0.0409\n",
+      "    ENCFF884LDL/pearson: -0.0409\n",
+      "Step 27/32 | Loss: 0.4339 | Mean Pearson: -0.0887 | Tokens: 110,592\n",
+      "Step 29/32 | Loss: 0.4516 | Mean Pearson: -0.0763 | Tokens: 118,784\n",
       "\n",
       "Running validation at step 28...\n",
-      "  Validation Loss: 0.3162\n",
-      "  Validation Mean Pearson: -0.1008\n",
-      "    ENCFF884LDL/pearson: -0.1008\n",
-      "Step 31/32 | Loss: 0.5994 | Mean Pearson: -0.0107 | LR: 4.64e-10 | Tokens: 126,976\n",
+      "  Validation Loss: 0.3076\n",
+      "  Validation Mean Pearson: -0.0861\n",
+      "    ENCFF884LDL/pearson: -0.0861\n",
+      "Step 31/32 | Loss: 0.4121 | Mean Pearson: -0.0530 | Tokens: 126,976\n",
       "\n",
       "Training completed after 32 steps!\n"
      ]
@@ -1229,7 +1208,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 43,
    "metadata": {},
    "outputs": [],
    "source": [