Latest checkpoint (step 3000)

checkpoint-latest/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56c013ad54b139dec7b9011370cd41afac9c0938272987adcbfbada71c3ee6f6
+oid sha256:d1800d7755dad7cc3e40c3741acc7b918a441c2d1dfea31a6e56efcf4081a094
 size 7098016

checkpoint-latest/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa6b3b9307c182e3ec3833d943c9fbef0a93a06664d1eb51c4ba9cc23df98a99
+oid sha256:74c2469bbe7f45a52b3e15a7bd4b6235ed1cad3192602010536a3ae43528459b
 size 14277259

checkpoint-latest/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d6637676db4b6dd604660062b1e5095964ae7941bd41cd906074b3c9b8a4f67
+oid sha256:b77e06d9e8a9ec437ce19bad2fcf9c3fc37002aceb56c45b0d6882f78fb7fc88
 size 14645

checkpoint-latest/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd17185e93eafdc9bdff3d9c17b7e92324daef9d95996192a18dd54f1490272d
+oid sha256:991fcba719acec99b5539d96fcc576240d1d3bb15cabef2962124555f60c56e9
 size 1465

checkpoint-latest/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.5003042221069336,
   "best_model_checkpoint": "./t5_checkpoints_full/checkpoint-1000",
-  "epoch": 0.3136516897984788,
+  "epoch": 0.4704775346977182,
   "eval_steps": 1000,
-  "global_step": 2000,
+  "global_step": 3000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -263,6 +263,134 @@
       "eval_samples_per_second": 119.492,
       "eval_steps_per_second": 7.476,
       "step": 2000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00027542409276358167,
+      "loss": 1.4488,
+      "step": 2050
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00027461885333905943,
+      "loss": 1.455,
+      "step": 2100
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00027381361391453724,
+      "loss": 1.4353,
+      "step": 2150
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000273008374490015,
+      "loss": 1.4524,
+      "step": 2200
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0002722192398539832,
+      "loss": 1.4701,
+      "step": 2250
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.000271414000429461,
+      "loss": 1.4734,
+      "step": 2300
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0002706409705819197,
+      "loss": 1.5035,
+      "step": 2350
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00026983573115739744,
+      "loss": 1.4513,
+      "step": 2400
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00026904659652136567,
+      "loss": 1.4641,
+      "step": 2450
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0002682413570968434,
+      "loss": 1.4585,
+      "step": 2500
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00026743611767232123,
+      "loss": 1.4673,
+      "step": 2550
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002666469830362894,
+      "loss": 1.4671,
+      "step": 2600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002658578484002577,
+      "loss": 1.4702,
+      "step": 2650
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00026508481855271634,
+      "loss": 1.4612,
+      "step": 2700
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002642956839166845,
+      "loss": 1.4713,
+      "step": 2750
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00026350654928065275,
+      "loss": 1.4573,
+      "step": 2800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.000262717414644621,
+      "loss": 1.4586,
+      "step": 2850
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00026191217522009873,
+      "loss": 1.4674,
+      "step": 2900
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00026110693579557654,
+      "loss": 1.4466,
+      "step": 2950
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002603339059480352,
+      "loss": 1.4897,
+      "step": 3000
+    },
+    {
+      "epoch": 0.47,
+      "eval_loss": 1.2417596578598022,
+      "eval_runtime": 94.8105,
+      "eval_samples_per_second": 119.523,
+      "eval_steps_per_second": 7.478,
+      "step": 3000
     }
   ],
   "logging_steps": 50,
@@ -270,7 +398,7 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 1000,
-  "total_flos": 3.9321197346816e+16,
+  "total_flos": 5.8981796020224e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

checkpoint-latest/training_metrics.json

@@ -1,90 +1,90 @@
 {
-  "step": 2000,
-  "epoch": 0.3136516897984788,
+  "step": 3000,
+  "epoch": 0.4704775346977182,
   "best_eval_loss": 0.5003042221069336,
-  "checkpoint_number": 2,
+  "checkpoint_number": 3,
   "recent_metrics": [
     {
-      "step": 1600,
-      "epoch": 0.25,
-      "loss": 1.4561,
-      "learning_rate": 0.00028262293321881034,
+      "step": 2600,
+      "epoch": 0.41,
+      "loss": 1.4671,
+      "learning_rate": 0.0002666469830362894,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 8.8
+      "system_memory_percent": 6.7
     },
     {
-      "step": 1650,
-      "epoch": 0.26,
-      "loss": 1.4693,
-      "learning_rate": 0.0002818337985827786,
+      "step": 2650,
+      "epoch": 0.42,
+      "loss": 1.4702,
+      "learning_rate": 0.0002658578484002577,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 11.4
+      "system_memory_percent": 6.7
     },
     {
-      "step": 1700,
-      "epoch": 0.27,
-      "loss": 1.4729,
-      "learning_rate": 0.0002810285591582563,
+      "step": 2700,
+      "epoch": 0.42,
+      "loss": 1.4612,
+      "learning_rate": 0.00026508481855271634,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.3
+      "system_memory_percent": 6.8
     },
     {
-      "step": 1750,
-      "epoch": 0.27,
-      "loss": 1.4599,
-      "learning_rate": 0.00028022331973373413,
+      "step": 2750,
+      "epoch": 0.43,
+      "loss": 1.4713,
+      "learning_rate": 0.0002642956839166845,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.3
+      "system_memory_percent": 6.8
     },
     {
-      "step": 1800,
-      "epoch": 0.28,
-      "loss": 1.4725,
-      "learning_rate": 0.0002794180803092119,
+      "step": 2800,
+      "epoch": 0.44,
+      "loss": 1.4573,
+      "learning_rate": 0.00026350654928065275,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.2
+      "system_memory_percent": 6.8
     },
     {
-      "step": 1850,
-      "epoch": 0.29,
-      "loss": 1.4503,
-      "learning_rate": 0.0002786128408846897,
+      "step": 2850,
+      "epoch": 0.45,
+      "loss": 1.4586,
+      "learning_rate": 0.000262717414644621,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.2
+      "system_memory_percent": 6.8
     },
     {
-      "step": 1900,
-      "epoch": 0.3,
-      "loss": 1.4812,
-      "learning_rate": 0.0002778237062486579,
+      "step": 2900,
+      "epoch": 0.45,
+      "loss": 1.4674,
+      "learning_rate": 0.00026191217522009873,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.2
+      "system_memory_percent": 6.8
     },
     {
-      "step": 1950,
-      "epoch": 0.31,
-      "loss": 1.4761,
-      "learning_rate": 0.0002770184668241357,
+      "step": 2950,
+      "epoch": 0.46,
+      "loss": 1.4466,
+      "learning_rate": 0.00026110693579557654,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.2
+      "system_memory_percent": 6.8
     },
     {
-      "step": 2000,
-      "epoch": 0.31,
-      "loss": 1.496,
-      "learning_rate": 0.00027621322739961344,
+      "step": 3000,
+      "epoch": 0.47,
+      "loss": 1.4897,
+      "learning_rate": 0.0002603339059480352,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.3
+      "system_memory_percent": 6.8
     },
     {
-      "step": 2000,
-      "epoch": 0.31,
-      "eval_loss": 1.251204252243042,
-      "eval_runtime": 94.8348,
-      "eval_samples_per_second": 119.492,
-      "eval_steps_per_second": 7.476,
+      "step": 3000,
+      "epoch": 0.47,
+      "eval_loss": 1.2417596578598022,
+      "eval_runtime": 94.8105,
+      "eval_samples_per_second": 119.523,
+      "eval_steps_per_second": 7.478,
       "gpu_memory_gb": 0.8661794662475586,
-      "system_memory_percent": 13.2
+      "system_memory_percent": 6.7
     }
   ]
 }