Training in progress, step 19000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:175d3790f4e3bfb8245c7b39dd800f69c0db9fc536965d696ec8288a9a4a9102
+oid sha256:f397f4e5642ab9da594752daa8ae50f67bd1c0633a0f55e9742963fc8094fc07
 size 373077376

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:788c53706c70a599497c5b67318671aa687876ff14e4cb7b622bd54561a1949e
+oid sha256:2947a98ddfbd91d8b159be483158ff85ad080e24af7608cd8524985d5ff37696
 size 209816139

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9a15fe294d83e6dcaaac27e2da3864df4a3093a7010bb98cb24cc0788020fcf
+oid sha256:c4bf7692b5f5edcd474e14a77a5f13e3f7c7765bb40a870ecf6eeef166453cdc
 size 14917

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbd4cb683a8704b2a8823c0b9c5dfb0af1604e837d8fd4eb0a6b198fcadf9ed0
+oid sha256:67c08732e07123ac792d039d19d16df7f1963cb3c04d8bb64d087ce8609b973b
 size 14917

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3db8ebdfd646609f3b5f0928122cbb46e5703a2f230d68735c07eb7b6e67a5cd
+oid sha256:c27272cd09ade13a826f643a30f8708da8615de25fd4349edd6f9144bb4f5503
 size 1401

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": 2000,
   "best_metric": 9.218317031860352,
   "best_model_checkpoint": "./artifacts/models/base-250725-test/checkpoint-2000",
-  "epoch": 0.056215919099046205,
+  "epoch": 0.05933902571565988,
   "eval_steps": 1000,
-  "global_step": 18000,
+  "global_step": 19000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5192,6 +5192,294 @@
       "eval_samples_per_second": 50.835,
       "eval_steps_per_second": 3.185,
       "step": 18000
+    },
+    {
+      "epoch": 0.05629399676446155,
+      "grad_norm": 43.5,
+      "learning_rate": 0.0009998918390993648,
+      "loss": 30.9913,
+      "step": 18025
+    },
+    {
+      "epoch": 0.056372074429876885,
+      "grad_norm": 42.75,
+      "learning_rate": 0.000999889137307281,
+      "loss": 31.086,
+      "step": 18050
+    },
+    {
+      "epoch": 0.05645015209529223,
+      "grad_norm": 41.0,
+      "learning_rate": 0.0009998864021893864,
+      "loss": 31.0512,
+      "step": 18075
+    },
+    {
+      "epoch": 0.05652822976070757,
+      "grad_norm": 42.75,
+      "learning_rate": 0.0009998836337458629,
+      "loss": 31.2091,
+      "step": 18100
+    },
+    {
+      "epoch": 0.056606307426122916,
+      "grad_norm": 44.25,
+      "learning_rate": 0.0009998808319768954,
+      "loss": 31.1535,
+      "step": 18125
+    },
+    {
+      "epoch": 0.05668438509153825,
+      "grad_norm": 43.5,
+      "learning_rate": 0.0009998779968826707,
+      "loss": 31.3788,
+      "step": 18150
+    },
+    {
+      "epoch": 0.0567624627569536,
+      "grad_norm": 43.75,
+      "learning_rate": 0.0009998751284633779,
+      "loss": 31.3632,
+      "step": 18175
+    },
+    {
+      "epoch": 0.05684054042236894,
+      "grad_norm": 39.0,
+      "learning_rate": 0.0009998722267192076,
+      "loss": 31.101,
+      "step": 18200
+    },
+    {
+      "epoch": 0.056918618087784284,
+      "grad_norm": 38.25,
+      "learning_rate": 0.000999869291650354,
+      "loss": 30.8788,
+      "step": 18225
+    },
+    {
+      "epoch": 0.05699669575319962,
+      "grad_norm": 36.75,
+      "learning_rate": 0.0009998663232570122,
+      "loss": 31.0841,
+      "step": 18250
+    },
+    {
+      "epoch": 0.057074773418614964,
+      "grad_norm": 39.75,
+      "learning_rate": 0.0009998633215393805,
+      "loss": 31.4425,
+      "step": 18275
+    },
+    {
+      "epoch": 0.05715285108403031,
+      "grad_norm": 37.5,
+      "learning_rate": 0.000999860286497659,
+      "loss": 31.6592,
+      "step": 18300
+    },
+    {
+      "epoch": 0.05723092874944565,
+      "grad_norm": 40.0,
+      "learning_rate": 0.0009998572181320496,
+      "loss": 31.3277,
+      "step": 18325
+    },
+    {
+      "epoch": 0.05730900641486099,
+      "grad_norm": 39.75,
+      "learning_rate": 0.0009998541164427575,
+      "loss": 31.3697,
+      "step": 18350
+    },
+    {
+      "epoch": 0.05738708408027633,
+      "grad_norm": 35.0,
+      "learning_rate": 0.0009998509814299888,
+      "loss": 31.2663,
+      "step": 18375
+    },
+    {
+      "epoch": 0.057465161745691676,
+      "grad_norm": 37.25,
+      "learning_rate": 0.000999847813093953,
+      "loss": 31.6682,
+      "step": 18400
+    },
+    {
+      "epoch": 0.05754323941110702,
+      "grad_norm": 38.75,
+      "learning_rate": 0.0009998446114348612,
+      "loss": 31.7364,
+      "step": 18425
+    },
+    {
+      "epoch": 0.057621317076522356,
+      "grad_norm": 48.5,
+      "learning_rate": 0.0009998413764529266,
+      "loss": 31.8273,
+      "step": 18450
+    },
+    {
+      "epoch": 0.0576993947419377,
+      "grad_norm": 39.5,
+      "learning_rate": 0.0009998381081483651,
+      "loss": 32.178,
+      "step": 18475
+    },
+    {
+      "epoch": 0.057777472407353044,
+      "grad_norm": 38.75,
+      "learning_rate": 0.0009998348065213946,
+      "loss": 32.3324,
+      "step": 18500
+    },
+    {
+      "epoch": 0.05785555007276839,
+      "grad_norm": 41.75,
+      "learning_rate": 0.000999831471572235,
+      "loss": 32.6464,
+      "step": 18525
+    },
+    {
+      "epoch": 0.057933627738183724,
+      "grad_norm": 42.0,
+      "learning_rate": 0.0009998281033011091,
+      "loss": 32.1848,
+      "step": 18550
+    },
+    {
+      "epoch": 0.05801170540359907,
+      "grad_norm": 39.75,
+      "learning_rate": 0.000999824701708241,
+      "loss": 32.543,
+      "step": 18575
+    },
+    {
+      "epoch": 0.05808978306901441,
+      "grad_norm": 48.5,
+      "learning_rate": 0.0009998212667938578,
+      "loss": 32.4726,
+      "step": 18600
+    },
+    {
+      "epoch": 0.058167860734429755,
+      "grad_norm": 45.0,
+      "learning_rate": 0.000999817798558188,
+      "loss": 32.2877,
+      "step": 18625
+    },
+    {
+      "epoch": 0.05824593839984509,
+      "grad_norm": 38.25,
+      "learning_rate": 0.0009998142970014633,
+      "loss": 32.4187,
+      "step": 18650
+    },
+    {
+      "epoch": 0.058324016065260435,
+      "grad_norm": 51.5,
+      "learning_rate": 0.0009998107621239168,
+      "loss": 32.6334,
+      "step": 18675
+    },
+    {
+      "epoch": 0.05840209373067578,
+      "grad_norm": 48.5,
+      "learning_rate": 0.0009998071939257842,
+      "loss": 33.0217,
+      "step": 18700
+    },
+    {
+      "epoch": 0.05848017139609112,
+      "grad_norm": 50.0,
+      "learning_rate": 0.0009998035924073036,
+      "loss": 32.839,
+      "step": 18725
+    },
+    {
+      "epoch": 0.05855824906150646,
+      "grad_norm": 41.75,
+      "learning_rate": 0.000999799957568715,
+      "loss": 32.84,
+      "step": 18750
+    },
+    {
+      "epoch": 0.0586363267269218,
+      "grad_norm": 55.5,
+      "learning_rate": 0.0009997962894102608,
+      "loss": 33.0097,
+      "step": 18775
+    },
+    {
+      "epoch": 0.05871440439233715,
+      "grad_norm": 52.5,
+      "learning_rate": 0.0009997925879321854,
+      "loss": 33.0055,
+      "step": 18800
+    },
+    {
+      "epoch": 0.05879248205775249,
+      "grad_norm": 47.25,
+      "learning_rate": 0.0009997888531347358,
+      "loss": 33.3652,
+      "step": 18825
+    },
+    {
+      "epoch": 0.05887055972316783,
+      "grad_norm": 41.25,
+      "learning_rate": 0.0009997850850181605,
+      "loss": 33.1608,
+      "step": 18850
+    },
+    {
+      "epoch": 0.05894863738858317,
+      "grad_norm": 42.75,
+      "learning_rate": 0.000999781283582711,
+      "loss": 33.2872,
+      "step": 18875
+    },
+    {
+      "epoch": 0.059026715053998514,
+      "grad_norm": 43.25,
+      "learning_rate": 0.0009997774488286408,
+      "loss": 33.0581,
+      "step": 18900
+    },
+    {
+      "epoch": 0.05910479271941386,
+      "grad_norm": 48.0,
+      "learning_rate": 0.0009997735807562055,
+      "loss": 33.0212,
+      "step": 18925
+    },
+    {
+      "epoch": 0.059182870384829195,
+      "grad_norm": 39.0,
+      "learning_rate": 0.000999769679365663,
+      "loss": 32.7047,
+      "step": 18950
+    },
+    {
+      "epoch": 0.05926094805024454,
+      "grad_norm": 41.25,
+      "learning_rate": 0.0009997657446572735,
+      "loss": 32.7831,
+      "step": 18975
+    },
+    {
+      "epoch": 0.05933902571565988,
+      "grad_norm": 42.75,
+      "learning_rate": 0.0009997617766312988,
+      "loss": 32.8744,
+      "step": 19000
+    },
+    {
+      "epoch": 0.05933902571565988,
+      "eval_loss": 32.887264251708984,
+      "eval_runtime": 102.2215,
+      "eval_samples_per_second": 50.899,
+      "eval_steps_per_second": 3.189,
+      "step": 19000
     }
   ],
   "logging_steps": 25,
@@ -5211,7 +5499,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.282666343213826e+18,
+  "total_flos": 2.4094740084298875e+18,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null