Training in progress, step 21000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df03cae2dd432c211456aab943782bf83ba84e08565c4c981659cb89c83a578e
+oid sha256:ba9d75a78fad20f4b1e389f6c85dda0f453be86d800ed2eba32953160cc02033
 size 373077376

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4292287a7fa690fe53e7b389faee8373877f88d995cc45d3321aeb77bf8c4af6
+oid sha256:df2f641838670afd6d1bb0181e8efde74cebba7ddaeaad933397844d1eb9afb6
 size 209816139

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22e8bb13b8b5cd110e015717953ca96d5c03c35ddfe30ca45c1fab9651d07421
+oid sha256:eda74d083cd5d9b07d403914b5a235c44dd87bc93a29636e940f36b95f8743f9
 size 14917

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76ace0471241ab08ffd32878e593821b741d6b0b68bcb601ea44671e5ef83eef
+oid sha256:91f1feed6ec98326449107f6ac06aad035f8176b90aa697c6edf6a509039a50c
 size 14917

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba4436ed0869bacf238e760f8e2f2044a22ff86693a77a3015046ef89f00fc7e
+oid sha256:c6645e7dc37725bbae83eaf70fb81001a75be54d9a6554f43743dfb20cfc0984
 size 1401

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": 2000,
   "best_metric": 9.218317031860352,
   "best_model_checkpoint": "./artifacts/models/base-250725-test/checkpoint-2000",
-  "epoch": 0.06246213233227356,
+  "epoch": 0.06558523894888724,
   "eval_steps": 1000,
-  "global_step": 20000,
+  "global_step": 21000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5768,6 +5768,294 @@
       "eval_samples_per_second": 50.82,
       "eval_steps_per_second": 3.184,
       "step": 20000
+    },
+    {
+      "epoch": 0.0625402099976889,
+      "grad_norm": 50.0,
+      "learning_rate": 0.000999570404664504,
+      "loss": 34.3706,
+      "step": 20025
+    },
+    {
+      "epoch": 0.06261828766310425,
+      "grad_norm": 45.75,
+      "learning_rate": 0.0009995650375662492,
+      "loss": 34.1775,
+      "step": 20050
+    },
+    {
+      "epoch": 0.06269636532851959,
+      "grad_norm": 43.5,
+      "learning_rate": 0.0009995596371637897,
+      "loss": 34.3327,
+      "step": 20075
+    },
+    {
+      "epoch": 0.06277444299393492,
+      "grad_norm": 43.25,
+      "learning_rate": 0.0009995542034574863,
+      "loss": 34.3871,
+      "step": 20100
+    },
+    {
+      "epoch": 0.06285252065935026,
+      "grad_norm": 42.75,
+      "learning_rate": 0.0009995487364477004,
+      "loss": 33.8116,
+      "step": 20125
+    },
+    {
+      "epoch": 0.06293059832476561,
+      "grad_norm": 37.5,
+      "learning_rate": 0.0009995432361347971,
+      "loss": 33.9015,
+      "step": 20150
+    },
+    {
+      "epoch": 0.06300867599018095,
+      "grad_norm": 38.5,
+      "learning_rate": 0.0009995377025191427,
+      "loss": 33.8639,
+      "step": 20175
+    },
+    {
+      "epoch": 0.0630867536555963,
+      "grad_norm": 37.25,
+      "learning_rate": 0.0009995321356011063,
+      "loss": 33.6663,
+      "step": 20200
+    },
+    {
+      "epoch": 0.06316483132101164,
+      "grad_norm": 40.5,
+      "learning_rate": 0.0009995265353810589,
+      "loss": 33.8264,
+      "step": 20225
+    },
+    {
+      "epoch": 0.06324290898642698,
+      "grad_norm": 45.25,
+      "learning_rate": 0.0009995209018593737,
+      "loss": 33.6851,
+      "step": 20250
+    },
+    {
+      "epoch": 0.06332098665184233,
+      "grad_norm": 42.0,
+      "learning_rate": 0.0009995152350364266,
+      "loss": 33.5799,
+      "step": 20275
+    },
+    {
+      "epoch": 0.06339906431725766,
+      "grad_norm": 43.25,
+      "learning_rate": 0.000999509534912595,
+      "loss": 33.6905,
+      "step": 20300
+    },
+    {
+      "epoch": 0.063477141982673,
+      "grad_norm": 37.25,
+      "learning_rate": 0.0009995038014882593,
+      "loss": 33.4839,
+      "step": 20325
+    },
+    {
+      "epoch": 0.06355521964808834,
+      "grad_norm": 35.75,
+      "learning_rate": 0.0009994980347638016,
+      "loss": 33.6105,
+      "step": 20350
+    },
+    {
+      "epoch": 0.06363329731350369,
+      "grad_norm": 38.0,
+      "learning_rate": 0.0009994922347396063,
+      "loss": 33.9047,
+      "step": 20375
+    },
+    {
+      "epoch": 0.06371137497891903,
+      "grad_norm": 40.25,
+      "learning_rate": 0.00099948640141606,
+      "loss": 34.1876,
+      "step": 20400
+    },
+    {
+      "epoch": 0.06378945264433437,
+      "grad_norm": 45.75,
+      "learning_rate": 0.0009994805347935517,
+      "loss": 33.9303,
+      "step": 20425
+    },
+    {
+      "epoch": 0.06386753030974972,
+      "grad_norm": 42.75,
+      "learning_rate": 0.0009994746348724727,
+      "loss": 33.951,
+      "step": 20450
+    },
+    {
+      "epoch": 0.06394560797516506,
+      "grad_norm": 50.0,
+      "learning_rate": 0.000999468701653216,
+      "loss": 34.056,
+      "step": 20475
+    },
+    {
+      "epoch": 0.0640236856405804,
+      "grad_norm": 50.5,
+      "learning_rate": 0.0009994627351361772,
+      "loss": 33.9114,
+      "step": 20500
+    },
+    {
+      "epoch": 0.06410176330599573,
+      "grad_norm": 42.25,
+      "learning_rate": 0.0009994567353217541,
+      "loss": 34.2422,
+      "step": 20525
+    },
+    {
+      "epoch": 0.06417984097141108,
+      "grad_norm": 44.25,
+      "learning_rate": 0.0009994507022103465,
+      "loss": 34.0631,
+      "step": 20550
+    },
+    {
+      "epoch": 0.06425791863682642,
+      "grad_norm": 39.75,
+      "learning_rate": 0.000999444635802357,
+      "loss": 33.8447,
+      "step": 20575
+    },
+    {
+      "epoch": 0.06433599630224177,
+      "grad_norm": 44.75,
+      "learning_rate": 0.00099943853609819,
+      "loss": 33.8587,
+      "step": 20600
+    },
+    {
+      "epoch": 0.06441407396765711,
+      "grad_norm": 39.25,
+      "learning_rate": 0.0009994324030982518,
+      "loss": 33.943,
+      "step": 20625
+    },
+    {
+      "epoch": 0.06449215163307245,
+      "grad_norm": 41.75,
+      "learning_rate": 0.0009994262368029515,
+      "loss": 33.9425,
+      "step": 20650
+    },
+    {
+      "epoch": 0.0645702292984878,
+      "grad_norm": 44.5,
+      "learning_rate": 0.0009994200372127,
+      "loss": 34.0832,
+      "step": 20675
+    },
+    {
+      "epoch": 0.06464830696390314,
+      "grad_norm": 39.25,
+      "learning_rate": 0.000999413804327911,
+      "loss": 33.9888,
+      "step": 20700
+    },
+    {
+      "epoch": 0.06472638462931847,
+      "grad_norm": 43.75,
+      "learning_rate": 0.0009994075381489994,
+      "loss": 34.1022,
+      "step": 20725
+    },
+    {
+      "epoch": 0.06480446229473381,
+      "grad_norm": 44.25,
+      "learning_rate": 0.0009994012386763836,
+      "loss": 33.9719,
+      "step": 20750
+    },
+    {
+      "epoch": 0.06488253996014916,
+      "grad_norm": 42.0,
+      "learning_rate": 0.000999394905910483,
+      "loss": 33.7568,
+      "step": 20775
+    },
+    {
+      "epoch": 0.0649606176255645,
+      "grad_norm": 43.75,
+      "learning_rate": 0.0009993885398517201,
+      "loss": 33.7079,
+      "step": 20800
+    },
+    {
+      "epoch": 0.06503869529097984,
+      "grad_norm": 40.0,
+      "learning_rate": 0.0009993821405005195,
+      "loss": 33.8396,
+      "step": 20825
+    },
+    {
+      "epoch": 0.06511677295639519,
+      "grad_norm": 42.5,
+      "learning_rate": 0.0009993757078573073,
+      "loss": 33.6027,
+      "step": 20850
+    },
+    {
+      "epoch": 0.06519485062181053,
+      "grad_norm": 42.5,
+      "learning_rate": 0.0009993692419225126,
+      "loss": 33.5388,
+      "step": 20875
+    },
+    {
+      "epoch": 0.06527292828722588,
+      "grad_norm": 55.0,
+      "learning_rate": 0.0009993627426965667,
+      "loss": 33.775,
+      "step": 20900
+    },
+    {
+      "epoch": 0.0653510059526412,
+      "grad_norm": 39.0,
+      "learning_rate": 0.0009993562101799024,
+      "loss": 33.8984,
+      "step": 20925
+    },
+    {
+      "epoch": 0.06542908361805655,
+      "grad_norm": 41.5,
+      "learning_rate": 0.0009993496443729557,
+      "loss": 33.8582,
+      "step": 20950
+    },
+    {
+      "epoch": 0.06550716128347189,
+      "grad_norm": 37.25,
+      "learning_rate": 0.0009993430452761639,
+      "loss": 33.8915,
+      "step": 20975
+    },
+    {
+      "epoch": 0.06558523894888724,
+      "grad_norm": 35.0,
+      "learning_rate": 0.0009993364128899672,
+      "loss": 33.5705,
+      "step": 21000
+    },
+    {
+      "epoch": 0.06558523894888724,
+      "eval_loss": 33.73247146606445,
+      "eval_runtime": 102.3252,
+      "eval_samples_per_second": 50.848,
+      "eval_steps_per_second": 3.186,
+      "step": 21000
     }
   ],
   "logging_steps": 25,
@@ -5787,7 +6075,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.53630733446493e+18,
+  "total_flos": 2.663111367480836e+18,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null