Training in progress, step 9000, checkpoint

last-checkpoint/global_step9000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b2e4b5173ecac6a9f3534c1b14e6d36ec29577616c64122a8c1e0f65db43555
+size 5117197489

last-checkpoint/global_step9000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dc66a01d1ac7b2fd44ab1e6565c0a596ae931e03269eec0d60d8dbe27476c99
+size 859127933

last-checkpoint/latest

@@ -1 +1 @@
-global_step8000
+global_step9000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0823a45cd5f5f262d4113d9c6af3e480a93b1328d895090e11d3841575e98029
+oid sha256:aef828c688fc4b40c4f970b4f1621324009e1b6de86d3a3ed65007b337b7f7e7
 size 962205216

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f2132517ec1780cf0e43d2d85e0457c9953dabc448540c499dce25e57e2b052
+oid sha256:64e572f1314b9da8f922a0fbf0c91986e4b7b809f9a1dbb178f491f4b7541f4c
 size 14709

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cba89e97c806c2994342d3ee7fc823d23ef358301180bf2dcf6ac57f1ab3869c
+oid sha256:ec7cb829bad4c5e40215f974eb8875988bba1a68c4193a01021b2b11b0d8359f
 size 1465

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 84.13012729844414,
   "best_model_checkpoint": "./iteboshi_student_model_temp/checkpoint-7000",
-  "epoch": 8.810572687224669,
+  "epoch": 9.911894273127754,
   "eval_steps": 1000,
-  "global_step": 8000,
+  "global_step": 9000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2327,6 +2327,296 @@
       "eval_steps_per_second": 2.028,
       "eval_wer": 85.53512494106553,
       "step": 8000
+    },
+    {
+      "epoch": 8.838105726872246,
+      "grad_norm": 0.33088424801826477,
+      "learning_rate": 1.2282051282051283e-05,
+      "loss": 0.049,
+      "step": 8025
+    },
+    {
+      "epoch": 8.865638766519824,
+      "grad_norm": 0.3602592945098877,
+      "learning_rate": 1.2256410256410259e-05,
+      "loss": 0.0572,
+      "step": 8050
+    },
+    {
+      "epoch": 8.8931718061674,
+      "grad_norm": 0.35909441113471985,
+      "learning_rate": 1.2230769230769232e-05,
+      "loss": 0.051,
+      "step": 8075
+    },
+    {
+      "epoch": 8.920704845814978,
+      "grad_norm": 0.45318055152893066,
+      "learning_rate": 1.2205128205128208e-05,
+      "loss": 0.0642,
+      "step": 8100
+    },
+    {
+      "epoch": 8.948237885462555,
+      "grad_norm": 1.001381754875183,
+      "learning_rate": 1.217948717948718e-05,
+      "loss": 0.0522,
+      "step": 8125
+    },
+    {
+      "epoch": 8.975770925110131,
+      "grad_norm": 0.5000578761100769,
+      "learning_rate": 1.2153846153846153e-05,
+      "loss": 0.0423,
+      "step": 8150
+    },
+    {
+      "epoch": 9.003303964757709,
+      "grad_norm": 0.29771438241004944,
+      "learning_rate": 1.2128205128205129e-05,
+      "loss": 0.063,
+      "step": 8175
+    },
+    {
+      "epoch": 9.030837004405287,
+      "grad_norm": 0.43198081851005554,
+      "learning_rate": 1.2102564102564102e-05,
+      "loss": 0.0416,
+      "step": 8200
+    },
+    {
+      "epoch": 9.058370044052863,
+      "grad_norm": 0.31377923488616943,
+      "learning_rate": 1.2076923076923078e-05,
+      "loss": 0.0358,
+      "step": 8225
+    },
+    {
+      "epoch": 9.08590308370044,
+      "grad_norm": 1.0352481603622437,
+      "learning_rate": 1.2051282051282051e-05,
+      "loss": 0.0348,
+      "step": 8250
+    },
+    {
+      "epoch": 9.113436123348018,
+      "grad_norm": 0.421749472618103,
+      "learning_rate": 1.2025641025641027e-05,
+      "loss": 0.0403,
+      "step": 8275
+    },
+    {
+      "epoch": 9.140969162995594,
+      "grad_norm": 0.39680853486061096,
+      "learning_rate": 1.2e-05,
+      "loss": 0.0348,
+      "step": 8300
+    },
+    {
+      "epoch": 9.168502202643172,
+      "grad_norm": 0.25886261463165283,
+      "learning_rate": 1.1974358974358976e-05,
+      "loss": 0.0255,
+      "step": 8325
+    },
+    {
+      "epoch": 9.19603524229075,
+      "grad_norm": 0.29378727078437805,
+      "learning_rate": 1.194871794871795e-05,
+      "loss": 0.0421,
+      "step": 8350
+    },
+    {
+      "epoch": 9.223568281938325,
+      "grad_norm": 0.14189021289348602,
+      "learning_rate": 1.1923076923076925e-05,
+      "loss": 0.0346,
+      "step": 8375
+    },
+    {
+      "epoch": 9.251101321585903,
+      "grad_norm": 0.3648456335067749,
+      "learning_rate": 1.1897435897435898e-05,
+      "loss": 0.0358,
+      "step": 8400
+    },
+    {
+      "epoch": 9.27863436123348,
+      "grad_norm": 0.22953101992607117,
+      "learning_rate": 1.1871794871794872e-05,
+      "loss": 0.0377,
+      "step": 8425
+    },
+    {
+      "epoch": 9.306167400881057,
+      "grad_norm": 0.13100098073482513,
+      "learning_rate": 1.1846153846153847e-05,
+      "loss": 0.0345,
+      "step": 8450
+    },
+    {
+      "epoch": 9.333700440528634,
+      "grad_norm": 0.41983163356781006,
+      "learning_rate": 1.1820512820512821e-05,
+      "loss": 0.0358,
+      "step": 8475
+    },
+    {
+      "epoch": 9.361233480176212,
+      "grad_norm": 0.28245824575424194,
+      "learning_rate": 1.1794871794871796e-05,
+      "loss": 0.0467,
+      "step": 8500
+    },
+    {
+      "epoch": 9.388766519823788,
+      "grad_norm": 0.46235162019729614,
+      "learning_rate": 1.176923076923077e-05,
+      "loss": 0.0407,
+      "step": 8525
+    },
+    {
+      "epoch": 9.416299559471366,
+      "grad_norm": 0.4608246684074402,
+      "learning_rate": 1.1743589743589745e-05,
+      "loss": 0.0407,
+      "step": 8550
+    },
+    {
+      "epoch": 9.443832599118943,
+      "grad_norm": 0.32346612215042114,
+      "learning_rate": 1.1717948717948719e-05,
+      "loss": 0.0347,
+      "step": 8575
+    },
+    {
+      "epoch": 9.47136563876652,
+      "grad_norm": 0.3498935401439667,
+      "learning_rate": 1.1692307692307694e-05,
+      "loss": 0.0413,
+      "step": 8600
+    },
+    {
+      "epoch": 9.498898678414097,
+      "grad_norm": 0.48518890142440796,
+      "learning_rate": 1.1666666666666668e-05,
+      "loss": 0.0503,
+      "step": 8625
+    },
+    {
+      "epoch": 9.526431718061675,
+      "grad_norm": 0.20332852005958557,
+      "learning_rate": 1.1641025641025643e-05,
+      "loss": 0.0472,
+      "step": 8650
+    },
+    {
+      "epoch": 9.55396475770925,
+      "grad_norm": 0.3680901825428009,
+      "learning_rate": 1.1615384615384617e-05,
+      "loss": 0.039,
+      "step": 8675
+    },
+    {
+      "epoch": 9.581497797356828,
+      "grad_norm": 0.4770890176296234,
+      "learning_rate": 1.1589743589743592e-05,
+      "loss": 0.0325,
+      "step": 8700
+    },
+    {
+      "epoch": 9.609030837004406,
+      "grad_norm": 0.3051774501800537,
+      "learning_rate": 1.1564102564102566e-05,
+      "loss": 0.0406,
+      "step": 8725
+    },
+    {
+      "epoch": 9.636563876651982,
+      "grad_norm": 0.4181124269962311,
+      "learning_rate": 1.1538461538461538e-05,
+      "loss": 0.0425,
+      "step": 8750
+    },
+    {
+      "epoch": 9.66409691629956,
+      "grad_norm": 0.3570977449417114,
+      "learning_rate": 1.1512820512820513e-05,
+      "loss": 0.0349,
+      "step": 8775
+    },
+    {
+      "epoch": 9.691629955947137,
+      "grad_norm": 0.40113458037376404,
+      "learning_rate": 1.1487179487179487e-05,
+      "loss": 0.0332,
+      "step": 8800
+    },
+    {
+      "epoch": 9.719162995594713,
+      "grad_norm": 0.5174753665924072,
+      "learning_rate": 1.1461538461538462e-05,
+      "loss": 0.0365,
+      "step": 8825
+    },
+    {
+      "epoch": 9.746696035242291,
+      "grad_norm": 0.6039919853210449,
+      "learning_rate": 1.1435897435897436e-05,
+      "loss": 0.0423,
+      "step": 8850
+    },
+    {
+      "epoch": 9.774229074889869,
+      "grad_norm": 0.33104389905929565,
+      "learning_rate": 1.1410256410256411e-05,
+      "loss": 0.0492,
+      "step": 8875
+    },
+    {
+      "epoch": 9.801762114537445,
+      "grad_norm": 0.5245941877365112,
+      "learning_rate": 1.1384615384615385e-05,
+      "loss": 0.0403,
+      "step": 8900
+    },
+    {
+      "epoch": 9.829295154185022,
+      "grad_norm": 0.4713122546672821,
+      "learning_rate": 1.135897435897436e-05,
+      "loss": 0.0539,
+      "step": 8925
+    },
+    {
+      "epoch": 9.8568281938326,
+      "grad_norm": 0.3693457841873169,
+      "learning_rate": 1.1333333333333334e-05,
+      "loss": 0.0419,
+      "step": 8950
+    },
+    {
+      "epoch": 9.884361233480176,
+      "grad_norm": 0.4538707733154297,
+      "learning_rate": 1.1307692307692309e-05,
+      "loss": 0.0345,
+      "step": 8975
+    },
+    {
+      "epoch": 9.911894273127754,
+      "grad_norm": 0.25317588448524475,
+      "learning_rate": 1.1282051282051283e-05,
+      "loss": 0.0329,
+      "step": 9000
+    },
+    {
+      "epoch": 9.911894273127754,
+      "eval_cer": 24.903992755035365,
+      "eval_loss": 0.8292160034179688,
+      "eval_runtime": 1301.6266,
+      "eval_samples_per_second": 8.129,
+      "eval_steps_per_second": 2.033,
+      "eval_wer": 84.72418670438473,
+      "step": 9000
     }
   ],
   "logging_steps": 25,
@@ -2346,7 +2636,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.3745145491920781e+20,
+  "total_flos": 1.546328867841088e+20,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null