Training in progress, step 8000, checkpoint

last-checkpoint/global_step8000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aa42ddad4af96c74b65d7385afeb6f0d5fb2d664599cd656cd82fbd70ad62da
+size 5117197489

last-checkpoint/global_step8000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db5c9356593ff2b606f0316fc19dda6e153f3fa9391f9b150d5a17343e4eeeeb
+size 859127933

last-checkpoint/latest

@@ -1 +1 @@
-global_step7000
+global_step8000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:856733172381a37b6de12c25512bfa5cc33814241a1986b18ae46a3c6cd69ce1
+oid sha256:0823a45cd5f5f262d4113d9c6af3e480a93b1328d895090e11d3841575e98029
 size 962205216

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56a8ea7c15005d31ade663058f08a1d5a4619da6c77df5179c75f15bb9cc3f05
+oid sha256:2f2132517ec1780cf0e43d2d85e0457c9953dabc448540c499dce25e57e2b052
 size 14709

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34a83060f11df4fe46a27d45e8744a4c0e7bb60df156e5d496780133906eacd7
+oid sha256:cba89e97c806c2994342d3ee7fc823d23ef358301180bf2dcf6ac57f1ab3869c
 size 1465

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 84.13012729844414,
   "best_model_checkpoint": "./iteboshi_student_model_temp/checkpoint-7000",
-  "epoch": 7.709251101321586,
+  "epoch": 8.810572687224669,
   "eval_steps": 1000,
-  "global_step": 7000,
+  "global_step": 8000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2037,6 +2037,296 @@
       "eval_steps_per_second": 1.957,
       "eval_wer": 84.13012729844414,
       "step": 7000
+    },
+    {
+      "epoch": 7.736784140969163,
+      "grad_norm": 0.5741052627563477,
+      "learning_rate": 1.3307692307692309e-05,
+      "loss": 0.0909,
+      "step": 7025
+    },
+    {
+      "epoch": 7.76431718061674,
+      "grad_norm": 0.7617988586425781,
+      "learning_rate": 1.3282051282051282e-05,
+      "loss": 0.0753,
+      "step": 7050
+    },
+    {
+      "epoch": 7.791850220264317,
+      "grad_norm": 0.5416741967201233,
+      "learning_rate": 1.3256410256410258e-05,
+      "loss": 0.0731,
+      "step": 7075
+    },
+    {
+      "epoch": 7.819383259911894,
+      "grad_norm": 0.6349952816963196,
+      "learning_rate": 1.3230769230769231e-05,
+      "loss": 0.0703,
+      "step": 7100
+    },
+    {
+      "epoch": 7.846916299559472,
+      "grad_norm": 0.33196786046028137,
+      "learning_rate": 1.3205128205128207e-05,
+      "loss": 0.0656,
+      "step": 7125
+    },
+    {
+      "epoch": 7.8744493392070485,
+      "grad_norm": 0.379213809967041,
+      "learning_rate": 1.317948717948718e-05,
+      "loss": 0.0917,
+      "step": 7150
+    },
+    {
+      "epoch": 7.901982378854625,
+      "grad_norm": 0.34401291608810425,
+      "learning_rate": 1.3153846153846156e-05,
+      "loss": 0.0911,
+      "step": 7175
+    },
+    {
+      "epoch": 7.929515418502203,
+      "grad_norm": 0.4732189178466797,
+      "learning_rate": 1.312820512820513e-05,
+      "loss": 0.101,
+      "step": 7200
+    },
+    {
+      "epoch": 7.95704845814978,
+      "grad_norm": 0.5580617785453796,
+      "learning_rate": 1.3102564102564103e-05,
+      "loss": 0.0898,
+      "step": 7225
+    },
+    {
+      "epoch": 7.984581497797357,
+      "grad_norm": 0.42180871963500977,
+      "learning_rate": 1.3076923076923078e-05,
+      "loss": 0.086,
+      "step": 7250
+    },
+    {
+      "epoch": 8.012114537444933,
+      "grad_norm": 0.2615683376789093,
+      "learning_rate": 1.3051282051282052e-05,
+      "loss": 0.0898,
+      "step": 7275
+    },
+    {
+      "epoch": 8.039647577092511,
+      "grad_norm": 0.44722801446914673,
+      "learning_rate": 1.3025641025641027e-05,
+      "loss": 0.0602,
+      "step": 7300
+    },
+    {
+      "epoch": 8.067180616740089,
+      "grad_norm": 0.3499121367931366,
+      "learning_rate": 1.3000000000000001e-05,
+      "loss": 0.0549,
+      "step": 7325
+    },
+    {
+      "epoch": 8.094713656387665,
+      "grad_norm": 0.37767261266708374,
+      "learning_rate": 1.2974358974358976e-05,
+      "loss": 0.0573,
+      "step": 7350
+    },
+    {
+      "epoch": 8.122246696035242,
+      "grad_norm": 0.4645783007144928,
+      "learning_rate": 1.294871794871795e-05,
+      "loss": 0.0566,
+      "step": 7375
+    },
+    {
+      "epoch": 8.14977973568282,
+      "grad_norm": 0.6134966015815735,
+      "learning_rate": 1.2923076923076925e-05,
+      "loss": 0.0507,
+      "step": 7400
+    },
+    {
+      "epoch": 8.177312775330396,
+      "grad_norm": 0.28678062558174133,
+      "learning_rate": 1.2897435897435899e-05,
+      "loss": 0.051,
+      "step": 7425
+    },
+    {
+      "epoch": 8.204845814977974,
+      "grad_norm": 0.299078106880188,
+      "learning_rate": 1.2871794871794874e-05,
+      "loss": 0.045,
+      "step": 7450
+    },
+    {
+      "epoch": 8.232378854625551,
+      "grad_norm": 0.6386272311210632,
+      "learning_rate": 1.2846153846153848e-05,
+      "loss": 0.0665,
+      "step": 7475
+    },
+    {
+      "epoch": 8.259911894273127,
+      "grad_norm": 0.1515616923570633,
+      "learning_rate": 1.2820512820512823e-05,
+      "loss": 0.052,
+      "step": 7500
+    },
+    {
+      "epoch": 8.287444933920705,
+      "grad_norm": 0.61899733543396,
+      "learning_rate": 1.2794871794871795e-05,
+      "loss": 0.0462,
+      "step": 7525
+    },
+    {
+      "epoch": 8.314977973568283,
+      "grad_norm": 0.6535860300064087,
+      "learning_rate": 1.2769230769230769e-05,
+      "loss": 0.0518,
+      "step": 7550
+    },
+    {
+      "epoch": 8.342511013215859,
+      "grad_norm": 0.4084964692592621,
+      "learning_rate": 1.2743589743589744e-05,
+      "loss": 0.0574,
+      "step": 7575
+    },
+    {
+      "epoch": 8.370044052863436,
+      "grad_norm": 0.4185622036457062,
+      "learning_rate": 1.2717948717948718e-05,
+      "loss": 0.0466,
+      "step": 7600
+    },
+    {
+      "epoch": 8.397577092511014,
+      "grad_norm": 0.5417298078536987,
+      "learning_rate": 1.2692307692307693e-05,
+      "loss": 0.0595,
+      "step": 7625
+    },
+    {
+      "epoch": 8.42511013215859,
+      "grad_norm": 0.0882943645119667,
+      "learning_rate": 1.2666666666666667e-05,
+      "loss": 0.0441,
+      "step": 7650
+    },
+    {
+      "epoch": 8.452643171806168,
+      "grad_norm": 0.5028131008148193,
+      "learning_rate": 1.2641025641025642e-05,
+      "loss": 0.0584,
+      "step": 7675
+    },
+    {
+      "epoch": 8.480176211453745,
+      "grad_norm": 0.32492244243621826,
+      "learning_rate": 1.2615384615384616e-05,
+      "loss": 0.0519,
+      "step": 7700
+    },
+    {
+      "epoch": 8.507709251101321,
+      "grad_norm": 0.199100524187088,
+      "learning_rate": 1.2589743589743591e-05,
+      "loss": 0.0519,
+      "step": 7725
+    },
+    {
+      "epoch": 8.535242290748899,
+      "grad_norm": 0.560196578502655,
+      "learning_rate": 1.2564102564102565e-05,
+      "loss": 0.0601,
+      "step": 7750
+    },
+    {
+      "epoch": 8.562775330396477,
+      "grad_norm": 0.3848872780799866,
+      "learning_rate": 1.253846153846154e-05,
+      "loss": 0.0561,
+      "step": 7775
+    },
+    {
+      "epoch": 8.590308370044053,
+      "grad_norm": 0.6430539488792419,
+      "learning_rate": 1.2512820512820514e-05,
+      "loss": 0.0573,
+      "step": 7800
+    },
+    {
+      "epoch": 8.61784140969163,
+      "grad_norm": 0.1402147263288498,
+      "learning_rate": 1.2487179487179487e-05,
+      "loss": 0.0613,
+      "step": 7825
+    },
+    {
+      "epoch": 8.645374449339208,
+      "grad_norm": 0.3411605656147003,
+      "learning_rate": 1.2461538461538463e-05,
+      "loss": 0.0401,
+      "step": 7850
+    },
+    {
+      "epoch": 8.672907488986784,
+      "grad_norm": 0.4999459981918335,
+      "learning_rate": 1.2435897435897436e-05,
+      "loss": 0.0393,
+      "step": 7875
+    },
+    {
+      "epoch": 8.700440528634362,
+      "grad_norm": 0.6794586777687073,
+      "learning_rate": 1.2410256410256412e-05,
+      "loss": 0.061,
+      "step": 7900
+    },
+    {
+      "epoch": 8.72797356828194,
+      "grad_norm": 0.30914783477783203,
+      "learning_rate": 1.2384615384615385e-05,
+      "loss": 0.0552,
+      "step": 7925
+    },
+    {
+      "epoch": 8.755506607929515,
+      "grad_norm": 0.311613529920578,
+      "learning_rate": 1.235897435897436e-05,
+      "loss": 0.0556,
+      "step": 7950
+    },
+    {
+      "epoch": 8.783039647577093,
+      "grad_norm": 0.48470553755760193,
+      "learning_rate": 1.2333333333333334e-05,
+      "loss": 0.0553,
+      "step": 7975
+    },
+    {
+      "epoch": 8.810572687224669,
+      "grad_norm": 0.5605005621910095,
+      "learning_rate": 1.230769230769231e-05,
+      "loss": 0.0673,
+      "step": 8000
+    },
+    {
+      "epoch": 8.810572687224669,
+      "eval_cer": 25.173386218518225,
+      "eval_loss": 0.8125333189964294,
+      "eval_runtime": 1304.9459,
+      "eval_samples_per_second": 8.108,
+      "eval_steps_per_second": 2.028,
+      "eval_wer": 85.53512494106553,
+      "step": 8000
     }
   ],
   "logging_steps": 25,
@@ -2056,7 +2346,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.2027002305430684e+20,
+  "total_flos": 1.3745145491920781e+20,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null