Training in progress, step 5000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4cac545c69a5d0e5299593a55caef83ea42a94b1015f419b3f94c28959dee30c
+oid sha256:81eac720b158c7f43a3b9b48f3c680e3548bab4820189790d8de2f257ac92036
 size 12017472

last-checkpoint/global_step5000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24b636b0dcd0034e45777d07aa99efeeb1e9bd93768e06fd11b065259b652903
+size 71982309

last-checkpoint/global_step5000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce9f51afe184e05ff59d6f27becb3371bfb5d8d783725100888f6fa45968627f
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step4500
+global_step5000

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32ebf6aab10b0ca8b50125889413e725d88350e766faf5e41b640aefb228c7f9
+oid sha256:3dd594d08139e0846701d4c186ee22eb3ed05631cdda05ef04a8843616048835
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 4500,
-  "best_metric": 0.5972464680671692,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-4500",
-  "epoch": 3.2704962734048353,
+  "best_global_step": 5000,
+  "best_metric": 0.5900602340698242,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-5000",
+  "epoch": 3.6340665333575712,
   "eval_steps": 250,
-  "global_step": 4500,
+  "global_step": 5000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1817,6 +1817,206 @@
       "eval_samples_per_second": 43.784,
       "eval_steps_per_second": 5.48,
       "step": 4500
+    },
+    {
+      "epoch": 3.2886747864024723,
+      "grad_norm": 0.7829866409301758,
+      "learning_rate": 6.64882949515662e-05,
+      "loss": 0.5908,
+      "mean_token_accuracy": 0.8166057634353637,
+      "num_tokens": 99645620.0,
+      "step": 4525
+    },
+    {
+      "epoch": 3.306853299400109,
+      "grad_norm": 0.8376955389976501,
+      "learning_rate": 6.634455689277093e-05,
+      "loss": 0.5982,
+      "mean_token_accuracy": 0.8151650968194007,
+      "num_tokens": 100194494.0,
+      "step": 4550
+    },
+    {
+      "epoch": 3.325031812397746,
+      "grad_norm": 0.8199899792671204,
+      "learning_rate": 6.620021551523535e-05,
+      "loss": 0.5958,
+      "mean_token_accuracy": 0.8154050391912461,
+      "num_tokens": 100737169.0,
+      "step": 4575
+    },
+    {
+      "epoch": 3.3432103253953827,
+      "grad_norm": 0.8258052468299866,
+      "learning_rate": 6.605527412453255e-05,
+      "loss": 0.5923,
+      "mean_token_accuracy": 0.8159717765450477,
+      "num_tokens": 101281003.0,
+      "step": 4600
+    },
+    {
+      "epoch": 3.3613888383930193,
+      "grad_norm": 0.8162449598312378,
+      "learning_rate": 6.590973603997654e-05,
+      "loss": 0.5911,
+      "mean_token_accuracy": 0.8167745867371559,
+      "num_tokens": 101832682.0,
+      "step": 4625
+    },
+    {
+      "epoch": 3.3795673513906563,
+      "grad_norm": 0.8238963484764099,
+      "learning_rate": 6.57636045945463e-05,
+      "loss": 0.5831,
+      "mean_token_accuracy": 0.8192883601784706,
+      "num_tokens": 102378220.0,
+      "step": 4650
+    },
+    {
+      "epoch": 3.397745864388293,
+      "grad_norm": 0.7991573214530945,
+      "learning_rate": 6.561688313480939e-05,
+      "loss": 0.5911,
+      "mean_token_accuracy": 0.817092213332653,
+      "num_tokens": 102926110.0,
+      "step": 4675
+    },
+    {
+      "epoch": 3.41592437738593,
+      "grad_norm": 0.8273572325706482,
+      "learning_rate": 6.546957502084532e-05,
+      "loss": 0.586,
+      "mean_token_accuracy": 0.8180428540706635,
+      "num_tokens": 103478461.0,
+      "step": 4700
+    },
+    {
+      "epoch": 3.4341028903835666,
+      "grad_norm": 0.8324493169784546,
+      "learning_rate": 6.532168362616866e-05,
+      "loss": 0.5855,
+      "mean_token_accuracy": 0.8184285718202591,
+      "num_tokens": 104030963.0,
+      "step": 4725
+    },
+    {
+      "epoch": 3.4522814033812033,
+      "grad_norm": 0.8379449844360352,
+      "learning_rate": 6.517321233765167e-05,
+      "loss": 0.5864,
+      "mean_token_accuracy": 0.8167688602209091,
+      "num_tokens": 104589296.0,
+      "step": 4750
+    },
+    {
+      "epoch": 3.4522814033812033,
+      "eval_loss": 0.593262791633606,
+      "eval_mean_token_accuracy": 0.8153588095911188,
+      "eval_num_tokens": 104589296.0,
+      "eval_runtime": 112.1372,
+      "eval_samples_per_second": 43.607,
+      "eval_steps_per_second": 5.458,
+      "step": 4750
+    },
+    {
+      "epoch": 3.4704599163788403,
+      "grad_norm": 0.7619993686676025,
+      "learning_rate": 6.502416455544687e-05,
+      "loss": 0.5902,
+      "mean_token_accuracy": 0.8169645836949349,
+      "num_tokens": 105136117.0,
+      "step": 4775
+    },
+    {
+      "epoch": 3.488638429376477,
+      "grad_norm": 0.8142008781433105,
+      "learning_rate": 6.487454369290907e-05,
+      "loss": 0.5805,
+      "mean_token_accuracy": 0.819823622405529,
+      "num_tokens": 105676793.0,
+      "step": 4800
+    },
+    {
+      "epoch": 3.5068169423741136,
+      "grad_norm": 0.7336195111274719,
+      "learning_rate": 6.472435317651725e-05,
+      "loss": 0.5836,
+      "mean_token_accuracy": 0.8191494596004486,
+      "num_tokens": 106237943.0,
+      "step": 4825
+    },
+    {
+      "epoch": 3.5249954553717506,
+      "grad_norm": 0.7633249759674072,
+      "learning_rate": 6.457359644579607e-05,
+      "loss": 0.5845,
+      "mean_token_accuracy": 0.8191626858711243,
+      "num_tokens": 106773949.0,
+      "step": 4850
+    },
+    {
+      "epoch": 3.5431739683693873,
+      "grad_norm": 0.7786067724227905,
+      "learning_rate": 6.44222769532371e-05,
+      "loss": 0.5854,
+      "mean_token_accuracy": 0.8184454745054245,
+      "num_tokens": 107324714.0,
+      "step": 4875
+    },
+    {
+      "epoch": 3.5613524813670243,
+      "grad_norm": 0.7811067700386047,
+      "learning_rate": 6.42703981642198e-05,
+      "loss": 0.5833,
+      "mean_token_accuracy": 0.8200912246108055,
+      "num_tokens": 107873998.0,
+      "step": 4900
+    },
+    {
+      "epoch": 3.579530994364661,
+      "grad_norm": 0.8161619901657104,
+      "learning_rate": 6.411796355693206e-05,
+      "loss": 0.591,
+      "mean_token_accuracy": 0.8165828287601471,
+      "num_tokens": 108419757.0,
+      "step": 4925
+    },
+    {
+      "epoch": 3.597709507362298,
+      "grad_norm": 0.8599359393119812,
+      "learning_rate": 6.396497662229067e-05,
+      "loss": 0.5843,
+      "mean_token_accuracy": 0.8185628071427345,
+      "num_tokens": 108960024.0,
+      "step": 4950
+    },
+    {
+      "epoch": 3.6158880203599346,
+      "grad_norm": 0.7843888401985168,
+      "learning_rate": 6.381144086386126e-05,
+      "loss": 0.5803,
+      "mean_token_accuracy": 0.819676850438118,
+      "num_tokens": 109508330.0,
+      "step": 4975
+    },
+    {
+      "epoch": 3.6340665333575712,
+      "grad_norm": 0.7887631058692932,
+      "learning_rate": 6.365735979777816e-05,
+      "loss": 0.5944,
+      "mean_token_accuracy": 0.8151102581620217,
+      "num_tokens": 110076014.0,
+      "step": 5000
+    },
+    {
+      "epoch": 3.6340665333575712,
+      "eval_loss": 0.5900602340698242,
+      "eval_mean_token_accuracy": 0.8163315599260766,
+      "eval_num_tokens": 110076014.0,
+      "eval_runtime": 111.9302,
+      "eval_samples_per_second": 43.688,
+      "eval_steps_per_second": 5.468,
+      "step": 5000
     }
   ],
   "logging_steps": 25,
@@ -1836,7 +2036,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.498553276362916e+17,
+  "total_flos": 2.7767572610102067e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null