Training in progress, step 7500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e18628340448d6cc6532411e4697ff75d543f26260e6f7133ea8b56b72f7242
+oid sha256:e218a5bbad6f972caad0894b0b220511e0cb0cb3787c44a85e875a3ce67f3813
 size 12017472

last-checkpoint/global_step7500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f0c6a776f557f95f4a5de2e5594e410d029df9d00a110f71924423c987a5e8b
+size 71982309

last-checkpoint/global_step7500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc6aedf984eb49a766b5397998d1ccdee863f9d0b635a66e9096c9fb5555965a
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step7000
+global_step7500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26376c5c3b634bbc75ff000a6f0bd179c575c9ddbba230c22db308946450acc0
+oid sha256:f5c58bb9b510ddfd192f4d2021c0156080905e1ad4e17052f1d4a70bda5c74ec
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 7000,
-  "best_metric": 0.5689130425453186,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-7000",
-  "epoch": 5.0872568623886565,
+  "best_global_step": 7500,
+  "best_metric": 0.5647426843643188,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-7500",
+  "epoch": 5.450827122341392,
   "eval_steps": 250,
-  "global_step": 7000,
+  "global_step": 7500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2817,6 +2817,206 @@
       "eval_samples_per_second": 43.506,
       "eval_steps_per_second": 5.445,
       "step": 7000
+    },
+    {
+      "epoch": 5.1054353753862936,
+      "grad_norm": 0.7935928702354431,
+      "learning_rate": 4.971038696306446e-05,
+      "loss": 0.5501,
+      "mean_token_accuracy": 0.8285538706183434,
+      "num_tokens": 154698091.0,
+      "step": 7025
+    },
+    {
+      "epoch": 5.123613888383931,
+      "grad_norm": 0.7594472169876099,
+      "learning_rate": 4.952458208239385e-05,
+      "loss": 0.5487,
+      "mean_token_accuracy": 0.8303073984384537,
+      "num_tokens": 155238389.0,
+      "step": 7050
+    },
+    {
+      "epoch": 5.141792401381567,
+      "grad_norm": 0.7793622016906738,
+      "learning_rate": 4.933855907853041e-05,
+      "loss": 0.5526,
+      "mean_token_accuracy": 0.828688297867775,
+      "num_tokens": 155796109.0,
+      "step": 7075
+    },
+    {
+      "epoch": 5.159970914379204,
+      "grad_norm": 0.7963124513626099,
+      "learning_rate": 4.9152322211601326e-05,
+      "loss": 0.5617,
+      "mean_token_accuracy": 0.8250745138525963,
+      "num_tokens": 156367817.0,
+      "step": 7100
+    },
+    {
+      "epoch": 5.178149427376841,
+      "grad_norm": 0.815303385257721,
+      "learning_rate": 4.8965875746631553e-05,
+      "loss": 0.5527,
+      "mean_token_accuracy": 0.8272364658117294,
+      "num_tokens": 156937564.0,
+      "step": 7125
+    },
+    {
+      "epoch": 5.196327940374477,
+      "grad_norm": 0.7769586443901062,
+      "learning_rate": 4.8779223953446054e-05,
+      "loss": 0.5539,
+      "mean_token_accuracy": 0.8281649795174598,
+      "num_tokens": 157487986.0,
+      "step": 7150
+    },
+    {
+      "epoch": 5.214506453372114,
+      "grad_norm": 0.7640786170959473,
+      "learning_rate": 4.8592371106571984e-05,
+      "loss": 0.5553,
+      "mean_token_accuracy": 0.8278635969758034,
+      "num_tokens": 158049502.0,
+      "step": 7175
+    },
+    {
+      "epoch": 5.232684966369751,
+      "grad_norm": 0.7943294644355774,
+      "learning_rate": 4.8405321485140926e-05,
+      "loss": 0.5515,
+      "mean_token_accuracy": 0.8292607891559601,
+      "num_tokens": 158573810.0,
+      "step": 7200
+    },
+    {
+      "epoch": 5.250863479367387,
+      "grad_norm": 0.8353666067123413,
+      "learning_rate": 4.821807937279074e-05,
+      "loss": 0.5493,
+      "mean_token_accuracy": 0.8291581255197525,
+      "num_tokens": 159126021.0,
+      "step": 7225
+    },
+    {
+      "epoch": 5.2690419923650245,
+      "grad_norm": 0.8323714137077332,
+      "learning_rate": 4.8030649057567545e-05,
+      "loss": 0.5574,
+      "mean_token_accuracy": 0.8271696311235428,
+      "num_tokens": 159687774.0,
+      "step": 7250
+    },
+    {
+      "epoch": 5.2690419923650245,
+      "eval_loss": 0.5662592053413391,
+      "eval_mean_token_accuracy": 0.8236163130967445,
+      "eval_num_tokens": 159687774.0,
+      "eval_runtime": 112.894,
+      "eval_samples_per_second": 43.315,
+      "eval_steps_per_second": 5.421,
+      "step": 7250
+    },
+    {
+      "epoch": 5.2872205053626615,
+      "grad_norm": 0.7205966114997864,
+      "learning_rate": 4.784303483182755e-05,
+      "loss": 0.553,
+      "mean_token_accuracy": 0.8278142037987709,
+      "num_tokens": 160241228.0,
+      "step": 7275
+    },
+    {
+      "epoch": 5.305399018360298,
+      "grad_norm": 0.8180447816848755,
+      "learning_rate": 4.7655240992138677e-05,
+      "loss": 0.5491,
+      "mean_token_accuracy": 0.829489229619503,
+      "num_tokens": 160767430.0,
+      "step": 7300
+    },
+    {
+      "epoch": 5.323577531357935,
+      "grad_norm": 0.7637699842453003,
+      "learning_rate": 4.746727183918221e-05,
+      "loss": 0.5595,
+      "mean_token_accuracy": 0.8261320424079895,
+      "num_tokens": 161318820.0,
+      "step": 7325
+    },
+    {
+      "epoch": 5.341756044355572,
+      "grad_norm": 0.7907775640487671,
+      "learning_rate": 4.727913167765431e-05,
+      "loss": 0.5525,
+      "mean_token_accuracy": 0.8275396654009819,
+      "num_tokens": 161877946.0,
+      "step": 7350
+    },
+    {
+      "epoch": 5.359934557353209,
+      "grad_norm": 0.7488855719566345,
+      "learning_rate": 4.7090824816167384e-05,
+      "loss": 0.5516,
+      "mean_token_accuracy": 0.8294497436285019,
+      "num_tokens": 162425201.0,
+      "step": 7375
+    },
+    {
+      "epoch": 5.378113070350845,
+      "grad_norm": 0.8791596293449402,
+      "learning_rate": 4.6902355567151486e-05,
+      "loss": 0.5533,
+      "mean_token_accuracy": 0.8279849541187286,
+      "num_tokens": 162980062.0,
+      "step": 7400
+    },
+    {
+      "epoch": 5.396291583348482,
+      "grad_norm": 0.8288064002990723,
+      "learning_rate": 4.671372824675549e-05,
+      "loss": 0.5463,
+      "mean_token_accuracy": 0.8298707720637322,
+      "num_tokens": 163536063.0,
+      "step": 7425
+    },
+    {
+      "epoch": 5.414470096346119,
+      "grad_norm": 0.8334540724754333,
+      "learning_rate": 4.65249471747483e-05,
+      "loss": 0.5488,
+      "mean_token_accuracy": 0.8298037537932396,
+      "num_tokens": 164096275.0,
+      "step": 7450
+    },
+    {
+      "epoch": 5.432648609343755,
+      "grad_norm": 0.7712641358375549,
+      "learning_rate": 4.6336016674419886e-05,
+      "loss": 0.5423,
+      "mean_token_accuracy": 0.8307902818918228,
+      "num_tokens": 164633060.0,
+      "step": 7475
+    },
+    {
+      "epoch": 5.450827122341392,
+      "grad_norm": 0.7974975109100342,
+      "learning_rate": 4.614694107248228e-05,
+      "loss": 0.5527,
+      "mean_token_accuracy": 0.8289200633764267,
+      "num_tokens": 165169760.0,
+      "step": 7500
+    },
+    {
+      "epoch": 5.450827122341392,
+      "eval_loss": 0.5647426843643188,
+      "eval_mean_token_accuracy": 0.82398138930595,
+      "eval_num_tokens": 165169760.0,
+      "eval_runtime": 113.2742,
+      "eval_samples_per_second": 43.17,
+      "eval_steps_per_second": 5.403,
+      "step": 7500
     }
   ],
   "logging_steps": 25,
@@ -2836,7 +3036,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.888375314430034e+17,
+  "total_flos": 4.16634638434304e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null