Training in progress, step 6000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e175cfbb3b1e5047d2a07a1f65e6011d48b21f6ac86f4b54bb7a003b3e25ddd9
+oid sha256:18b75b4abf894259d125060201a5ccf51810364d91fd3cea0d60d17a6403b9f6
 size 12017472

last-checkpoint/global_step6000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f20cbcc97cfbc77c9b6781af3fb327a8e590f45e5e58ba4784769f47e7581826
+size 71982309

last-checkpoint/global_step6000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4542b69e5e21b1b4b16e2e496d3d3499a0b7f568477280db01e3d784c01ad1d0
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step5500
+global_step6000

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c75d63279b47e795ad4622a2e3404a0983cd22c3e120a053ebabf9c78e50af21
+oid sha256:8941841568d79a9d2dc41b93071be9e47935d7e9eda669b8a7caaabc7faae599
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 5500,
-  "best_metric": 0.5829094648361206,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-5500",
-  "epoch": 3.997636793310307,
+  "best_global_step": 6000,
+  "best_metric": 0.578772783279419,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-6000",
+  "epoch": 4.360661697873114,
   "eval_steps": 250,
-  "global_step": 5500,
+  "global_step": 6000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2217,6 +2217,206 @@
       "eval_samples_per_second": 43.492,
       "eval_steps_per_second": 5.443,
       "step": 5500
+    },
+    {
+      "epoch": 4.0152699509180145,
+      "grad_norm": 0.7754128575325012,
+      "learning_rate": 6.030204764247823e-05,
+      "loss": 0.5672,
+      "mean_token_accuracy": 0.825141193633227,
+      "num_tokens": 121668495.0,
+      "step": 5525
+    },
+    {
+      "epoch": 4.033448463915652,
+      "grad_norm": 0.8203994631767273,
+      "learning_rate": 6.0136883795470986e-05,
+      "loss": 0.5715,
+      "mean_token_accuracy": 0.8235750555992126,
+      "num_tokens": 122202715.0,
+      "step": 5550
+    },
+    {
+      "epoch": 4.051626976913289,
+      "grad_norm": 0.7772579193115234,
+      "learning_rate": 5.997125879212641e-05,
+      "loss": 0.567,
+      "mean_token_accuracy": 0.8232873624563217,
+      "num_tokens": 122751905.0,
+      "step": 5575
+    },
+    {
+      "epoch": 4.069805489910926,
+      "grad_norm": 0.7624217867851257,
+      "learning_rate": 5.9805176425435554e-05,
+      "loss": 0.5715,
+      "mean_token_accuracy": 0.8219173887372017,
+      "num_tokens": 123296741.0,
+      "step": 5600
+    },
+    {
+      "epoch": 4.087984002908562,
+      "grad_norm": 0.8489812612533569,
+      "learning_rate": 5.963864049886357e-05,
+      "loss": 0.5643,
+      "mean_token_accuracy": 0.8252894932031631,
+      "num_tokens": 123839156.0,
+      "step": 5625
+    },
+    {
+      "epoch": 4.106162515906199,
+      "grad_norm": 0.7331461906433105,
+      "learning_rate": 5.947165482626263e-05,
+      "loss": 0.5717,
+      "mean_token_accuracy": 0.8214939990639687,
+      "num_tokens": 124404023.0,
+      "step": 5650
+    },
+    {
+      "epoch": 4.124341028903836,
+      "grad_norm": 0.8406916856765747,
+      "learning_rate": 5.930422323178458e-05,
+      "loss": 0.5707,
+      "mean_token_accuracy": 0.8230401134490967,
+      "num_tokens": 124946377.0,
+      "step": 5675
+    },
+    {
+      "epoch": 4.142519541901472,
+      "grad_norm": 0.8017742037773132,
+      "learning_rate": 5.9136349549793323e-05,
+      "loss": 0.5714,
+      "mean_token_accuracy": 0.8239132612943649,
+      "num_tokens": 125485916.0,
+      "step": 5700
+    },
+    {
+      "epoch": 4.160698054899109,
+      "grad_norm": 0.7596368789672852,
+      "learning_rate": 5.89680376247771e-05,
+      "loss": 0.567,
+      "mean_token_accuracy": 0.8242393881082535,
+      "num_tokens": 126031951.0,
+      "step": 5725
+    },
+    {
+      "epoch": 4.178876567896746,
+      "grad_norm": 0.8254388570785522,
+      "learning_rate": 5.879929131126035e-05,
+      "loss": 0.5663,
+      "mean_token_accuracy": 0.8245700207352639,
+      "num_tokens": 126583391.0,
+      "step": 5750
+    },
+    {
+      "epoch": 4.178876567896746,
+      "eval_loss": 0.5802226662635803,
+      "eval_mean_token_accuracy": 0.8190604671348933,
+      "eval_num_tokens": 126583391.0,
+      "eval_runtime": 113.2756,
+      "eval_samples_per_second": 43.169,
+      "eval_steps_per_second": 5.403,
+      "step": 5750
+    },
+    {
+      "epoch": 4.1970550808943825,
+      "grad_norm": 0.7768835425376892,
+      "learning_rate": 5.8630114473715466e-05,
+      "loss": 0.571,
+      "mean_token_accuracy": 0.822128147482872,
+      "num_tokens": 127141535.0,
+      "step": 5775
+    },
+    {
+      "epoch": 4.21523359389202,
+      "grad_norm": 0.800995945930481,
+      "learning_rate": 5.846051098647433e-05,
+      "loss": 0.566,
+      "mean_token_accuracy": 0.8245965147018433,
+      "num_tokens": 127703604.0,
+      "step": 5800
+    },
+    {
+      "epoch": 4.233412106889657,
+      "grad_norm": 0.8068668246269226,
+      "learning_rate": 5.8290484733639566e-05,
+      "loss": 0.5738,
+      "mean_token_accuracy": 0.8229434779286384,
+      "num_tokens": 128229875.0,
+      "step": 5825
+    },
+    {
+      "epoch": 4.251590619887293,
+      "grad_norm": 0.7782077789306641,
+      "learning_rate": 5.812003960899557e-05,
+      "loss": 0.5698,
+      "mean_token_accuracy": 0.8222619444131851,
+      "num_tokens": 128785210.0,
+      "step": 5850
+    },
+    {
+      "epoch": 4.26976913288493,
+      "grad_norm": 0.7673845887184143,
+      "learning_rate": 5.7949179515919366e-05,
+      "loss": 0.5725,
+      "mean_token_accuracy": 0.8225021129846573,
+      "num_tokens": 129340256.0,
+      "step": 5875
+    },
+    {
+      "epoch": 4.287947645882567,
+      "grad_norm": 0.7331883311271667,
+      "learning_rate": 5.777790836729117e-05,
+      "loss": 0.5642,
+      "mean_token_accuracy": 0.8251486110687256,
+      "num_tokens": 129879617.0,
+      "step": 5900
+    },
+    {
+      "epoch": 4.306126158880204,
+      "grad_norm": 0.8080233931541443,
+      "learning_rate": 5.760623008540487e-05,
+      "loss": 0.5637,
+      "mean_token_accuracy": 0.8252076309919357,
+      "num_tokens": 130438521.0,
+      "step": 5925
+    },
+    {
+      "epoch": 4.32430467187784,
+      "grad_norm": 0.8210439085960388,
+      "learning_rate": 5.743414860187809e-05,
+      "loss": 0.5667,
+      "mean_token_accuracy": 0.8240964418649673,
+      "num_tokens": 130988259.0,
+      "step": 5950
+    },
+    {
+      "epoch": 4.342483184875477,
+      "grad_norm": 0.7626243233680725,
+      "learning_rate": 5.726166785756224e-05,
+      "loss": 0.5722,
+      "mean_token_accuracy": 0.822701002061367,
+      "num_tokens": 131539988.0,
+      "step": 5975
+    },
+    {
+      "epoch": 4.360661697873114,
+      "grad_norm": 0.8430207967758179,
+      "learning_rate": 5.708879180245222e-05,
+      "loss": 0.5656,
+      "mean_token_accuracy": 0.8251918998360633,
+      "num_tokens": 132079465.0,
+      "step": 6000
+    },
+    {
+      "epoch": 4.360661697873114,
+      "eval_loss": 0.578772783279419,
+      "eval_mean_token_accuracy": 0.8196869551940681,
+      "eval_num_tokens": 132079465.0,
+      "eval_runtime": 112.7171,
+      "eval_samples_per_second": 43.383,
+      "eval_steps_per_second": 5.43,
+      "step": 6000
     }
   ],
   "logging_steps": 25,
@@ -2236,7 +2436,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.0548540768282214e+17,
+  "total_flos": 3.3318287854639514e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null