Training in progress, step 2000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e3e9ee8a611e7a57dccff25563a008747ed15810194baa91980ef853c11a0a7
+oid sha256:3b160330a699e6391aabdd6c326d1ca2154af597460c4109b821f3a27a3de51f
 size 12017472

last-checkpoint/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:075920d4fe69625abfe8ade60f18025bd5df07d45e21e94c515e87ef9a80ae16
+size 71982309

last-checkpoint/global_step2000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edecabba5f405ae2044dcff16f3f5c2a1215ca2313484c1f864808f888ecf949
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step1500
+global_step2000

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27afcafd6ed5692d8873208ba0cf57e46a0701e5eb0aa08cd9750d1e2b88cb5d
+oid sha256:d92ac44cc5eabc6a5deb9b9de409e8c10d46ff0d44b4e3a5b61bcb9e4a0349fe
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 1500,
-  "best_metric": 0.6834071278572083,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-1500",
-  "epoch": 1.0901654244682786,
+  "best_global_step": 2000,
+  "best_metric": 0.6596384644508362,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-2000",
+  "epoch": 1.4537356844210143,
   "eval_steps": 250,
-  "global_step": 1500,
+  "global_step": 2000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -617,6 +617,206 @@
       "eval_samples_per_second": 43.157,
       "eval_steps_per_second": 5.401,
       "step": 1500
+    },
+    {
+      "epoch": 1.1083439374659152,
+      "grad_norm": 0.988451361656189,
+      "learning_rate": 7.852108931102753e-05,
+      "loss": 0.6679,
+      "mean_token_accuracy": 0.7947028204798698,
+      "num_tokens": 33602251.0,
+      "step": 1525
+    },
+    {
+      "epoch": 1.126522450463552,
+      "grad_norm": 1.0758918523788452,
+      "learning_rate": 7.846907896999776e-05,
+      "loss": 0.6738,
+      "mean_token_accuracy": 0.7931618624925614,
+      "num_tokens": 34152752.0,
+      "step": 1550
+    },
+    {
+      "epoch": 1.1447009634611889,
+      "grad_norm": 0.9699676036834717,
+      "learning_rate": 7.841618764560739e-05,
+      "loss": 0.6814,
+      "mean_token_accuracy": 0.7907909327745437,
+      "num_tokens": 34714148.0,
+      "step": 1575
+    },
+    {
+      "epoch": 1.1628794764588257,
+      "grad_norm": 0.9129726886749268,
+      "learning_rate": 7.836241654912474e-05,
+      "loss": 0.6799,
+      "mean_token_accuracy": 0.7906370875239372,
+      "num_tokens": 35275146.0,
+      "step": 1600
+    },
+    {
+      "epoch": 1.1810579894564626,
+      "grad_norm": 0.9198676347732544,
+      "learning_rate": 7.830776691196585e-05,
+      "loss": 0.6699,
+      "mean_token_accuracy": 0.7948868894577026,
+      "num_tokens": 35821013.0,
+      "step": 1625
+    },
+    {
+      "epoch": 1.1992365024540992,
+      "grad_norm": 0.9604835510253906,
+      "learning_rate": 7.825223998566632e-05,
+      "loss": 0.6855,
+      "mean_token_accuracy": 0.790110493004322,
+      "num_tokens": 36367326.0,
+      "step": 1650
+    },
+    {
+      "epoch": 1.217415015451736,
+      "grad_norm": 0.9292364716529846,
+      "learning_rate": 7.819583704185258e-05,
+      "loss": 0.6665,
+      "mean_token_accuracy": 0.7950288987159729,
+      "num_tokens": 36904100.0,
+      "step": 1675
+    },
+    {
+      "epoch": 1.2355935284493729,
+      "grad_norm": 0.9496335387229919,
+      "learning_rate": 7.813855937221283e-05,
+      "loss": 0.6793,
+      "mean_token_accuracy": 0.7916408607363701,
+      "num_tokens": 37451860.0,
+      "step": 1700
+    },
+    {
+      "epoch": 1.2537720414470097,
+      "grad_norm": 0.9605362415313721,
+      "learning_rate": 7.808040828846742e-05,
+      "loss": 0.6703,
+      "mean_token_accuracy": 0.7932550877332687,
+      "num_tokens": 38012329.0,
+      "step": 1725
+    },
+    {
+      "epoch": 1.2719505544446466,
+      "grad_norm": 0.9731937646865845,
+      "learning_rate": 7.80213851223388e-05,
+      "loss": 0.6631,
+      "mean_token_accuracy": 0.7963846024870872,
+      "num_tokens": 38545490.0,
+      "step": 1750
+    },
+    {
+      "epoch": 1.2719505544446466,
+      "eval_loss": 0.6696051359176636,
+      "eval_mean_token_accuracy": 0.792529649204678,
+      "eval_num_tokens": 38545490.0,
+      "eval_runtime": 114.5795,
+      "eval_samples_per_second": 42.678,
+      "eval_steps_per_second": 5.341,
+      "step": 1750
+    },
+    {
+      "epoch": 1.2901290674422832,
+      "grad_norm": 0.9220979809761047,
+      "learning_rate": 7.796149122552112e-05,
+      "loss": 0.6663,
+      "mean_token_accuracy": 0.7952693116664886,
+      "num_tokens": 39090734.0,
+      "step": 1775
+    },
+    {
+      "epoch": 1.30830758043992,
+      "grad_norm": 0.883160412311554,
+      "learning_rate": 7.790072796964914e-05,
+      "loss": 0.6645,
+      "mean_token_accuracy": 0.796334767639637,
+      "num_tokens": 39651191.0,
+      "step": 1800
+    },
+    {
+      "epoch": 1.3264860934375569,
+      "grad_norm": 0.940244734287262,
+      "learning_rate": 7.783909674626689e-05,
+      "loss": 0.6696,
+      "mean_token_accuracy": 0.794621022939682,
+      "num_tokens": 40201262.0,
+      "step": 1825
+    },
+    {
+      "epoch": 1.3446646064351935,
+      "grad_norm": 0.9481264352798462,
+      "learning_rate": 7.77765989667958e-05,
+      "loss": 0.6594,
+      "mean_token_accuracy": 0.797239051759243,
+      "num_tokens": 40729247.0,
+      "step": 1850
+    },
+    {
+      "epoch": 1.3628431194328303,
+      "grad_norm": 0.8973710536956787,
+      "learning_rate": 7.771323606250233e-05,
+      "loss": 0.6729,
+      "mean_token_accuracy": 0.7936947122216225,
+      "num_tokens": 41294203.0,
+      "step": 1875
+    },
+    {
+      "epoch": 1.3810216324304672,
+      "grad_norm": 0.9314188361167908,
+      "learning_rate": 7.764900948446533e-05,
+      "loss": 0.6673,
+      "mean_token_accuracy": 0.7956089550256729,
+      "num_tokens": 41844911.0,
+      "step": 1900
+    },
+    {
+      "epoch": 1.399200145428104,
+      "grad_norm": 0.9455300569534302,
+      "learning_rate": 7.758392070354259e-05,
+      "loss": 0.6705,
+      "mean_token_accuracy": 0.7935251343250275,
+      "num_tokens": 42404008.0,
+      "step": 1925
+    },
+    {
+      "epoch": 1.4173786584257408,
+      "grad_norm": 0.9419692754745483,
+      "learning_rate": 7.751797121033737e-05,
+      "loss": 0.6595,
+      "mean_token_accuracy": 0.7975886738300324,
+      "num_tokens": 42936579.0,
+      "step": 1950
+    },
+    {
+      "epoch": 1.4355571714233775,
+      "grad_norm": 0.8725437521934509,
+      "learning_rate": 7.745116251516407e-05,
+      "loss": 0.6603,
+      "mean_token_accuracy": 0.795488908290863,
+      "num_tokens": 43483670.0,
+      "step": 1975
+    },
+    {
+      "epoch": 1.4537356844210143,
+      "grad_norm": 0.9226874113082886,
+      "learning_rate": 7.738349614801387e-05,
+      "loss": 0.6642,
+      "mean_token_accuracy": 0.7964420530200005,
+      "num_tokens": 44044259.0,
+      "step": 2000
+    },
+    {
+      "epoch": 1.4537356844210143,
+      "eval_loss": 0.6596384644508362,
+      "eval_mean_token_accuracy": 0.7954148624847138,
+      "eval_num_tokens": 44044259.0,
+      "eval_runtime": 112.7192,
+      "eval_samples_per_second": 43.382,
+      "eval_steps_per_second": 5.429,
+      "step": 2000
     }
   ],
   "logging_steps": 25,
@@ -636,7 +836,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8.33876308084654e+16,
+  "total_flos": 1.111102656169902e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null