Training in progress, step 16500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:748ac312d2e44467fcda4fec5d9101ba80583460b392693c16cb03d972d0d373
+oid sha256:84490a0ba5e68272e718e1b587f957c02192d8adb3af51f4ee9f3e16c57d8791
 size 12017472

last-checkpoint/global_step16500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecddbe1aff6e55389d660e2ef4b3e28b3ddd7f5e51d8ff2861b1f4dfb4439e89
+size 71982309

last-checkpoint/global_step16500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a3e559e7998fe0b3e9109e5a1fcca3316d4f86d200dd5db4d8ea7b7da1c59c0
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step16000
+global_step16500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e7f4f223e9a53b3be578c7196145efd3d2905fa7329c5f3661f540d7f3b0a13
+oid sha256:25f34d42adfb703043e77cccecb301eb61eb2fb980871ddcbacc2771de139a9c
 size 14709

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": 16000,
   "best_metric": 0.5378558039665222,
   "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-16000",
-  "epoch": 11.628249409198327,
+  "epoch": 11.991819669151063,
   "eval_steps": 250,
-  "global_step": 16000,
+  "global_step": 16500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -6417,6 +6417,206 @@
       "eval_samples_per_second": 43.873,
       "eval_steps_per_second": 5.491,
       "step": 16000
+    },
+    {
+      "epoch": 11.646427922195965,
+      "grad_norm": 0.8521804213523865,
+      "learning_rate": 1.7368012049631344e-07,
+      "loss": 0.5054,
+      "mean_token_accuracy": 0.8433844763040542,
+      "num_tokens": 352828019.0,
+      "step": 16025
+    },
+    {
+      "epoch": 11.664606435193601,
+      "grad_norm": 0.8544715642929077,
+      "learning_rate": 1.5631750216815733e-07,
+      "loss": 0.5127,
+      "mean_token_accuracy": 0.8406408429145813,
+      "num_tokens": 353383848.0,
+      "step": 16050
+    },
+    {
+      "epoch": 11.682784948191237,
+      "grad_norm": 0.8408353924751282,
+      "learning_rate": 1.398673471025891e-07,
+      "loss": 0.5119,
+      "mean_token_accuracy": 0.8391477057337761,
+      "num_tokens": 353947699.0,
+      "step": 16075
+    },
+    {
+      "epoch": 11.700963461188875,
+      "grad_norm": 0.9233898520469666,
+      "learning_rate": 1.2433003202588113e-07,
+      "loss": 0.5064,
+      "mean_token_accuracy": 0.8429214411973953,
+      "num_tokens": 354490412.0,
+      "step": 16100
+    },
+    {
+      "epoch": 11.719141974186511,
+      "grad_norm": 0.8499307036399841,
+      "learning_rate": 1.0970591275929476e-07,
+      "loss": 0.4985,
+      "mean_token_accuracy": 0.8445121836662293,
+      "num_tokens": 355031092.0,
+      "step": 16125
+    },
+    {
+      "epoch": 11.737320487184148,
+      "grad_norm": 0.814639687538147,
+      "learning_rate": 9.599532421092239e-08,
+      "loss": 0.4986,
+      "mean_token_accuracy": 0.8441943645477294,
+      "num_tokens": 355575121.0,
+      "step": 16150
+    },
+    {
+      "epoch": 11.755499000181786,
+      "grad_norm": 0.9159456491470337,
+      "learning_rate": 8.31985803680091e-08,
+      "loss": 0.5077,
+      "mean_token_accuracy": 0.8415706121921539,
+      "num_tokens": 356143674.0,
+      "step": 16175
+    },
+    {
+      "epoch": 11.773677513179422,
+      "grad_norm": 0.9033915996551514,
+      "learning_rate": 7.13159742897851e-08,
+      "loss": 0.5093,
+      "mean_token_accuracy": 0.8420270484685898,
+      "num_tokens": 356687741.0,
+      "step": 16200
+    },
+    {
+      "epoch": 11.791856026177058,
+      "grad_norm": 0.9012133479118347,
+      "learning_rate": 6.034777810072445e-08,
+      "loss": 0.5157,
+      "mean_token_accuracy": 0.8388536632061004,
+      "num_tokens": 357249933.0,
+      "step": 16225
+    },
+    {
+      "epoch": 11.810034539174696,
+      "grad_norm": 0.8706744909286499,
+      "learning_rate": 5.0294242984345595e-08,
+      "loss": 0.5077,
+      "mean_token_accuracy": 0.8408184266090393,
+      "num_tokens": 357820486.0,
+      "step": 16250
+    },
+    {
+      "epoch": 11.810034539174696,
+      "eval_loss": 0.5378739237785339,
+      "eval_mean_token_accuracy": 0.8327080023054984,
+      "eval_num_tokens": 357820486.0,
+      "eval_runtime": 111.4072,
+      "eval_samples_per_second": 43.893,
+      "eval_steps_per_second": 5.493,
+      "step": 16250
+    },
+    {
+      "epoch": 11.828213052172332,
+      "grad_norm": 0.8344452381134033,
+      "learning_rate": 4.1155599177433725e-08,
+      "loss": 0.5054,
+      "mean_token_accuracy": 0.842702434360981,
+      "num_tokens": 358376267.0,
+      "step": 16275
+    },
+    {
+      "epoch": 11.846391565169968,
+      "grad_norm": 0.9577372074127197,
+      "learning_rate": 3.293205596477833e-08,
+      "loss": 0.5119,
+      "mean_token_accuracy": 0.8426962018013,
+      "num_tokens": 358925005.0,
+      "step": 16300
+    },
+    {
+      "epoch": 11.864570078167606,
+      "grad_norm": 0.8274015188217163,
+      "learning_rate": 2.5623801674381498e-08,
+      "loss": 0.5108,
+      "mean_token_accuracy": 0.8408406323194504,
+      "num_tokens": 359493299.0,
+      "step": 16325
+    },
+    {
+      "epoch": 11.882748591165242,
+      "grad_norm": 0.8797160387039185,
+      "learning_rate": 1.9231003673145788e-08,
+      "loss": 0.5047,
+      "mean_token_accuracy": 0.842636145055294,
+      "num_tokens": 360053289.0,
+      "step": 16350
+    },
+    {
+      "epoch": 11.900927104162879,
+      "grad_norm": 0.8720398545265198,
+      "learning_rate": 1.375380836302398e-08,
+      "loss": 0.5006,
+      "mean_token_accuracy": 0.8434941950440407,
+      "num_tokens": 360612686.0,
+      "step": 16375
+    },
+    {
+      "epoch": 11.919105617160517,
+      "grad_norm": 0.951812207698822,
+      "learning_rate": 9.192341177697295e-09,
+      "loss": 0.5031,
+      "mean_token_accuracy": 0.8426809054613114,
+      "num_tokens": 361163960.0,
+      "step": 16400
+    },
+    {
+      "epoch": 11.937284130158153,
+      "grad_norm": 0.8881998062133789,
+      "learning_rate": 5.546706579679928e-09,
+      "loss": 0.5146,
+      "mean_token_accuracy": 0.8401396802067757,
+      "num_tokens": 361709569.0,
+      "step": 16425
+    },
+    {
+      "epoch": 11.95546264315579,
+      "grad_norm": 0.864645779132843,
+      "learning_rate": 2.816988057929848e-09,
+      "loss": 0.516,
+      "mean_token_accuracy": 0.838928511440754,
+      "num_tokens": 362266243.0,
+      "step": 16450
+    },
+    {
+      "epoch": 11.973641156153427,
+      "grad_norm": 0.8667661547660828,
+      "learning_rate": 1.0032481259436566e-09,
+      "loss": 0.5195,
+      "mean_token_accuracy": 0.8384470102190972,
+      "num_tokens": 362824221.0,
+      "step": 16475
+    },
+    {
+      "epoch": 11.991819669151063,
+      "grad_norm": 0.8674113154411316,
+      "learning_rate": 1.0552832032217908e-10,
+      "loss": 0.508,
+      "mean_token_accuracy": 0.8425227817893028,
+      "num_tokens": 363367808.0,
+      "step": 16500
+    },
+    {
+      "epoch": 11.991819669151063,
+      "eval_loss": 0.5378732681274414,
+      "eval_mean_token_accuracy": 0.8326478998840244,
+      "eval_num_tokens": 363367808.0,
+      "eval_runtime": 111.938,
+      "eval_samples_per_second": 43.685,
+      "eval_steps_per_second": 5.467,
+      "step": 16500
     }
   ],
   "logging_steps": 25,
@@ -6436,7 +6636,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8.886881404730737e+17,
+  "total_flos": 9.164882254873231e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null