Training in progress, step 2500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b160330a699e6391aabdd6c326d1ca2154af597460c4109b821f3a27a3de51f
+oid sha256:e52bad05f2e4c26960ed218d0c8eb65c9304d2d27be834a92d821653ed150b67
 size 12017472

last-checkpoint/global_step2500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6003a02a1a66ef1745b42ac42443bddc34528a9fef8726db27fedbc72adb1572
+size 71982309

last-checkpoint/global_step2500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a5fc9fcbc092435c4e9b986a9ec53b21b22bc833e0c004c3f820523d58970b6
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step2000
+global_step2500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d92ac44cc5eabc6a5deb9b9de409e8c10d46ff0d44b4e3a5b61bcb9e4a0349fe
+oid sha256:af223d92fe6846f9d1e5ce7aaf1ae97c0e4e19a087e2147be916f38012f3d229
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 2000,
-  "best_metric": 0.6596384644508362,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-2000",
-  "epoch": 1.4537356844210143,
+  "best_global_step": 2500,
+  "best_metric": 0.6409846544265747,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-2500",
+  "epoch": 1.8173059443737503,
   "eval_steps": 250,
-  "global_step": 2000,
+  "global_step": 2500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -817,6 +817,206 @@
       "eval_samples_per_second": 43.382,
       "eval_steps_per_second": 5.429,
       "step": 2000
+    },
+    {
+      "epoch": 1.4719141974186511,
+      "grad_norm": 1.093906283378601,
+      "learning_rate": 7.731497365851944e-05,
+      "loss": 0.66,
+      "mean_token_accuracy": 0.7957050919532775,
+      "num_tokens": 44603467.0,
+      "step": 2025
+    },
+    {
+      "epoch": 1.490092710416288,
+      "grad_norm": 0.8411886692047119,
+      "learning_rate": 7.724559661591966e-05,
+      "loss": 0.6492,
+      "mean_token_accuracy": 0.799662963449955,
+      "num_tokens": 45144337.0,
+      "step": 2050
+    },
+    {
+      "epoch": 1.5082712234139248,
+      "grad_norm": 0.9079028964042664,
+      "learning_rate": 7.717536660902353e-05,
+      "loss": 0.6535,
+      "mean_token_accuracy": 0.7987899404764175,
+      "num_tokens": 45708073.0,
+      "step": 2075
+    },
+    {
+      "epoch": 1.5264497364115615,
+      "grad_norm": 0.9115111827850342,
+      "learning_rate": 7.710428524617389e-05,
+      "loss": 0.6516,
+      "mean_token_accuracy": 0.7993895325064659,
+      "num_tokens": 46249985.0,
+      "step": 2100
+    },
+    {
+      "epoch": 1.5446282494091983,
+      "grad_norm": 0.8034014105796814,
+      "learning_rate": 7.703235415521057e-05,
+      "loss": 0.6553,
+      "mean_token_accuracy": 0.7976609247922898,
+      "num_tokens": 46795146.0,
+      "step": 2125
+    },
+    {
+      "epoch": 1.5628067624068351,
+      "grad_norm": 1.0506081581115723,
+      "learning_rate": 7.695957498343304e-05,
+      "loss": 0.6542,
+      "mean_token_accuracy": 0.7982049816846848,
+      "num_tokens": 47345330.0,
+      "step": 2150
+    },
+    {
+      "epoch": 1.5809852754044718,
+      "grad_norm": 0.9649513959884644,
+      "learning_rate": 7.688594939756276e-05,
+      "loss": 0.6548,
+      "mean_token_accuracy": 0.7982990917563438,
+      "num_tokens": 47896343.0,
+      "step": 2175
+    },
+    {
+      "epoch": 1.5991637884021088,
+      "grad_norm": 0.8364529609680176,
+      "learning_rate": 7.681147908370497e-05,
+      "loss": 0.6476,
+      "mean_token_accuracy": 0.8009107887744904,
+      "num_tokens": 48443987.0,
+      "step": 2200
+    },
+    {
+      "epoch": 1.6173423013997454,
+      "grad_norm": 0.8900915384292603,
+      "learning_rate": 7.673616574731013e-05,
+      "loss": 0.6664,
+      "mean_token_accuracy": 0.796454921066761,
+      "num_tokens": 48993810.0,
+      "step": 2225
+    },
+    {
+      "epoch": 1.6355208143973823,
+      "grad_norm": 0.8416359424591064,
+      "learning_rate": 7.666001111313477e-05,
+      "loss": 0.656,
+      "mean_token_accuracy": 0.7976474016904831,
+      "num_tokens": 49564541.0,
+      "step": 2250
+    },
+    {
+      "epoch": 1.6355208143973823,
+      "eval_loss": 0.648926854133606,
+      "eval_mean_token_accuracy": 0.7985351690474678,
+      "eval_num_tokens": 49564541.0,
+      "eval_runtime": 111.8774,
+      "eval_samples_per_second": 43.709,
+      "eval_steps_per_second": 5.47,
+      "step": 2250
+    },
+    {
+      "epoch": 1.6536993273950191,
+      "grad_norm": 0.836439311504364,
+      "learning_rate": 7.658301692520209e-05,
+      "loss": 0.642,
+      "mean_token_accuracy": 0.8027165573835373,
+      "num_tokens": 50098122.0,
+      "step": 2275
+    },
+    {
+      "epoch": 1.6718778403926557,
+      "grad_norm": 0.8868879079818726,
+      "learning_rate": 7.650518494676194e-05,
+      "loss": 0.6537,
+      "mean_token_accuracy": 0.7993291038274765,
+      "num_tokens": 50648590.0,
+      "step": 2300
+    },
+    {
+      "epoch": 1.6900563533902928,
+      "grad_norm": 0.8488360047340393,
+      "learning_rate": 7.642651696025052e-05,
+      "loss": 0.6403,
+      "mean_token_accuracy": 0.8029101991653442,
+      "num_tokens": 51215679.0,
+      "step": 2325
+    },
+    {
+      "epoch": 1.7082348663879294,
+      "grad_norm": 0.8410452604293823,
+      "learning_rate": 7.634701476724948e-05,
+      "loss": 0.6528,
+      "mean_token_accuracy": 0.798929190337658,
+      "num_tokens": 51783858.0,
+      "step": 2350
+    },
+    {
+      "epoch": 1.7264133793855663,
+      "grad_norm": 0.8173678517341614,
+      "learning_rate": 7.626668018844469e-05,
+      "loss": 0.6545,
+      "mean_token_accuracy": 0.7984850916266442,
+      "num_tokens": 52329463.0,
+      "step": 2375
+    },
+    {
+      "epoch": 1.7445918923832031,
+      "grad_norm": 0.8305994868278503,
+      "learning_rate": 7.618551506358459e-05,
+      "loss": 0.6444,
+      "mean_token_accuracy": 0.8014543145895004,
+      "num_tokens": 52868102.0,
+      "step": 2400
+    },
+    {
+      "epoch": 1.7627704053808397,
+      "grad_norm": 0.8392990231513977,
+      "learning_rate": 7.610352125143798e-05,
+      "loss": 0.6407,
+      "mean_token_accuracy": 0.8039175960421562,
+      "num_tokens": 53412329.0,
+      "step": 2425
+    },
+    {
+      "epoch": 1.7809489183784768,
+      "grad_norm": 0.8528268337249756,
+      "learning_rate": 7.602070062975153e-05,
+      "loss": 0.6418,
+      "mean_token_accuracy": 0.802329548895359,
+      "num_tokens": 53960577.0,
+      "step": 2450
+    },
+    {
+      "epoch": 1.7991274313761134,
+      "grad_norm": 0.8892678022384644,
+      "learning_rate": 7.593705509520669e-05,
+      "loss": 0.6442,
+      "mean_token_accuracy": 0.801820527613163,
+      "num_tokens": 54508868.0,
+      "step": 2475
+    },
+    {
+      "epoch": 1.8173059443737503,
+      "grad_norm": 0.858299195766449,
+      "learning_rate": 7.585258656337637e-05,
+      "loss": 0.6464,
+      "mean_token_accuracy": 0.8014724615216255,
+      "num_tokens": 55070505.0,
+      "step": 2500
+    },
+    {
+      "epoch": 1.8173059443737503,
+      "eval_loss": 0.6409846544265747,
+      "eval_mean_token_accuracy": 0.8009895614159652,
+      "eval_num_tokens": 55070505.0,
+      "eval_runtime": 112.4439,
+      "eval_samples_per_second": 43.488,
+      "eval_steps_per_second": 5.443,
+      "step": 2500
     }
   ],
   "logging_steps": 25,
@@ -836,7 +1036,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.111102656169902e+17,
+  "total_flos": 1.3887661057612186e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null