Training in progress, step 3500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e8f2e633fe08d7eb7ff6e133c7cb6469ced1b595814d785aa46e33c6e65f452
+oid sha256:27d5b8ad136d1b37d0b53dfe5e54ffe63b01050eddd9539c59d73e1c91aa67b3
 size 12017472

last-checkpoint/global_step3500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cda2758edb2fb2c8a863a8995389fecad1ddd0807037e9db5f15db85ead9758d
+size 71982309

last-checkpoint/global_step3500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cbbf95b02a2067a2a4f20c353239d123a04495655920bfb42eb24afdd147c85
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step3000
+global_step3500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7573584526c2fe8e68dba3ab40f8f3cffab852e01de9ee2eb0aaf2a4192e0852
+oid sha256:327d808c225b6c78ea6a068082f3d00dba54671d88051aa14e820dd408eeac44
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 3000,
-  "best_metric": 0.6261406540870667,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-3000",
-  "epoch": 2.180330848936557,
+  "best_global_step": 3500,
+  "best_metric": 0.614472508430481,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-3500",
+  "epoch": 2.543901108889293,
   "eval_steps": 250,
-  "global_step": 3000,
+  "global_step": 3500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1217,6 +1217,206 @@
       "eval_samples_per_second": 43.351,
       "eval_steps_per_second": 5.425,
       "step": 3000
+    },
+    {
+      "epoch": 2.1985093619341938,
+      "grad_norm": 0.8226723074913025,
+      "learning_rate": 7.389222583480705e-05,
+      "loss": 0.6243,
+      "mean_token_accuracy": 0.8068913269042969,
+      "num_tokens": 66616416.0,
+      "step": 3025
+    },
+    {
+      "epoch": 2.2166878749318304,
+      "grad_norm": 0.8199797868728638,
+      "learning_rate": 7.379017366112587e-05,
+      "loss": 0.628,
+      "mean_token_accuracy": 0.8060924589633942,
+      "num_tokens": 67170306.0,
+      "step": 3050
+    },
+    {
+      "epoch": 2.2348663879294675,
+      "grad_norm": 0.8197723627090454,
+      "learning_rate": 7.368734765605741e-05,
+      "loss": 0.6125,
+      "mean_token_accuracy": 0.8108021330833435,
+      "num_tokens": 67707041.0,
+      "step": 3075
+    },
+    {
+      "epoch": 2.253044900927104,
+      "grad_norm": 0.8319080471992493,
+      "learning_rate": 7.358375017442797e-05,
+      "loss": 0.6204,
+      "mean_token_accuracy": 0.8081632897257804,
+      "num_tokens": 68243518.0,
+      "step": 3100
+    },
+    {
+      "epoch": 2.271223413924741,
+      "grad_norm": 0.8893775343894958,
+      "learning_rate": 7.347938358873149e-05,
+      "loss": 0.6138,
+      "mean_token_accuracy": 0.8099391725659371,
+      "num_tokens": 68787369.0,
+      "step": 3125
+    },
+    {
+      "epoch": 2.2894019269223778,
+      "grad_norm": 0.8154735565185547,
+      "learning_rate": 7.337425028907528e-05,
+      "loss": 0.6178,
+      "mean_token_accuracy": 0.8098280015587807,
+      "num_tokens": 69334791.0,
+      "step": 3150
+    },
+    {
+      "epoch": 2.3075804399200144,
+      "grad_norm": 0.8006751537322998,
+      "learning_rate": 7.326835268312518e-05,
+      "loss": 0.6158,
+      "mean_token_accuracy": 0.8086746591329574,
+      "num_tokens": 69884826.0,
+      "step": 3175
+    },
+    {
+      "epoch": 2.3257589529176514,
+      "grad_norm": 0.8786169290542603,
+      "learning_rate": 7.316169319605046e-05,
+      "loss": 0.6269,
+      "mean_token_accuracy": 0.8071727818250656,
+      "num_tokens": 70442756.0,
+      "step": 3200
+    },
+    {
+      "epoch": 2.343937465915288,
+      "grad_norm": 0.9075261950492859,
+      "learning_rate": 7.30542742704683e-05,
+      "loss": 0.6201,
+      "mean_token_accuracy": 0.8087817251682281,
+      "num_tokens": 70993300.0,
+      "step": 3225
+    },
+    {
+      "epoch": 2.362115978912925,
+      "grad_norm": 0.8171051740646362,
+      "learning_rate": 7.294609836638787e-05,
+      "loss": 0.6188,
+      "mean_token_accuracy": 0.8082248848676682,
+      "num_tokens": 71543391.0,
+      "step": 3250
+    },
+    {
+      "epoch": 2.362115978912925,
+      "eval_loss": 0.621147096157074,
+      "eval_mean_token_accuracy": 0.8068399137141657,
+      "eval_num_tokens": 71543391.0,
+      "eval_runtime": 112.0822,
+      "eval_samples_per_second": 43.629,
+      "eval_steps_per_second": 5.46,
+      "step": 3250
+    },
+    {
+      "epoch": 2.3802944919105617,
+      "grad_norm": 0.8513513207435608,
+      "learning_rate": 7.283716796115393e-05,
+      "loss": 0.6187,
+      "mean_token_accuracy": 0.8077478906512261,
+      "num_tokens": 72109371.0,
+      "step": 3275
+    },
+    {
+      "epoch": 2.3984730049081984,
+      "grad_norm": 0.901434063911438,
+      "learning_rate": 7.272748554939012e-05,
+      "loss": 0.6135,
+      "mean_token_accuracy": 0.8108441984653473,
+      "num_tokens": 72661191.0,
+      "step": 3300
+    },
+    {
+      "epoch": 2.4166515179058354,
+      "grad_norm": 0.8363370895385742,
+      "learning_rate": 7.261705364294188e-05,
+      "loss": 0.6124,
+      "mean_token_accuracy": 0.8096053293347358,
+      "num_tokens": 73213412.0,
+      "step": 3325
+    },
+    {
+      "epoch": 2.434830030903472,
+      "grad_norm": 0.875728189945221,
+      "learning_rate": 7.250587477081885e-05,
+      "loss": 0.6215,
+      "mean_token_accuracy": 0.8084959277510643,
+      "num_tokens": 73764375.0,
+      "step": 3350
+    },
+    {
+      "epoch": 2.4530085439011087,
+      "grad_norm": 0.7723637819290161,
+      "learning_rate": 7.2393951479137e-05,
+      "loss": 0.6066,
+      "mean_token_accuracy": 0.8145261201262474,
+      "num_tokens": 74309911.0,
+      "step": 3375
+    },
+    {
+      "epoch": 2.4711870568987457,
+      "grad_norm": 0.8123798370361328,
+      "learning_rate": 7.228128633106032e-05,
+      "loss": 0.6111,
+      "mean_token_accuracy": 0.8112738102674484,
+      "num_tokens": 74856337.0,
+      "step": 3400
+    },
+    {
+      "epoch": 2.4893655698963824,
+      "grad_norm": 0.8313596844673157,
+      "learning_rate": 7.21678819067421e-05,
+      "loss": 0.6258,
+      "mean_token_accuracy": 0.8076113468408584,
+      "num_tokens": 75410769.0,
+      "step": 3425
+    },
+    {
+      "epoch": 2.5075440828940194,
+      "grad_norm": 0.8260684013366699,
+      "learning_rate": 7.205374080326585e-05,
+      "loss": 0.6147,
+      "mean_token_accuracy": 0.8095012375712395,
+      "num_tokens": 75963770.0,
+      "step": 3450
+    },
+    {
+      "epoch": 2.525722595891656,
+      "grad_norm": 0.7737406492233276,
+      "learning_rate": 7.193886563458585e-05,
+      "loss": 0.6191,
+      "mean_token_accuracy": 0.8077240213751793,
+      "num_tokens": 76528809.0,
+      "step": 3475
+    },
+    {
+      "epoch": 2.543901108889293,
+      "grad_norm": 0.7885979413986206,
+      "learning_rate": 7.182325903146721e-05,
+      "loss": 0.6168,
+      "mean_token_accuracy": 0.8091749155521393,
+      "num_tokens": 77090179.0,
+      "step": 3500
+    },
+    {
+      "epoch": 2.543901108889293,
+      "eval_loss": 0.614472508430481,
+      "eval_mean_token_accuracy": 0.8088948371168835,
+      "eval_num_tokens": 77090179.0,
+      "eval_runtime": 112.1372,
+      "eval_samples_per_second": 43.607,
+      "eval_steps_per_second": 5.458,
+      "step": 3500
     }
   ],
   "logging_steps": 25,
@@ -1236,7 +1436,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.6655712651942298e+17,
+  "total_flos": 1.9432793569440563e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null