Training in progress, step 10500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd86a548e36d159ece49e920c99cc6f2a4355147d1037cfe13009c71f586f637
+oid sha256:6a937226536c59143efd81bff0e0a4590cf5c5fdd055ca661ac115dd2f29aeb4
 size 12017472

last-checkpoint/global_step10500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3bfd20710412dec759281428025cd6a7ca3b4586b49afd63f0130bc3a42f73e
+size 71982309

last-checkpoint/global_step10500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a982dea63ab64518dc66c41523d6333f4016ba19fbf6f50ec212a905606961
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step10000
+global_step10500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d08b39a90876d1c53caff700b09ec23dfbb515477f4d843fcec122065a6e076e
+oid sha256:175332e303b3de70fe9a28dccd1b78ad14e4b9e5741fcec3568fbb0aef10ae34
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 10000,
-  "best_metric": 0.5502750873565674,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-10000",
-  "epoch": 7.267587711325214,
+  "best_global_step": 10500,
+  "best_metric": 0.5472510457038879,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-10500",
+  "epoch": 7.63115797127795,
   "eval_steps": 250,
-  "global_step": 10000,
+  "global_step": 10500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4017,6 +4017,206 @@
       "eval_samples_per_second": 42.893,
       "eval_steps_per_second": 5.368,
       "step": 10000
+    },
+    {
+      "epoch": 7.285766224322851,
+      "grad_norm": 0.7890622615814209,
+      "learning_rate": 2.70741670332471e-05,
+      "loss": 0.5316,
+      "mean_token_accuracy": 0.8335822412371635,
+      "num_tokens": 220804591.0,
+      "step": 10025
+    },
+    {
+      "epoch": 7.303944737320487,
+      "grad_norm": 0.8019230365753174,
+      "learning_rate": 2.689316513278237e-05,
+      "loss": 0.5205,
+      "mean_token_accuracy": 0.8377341836690902,
+      "num_tokens": 221351584.0,
+      "step": 10050
+    },
+    {
+      "epoch": 7.322123250318124,
+      "grad_norm": 0.7473861575126648,
+      "learning_rate": 2.671246339295931e-05,
+      "loss": 0.5311,
+      "mean_token_accuracy": 0.8348194608092308,
+      "num_tokens": 221924723.0,
+      "step": 10075
+    },
+    {
+      "epoch": 7.340301763315761,
+      "grad_norm": 0.8123943209648132,
+      "learning_rate": 2.6532065952042465e-05,
+      "loss": 0.5284,
+      "mean_token_accuracy": 0.8367439916729927,
+      "num_tokens": 222457441.0,
+      "step": 10100
+    },
+    {
+      "epoch": 7.358480276313397,
+      "grad_norm": 0.8315654993057251,
+      "learning_rate": 2.635197694132764e-05,
+      "loss": 0.5245,
+      "mean_token_accuracy": 0.836669889986515,
+      "num_tokens": 223015673.0,
+      "step": 10125
+    },
+    {
+      "epoch": 7.376658789311034,
+      "grad_norm": 0.7933287024497986,
+      "learning_rate": 2.6172200485047237e-05,
+      "loss": 0.5335,
+      "mean_token_accuracy": 0.8333874663710594,
+      "num_tokens": 223580969.0,
+      "step": 10150
+    },
+    {
+      "epoch": 7.394837302308671,
+      "grad_norm": 0.8061972856521606,
+      "learning_rate": 2.5992740700275807e-05,
+      "loss": 0.5247,
+      "mean_token_accuracy": 0.8361442899703979,
+      "num_tokens": 224127599.0,
+      "step": 10175
+    },
+    {
+      "epoch": 7.4130158153063075,
+      "grad_norm": 0.819487452507019,
+      "learning_rate": 2.5813601696835804e-05,
+      "loss": 0.5262,
+      "mean_token_accuracy": 0.8357849648594856,
+      "num_tokens": 224665789.0,
+      "step": 10200
+    },
+    {
+      "epoch": 7.4311943283039446,
+      "grad_norm": 0.8185529708862305,
+      "learning_rate": 2.5634787577203428e-05,
+      "loss": 0.5323,
+      "mean_token_accuracy": 0.8345727735757827,
+      "num_tokens": 225228917.0,
+      "step": 10225
+    },
+    {
+      "epoch": 7.449372841301582,
+      "grad_norm": 0.8035647869110107,
+      "learning_rate": 2.545630243641471e-05,
+      "loss": 0.5265,
+      "mean_token_accuracy": 0.8369637629389763,
+      "num_tokens": 225761918.0,
+      "step": 10250
+    },
+    {
+      "epoch": 7.449372841301582,
+      "eval_loss": 0.5480418801307678,
+      "eval_mean_token_accuracy": 0.8291744561561571,
+      "eval_num_tokens": 225761918.0,
+      "eval_runtime": 113.3094,
+      "eval_samples_per_second": 43.156,
+      "eval_steps_per_second": 5.401,
+      "step": 10250
+    },
+    {
+      "epoch": 7.467551354299219,
+      "grad_norm": 0.8327836394309998,
+      "learning_rate": 2.5278150361971698e-05,
+      "loss": 0.5349,
+      "mean_token_accuracy": 0.8348187085986137,
+      "num_tokens": 226303606.0,
+      "step": 10275
+    },
+    {
+      "epoch": 7.485729867296855,
+      "grad_norm": 0.8224396705627441,
+      "learning_rate": 2.510033543374886e-05,
+      "loss": 0.5292,
+      "mean_token_accuracy": 0.8360765823721885,
+      "num_tokens": 226841143.0,
+      "step": 10300
+    },
+    {
+      "epoch": 7.503908380294492,
+      "grad_norm": 0.8393549919128418,
+      "learning_rate": 2.4922861723899657e-05,
+      "loss": 0.5293,
+      "mean_token_accuracy": 0.835459913611412,
+      "num_tokens": 227388854.0,
+      "step": 10325
+    },
+    {
+      "epoch": 7.522086893292129,
+      "grad_norm": 0.8260053396224976,
+      "learning_rate": 2.4745733296763285e-05,
+      "loss": 0.5338,
+      "mean_token_accuracy": 0.8342707616090774,
+      "num_tokens": 227932125.0,
+      "step": 10350
+    },
+    {
+      "epoch": 7.540265406289765,
+      "grad_norm": 0.8399791121482849,
+      "learning_rate": 2.4568954208771584e-05,
+      "loss": 0.5174,
+      "mean_token_accuracy": 0.8391924080252647,
+      "num_tokens": 228480967.0,
+      "step": 10375
+    },
+    {
+      "epoch": 7.558443919287402,
+      "grad_norm": 0.8734251260757446,
+      "learning_rate": 2.439252850835615e-05,
+      "loss": 0.5215,
+      "mean_token_accuracy": 0.8371177944540977,
+      "num_tokens": 229031882.0,
+      "step": 10400
+    },
+    {
+      "epoch": 7.576622432285039,
+      "grad_norm": 0.8437944650650024,
+      "learning_rate": 2.4216460235855648e-05,
+      "loss": 0.5311,
+      "mean_token_accuracy": 0.8349981158971786,
+      "num_tokens": 229590045.0,
+      "step": 10425
+    },
+    {
+      "epoch": 7.594800945282676,
+      "grad_norm": 0.832741379737854,
+      "learning_rate": 2.404075342342323e-05,
+      "loss": 0.5316,
+      "mean_token_accuracy": 0.8340688073635101,
+      "num_tokens": 230132661.0,
+      "step": 10450
+    },
+    {
+      "epoch": 7.6129794582803125,
+      "grad_norm": 0.8090599179267883,
+      "learning_rate": 2.386541209493423e-05,
+      "loss": 0.5351,
+      "mean_token_accuracy": 0.8336004850268364,
+      "num_tokens": 230679191.0,
+      "step": 10475
+    },
+    {
+      "epoch": 7.63115797127795,
+      "grad_norm": 0.829035758972168,
+      "learning_rate": 2.369044026589401e-05,
+      "loss": 0.5253,
+      "mean_token_accuracy": 0.8360374090075493,
+      "num_tokens": 231245565.0,
+      "step": 10500
+    },
+    {
+      "epoch": 7.63115797127795,
+      "eval_loss": 0.5472510457038879,
+      "eval_mean_token_accuracy": 0.8294721552554298,
+      "eval_num_tokens": 231245565.0,
+      "eval_runtime": 113.6665,
+      "eval_samples_per_second": 43.021,
+      "eval_steps_per_second": 5.384,
+      "step": 10500
     }
   ],
   "logging_steps": 25,
@@ -4036,7 +4236,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 5.5542499168904806e+17,
+  "total_flos": 5.831638561052426e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null