Training in progress, step 6500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18b75b4abf894259d125060201a5ccf51810364d91fd3cea0d60d17a6403b9f6
+oid sha256:b37eb9bcf96214de200298cdc8128de031674bd22404378d2eb8711f7a1f0639
 size 12017472

last-checkpoint/global_step6500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de937880dd5d112efb485014c98ed5da078bb734ff16262f3b7d295b12c4511a
+size 71982309

last-checkpoint/global_step6500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a6f69d58549c317558501ef11c494883a7fbe070adb7912f9014823b8c0cc54
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step6000
+global_step6500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8941841568d79a9d2dc41b93071be9e47935d7e9eda669b8a7caaabc7faae599
+oid sha256:5fba9e07381b5475da97b7a36fb2ebd7e3dbf5fc84575cf7978cdcac8ad340c8
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 6000,
-  "best_metric": 0.578772783279419,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-6000",
-  "epoch": 4.360661697873114,
+  "best_global_step": 6500,
+  "best_metric": 0.5727137923240662,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-6500",
+  "epoch": 4.72423195782585,
   "eval_steps": 250,
-  "global_step": 6000,
+  "global_step": 6500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2417,6 +2417,206 @@
       "eval_samples_per_second": 43.383,
       "eval_steps_per_second": 5.43,
       "step": 6000
+    },
+    {
+      "epoch": 4.3788402108707505,
+      "grad_norm": 0.7740823030471802,
+      "learning_rate": 5.6915524395595964e-05,
+      "loss": 0.5768,
+      "mean_token_accuracy": 0.820943498313427,
+      "num_tokens": 132643717.0,
+      "step": 6025
+    },
+    {
+      "epoch": 4.3970187238683875,
+      "grad_norm": 0.7765363454818726,
+      "learning_rate": 5.674186960500381e-05,
+      "loss": 0.5673,
+      "mean_token_accuracy": 0.8246706813573838,
+      "num_tokens": 133190852.0,
+      "step": 6050
+    },
+    {
+      "epoch": 4.415197236866025,
+      "grad_norm": 0.7921891808509827,
+      "learning_rate": 5.656783140755755e-05,
+      "loss": 0.5745,
+      "mean_token_accuracy": 0.8214689978957176,
+      "num_tokens": 133742448.0,
+      "step": 6075
+    },
+    {
+      "epoch": 4.433375749863661,
+      "grad_norm": 0.7477549910545349,
+      "learning_rate": 5.639341378891948e-05,
+      "loss": 0.5649,
+      "mean_token_accuracy": 0.8251917722821236,
+      "num_tokens": 134278075.0,
+      "step": 6100
+    },
+    {
+      "epoch": 4.451554262861298,
+      "grad_norm": 0.8201608657836914,
+      "learning_rate": 5.6218620743440986e-05,
+      "loss": 0.5688,
+      "mean_token_accuracy": 0.8233516532182693,
+      "num_tokens": 134830841.0,
+      "step": 6125
+    },
+    {
+      "epoch": 4.469732775858935,
+      "grad_norm": 0.7502488493919373,
+      "learning_rate": 5.6043456274071164e-05,
+      "loss": 0.5667,
+      "mean_token_accuracy": 0.824826996922493,
+      "num_tokens": 135383251.0,
+      "step": 6150
+    },
+    {
+      "epoch": 4.487911288856571,
+      "grad_norm": 0.7607132196426392,
+      "learning_rate": 5.586792439226512e-05,
+      "loss": 0.5634,
+      "mean_token_accuracy": 0.8251435077190399,
+      "num_tokens": 135935805.0,
+      "step": 6175
+    },
+    {
+      "epoch": 4.506089801854208,
+      "grad_norm": 0.7952844500541687,
+      "learning_rate": 5.569202911789206e-05,
+      "loss": 0.566,
+      "mean_token_accuracy": 0.8234961378574371,
+      "num_tokens": 136496942.0,
+      "step": 6200
+    },
+    {
+      "epoch": 4.524268314851845,
+      "grad_norm": 0.7817985415458679,
+      "learning_rate": 5.551577447914333e-05,
+      "loss": 0.5724,
+      "mean_token_accuracy": 0.8220775499939919,
+      "num_tokens": 137049699.0,
+      "step": 6225
+    },
+    {
+      "epoch": 4.542446827849482,
+      "grad_norm": 0.8044058680534363,
+      "learning_rate": 5.533916451244008e-05,
+      "loss": 0.5707,
+      "mean_token_accuracy": 0.8230817872285843,
+      "num_tokens": 137612608.0,
+      "step": 6250
+    },
+    {
+      "epoch": 4.542446827849482,
+      "eval_loss": 0.5755720734596252,
+      "eval_mean_token_accuracy": 0.8204464651400747,
+      "eval_num_tokens": 137612608.0,
+      "eval_runtime": 114.4855,
+      "eval_samples_per_second": 42.713,
+      "eval_steps_per_second": 5.346,
+      "step": 6250
+    },
+    {
+      "epoch": 4.5606253408471185,
+      "grad_norm": 0.7777162194252014,
+      "learning_rate": 5.516220326234084e-05,
+      "loss": 0.5557,
+      "mean_token_accuracy": 0.8273163467645646,
+      "num_tokens": 138155250.0,
+      "step": 6275
+    },
+    {
+      "epoch": 4.5788038538447555,
+      "grad_norm": 0.7649146318435669,
+      "learning_rate": 5.4984894781448934e-05,
+      "loss": 0.5647,
+      "mean_token_accuracy": 0.8240767633914947,
+      "num_tokens": 138700597.0,
+      "step": 6300
+    },
+    {
+      "epoch": 4.596982366842393,
+      "grad_norm": 0.7625702023506165,
+      "learning_rate": 5.480724313031962e-05,
+      "loss": 0.5638,
+      "mean_token_accuracy": 0.8254871436953545,
+      "num_tokens": 139240284.0,
+      "step": 6325
+    },
+    {
+      "epoch": 4.615160879840029,
+      "grad_norm": 0.7333512306213379,
+      "learning_rate": 5.462925237736715e-05,
+      "loss": 0.5753,
+      "mean_token_accuracy": 0.8208840799331665,
+      "num_tokens": 139806757.0,
+      "step": 6350
+    },
+    {
+      "epoch": 4.633339392837666,
+      "grad_norm": 0.8279890418052673,
+      "learning_rate": 5.445092659877153e-05,
+      "loss": 0.5663,
+      "mean_token_accuracy": 0.8239589306712151,
+      "num_tokens": 140363358.0,
+      "step": 6375
+    },
+    {
+      "epoch": 4.651517905835303,
+      "grad_norm": 0.7528174519538879,
+      "learning_rate": 5.427226987838528e-05,
+      "loss": 0.5668,
+      "mean_token_accuracy": 0.8237173920869827,
+      "num_tokens": 140926439.0,
+      "step": 6400
+    },
+    {
+      "epoch": 4.66969641883294,
+      "grad_norm": 0.7415145039558411,
+      "learning_rate": 5.4093286307639786e-05,
+      "loss": 0.559,
+      "mean_token_accuracy": 0.8262969407439232,
+      "num_tokens": 141476080.0,
+      "step": 6425
+    },
+    {
+      "epoch": 4.687874931830576,
+      "grad_norm": 0.7791749238967896,
+      "learning_rate": 5.39139799854517e-05,
+      "loss": 0.5695,
+      "mean_token_accuracy": 0.8218407195806503,
+      "num_tokens": 142035408.0,
+      "step": 6450
+    },
+    {
+      "epoch": 4.706053444828213,
+      "grad_norm": 0.8288747668266296,
+      "learning_rate": 5.3734355018128974e-05,
+      "loss": 0.5658,
+      "mean_token_accuracy": 0.8245827797055244,
+      "num_tokens": 142577709.0,
+      "step": 6475
+    },
+    {
+      "epoch": 4.72423195782585,
+      "grad_norm": 0.7962896227836609,
+      "learning_rate": 5.355441551927695e-05,
+      "loss": 0.5666,
+      "mean_token_accuracy": 0.8251102951169014,
+      "num_tokens": 143123639.0,
+      "step": 6500
+    },
+    {
+      "epoch": 4.72423195782585,
+      "eval_loss": 0.5727137923240662,
+      "eval_mean_token_accuracy": 0.8213643729686737,
+      "eval_num_tokens": 143123639.0,
+      "eval_runtime": 113.5915,
+      "eval_samples_per_second": 43.049,
+      "eval_steps_per_second": 5.388,
+      "step": 6500
     }
   ],
   "logging_steps": 25,
@@ -2436,7 +2636,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.3318287854639514e+17,
+  "total_flos": 3.610250824258683e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null