Training in progress, step 1500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a4ad0fda5a25c3ff587c8fc3613dad924a91714c4d11cefa4533c354410f3d2
+oid sha256:2e3e9ee8a611e7a57dccff25563a008747ed15810194baa91980ef853c11a0a7
 size 12017472

last-checkpoint/global_step1500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:390228104a24054ada0fb3d185b75e1ee48590db75d7869429819a9c9ab05d18
+size 71982309

last-checkpoint/global_step1500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efc5ccbefb5e21781792c66a4a01865f2af9fb2a9db01cede684299d60e76df6
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step1000
+global_step1500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:162ea028bad2d13be08d5ca87f053c3d58c23e084a4fa2f7cf1e9194c8781bcb
+oid sha256:27afcafd6ed5692d8873208ba0cf57e46a0701e5eb0aa08cd9750d1e2b88cb5d
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 1000,
-  "best_metric": 0.7203578948974609,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-1000",
-  "epoch": 0.7271405199054717,
+  "best_global_step": 1500,
+  "best_metric": 0.6834071278572083,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-1500",
+  "epoch": 1.0901654244682786,
   "eval_steps": 250,
-  "global_step": 1000,
+  "global_step": 1500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -417,6 +417,206 @@
       "eval_samples_per_second": 43.127,
       "eval_steps_per_second": 5.397,
       "step": 1000
+    },
+    {
+      "epoch": 0.7453190329031085,
+      "grad_norm": 1.012831449508667,
+      "learning_rate": 7.937460379745742e-05,
+      "loss": 0.7165,
+      "mean_token_accuracy": 0.781532918214798,
+      "num_tokens": 22623696.0,
+      "step": 1025
+    },
+    {
+      "epoch": 0.7634975459007454,
+      "grad_norm": 1.1071596145629883,
+      "learning_rate": 7.934043623246033e-05,
+      "loss": 0.7161,
+      "mean_token_accuracy": 0.7818375152349472,
+      "num_tokens": 23152552.0,
+      "step": 1050
+    },
+    {
+      "epoch": 0.7816760588983821,
+      "grad_norm": 0.9036078453063965,
+      "learning_rate": 7.930536772908241e-05,
+      "loss": 0.7191,
+      "mean_token_accuracy": 0.7791232514381409,
+      "num_tokens": 23710635.0,
+      "step": 1075
+    },
+    {
+      "epoch": 0.7998545718960189,
+      "grad_norm": 0.9387621283531189,
+      "learning_rate": 7.926939909043018e-05,
+      "loss": 0.7201,
+      "mean_token_accuracy": 0.7794615784287453,
+      "num_tokens": 24277083.0,
+      "step": 1100
+    },
+    {
+      "epoch": 0.8180330848936557,
+      "grad_norm": 0.9713501930236816,
+      "learning_rate": 7.92325311402242e-05,
+      "loss": 0.7103,
+      "mean_token_accuracy": 0.7832019272446632,
+      "num_tokens": 24817217.0,
+      "step": 1125
+    },
+    {
+      "epoch": 0.8362115978912925,
+      "grad_norm": 0.9385744333267212,
+      "learning_rate": 7.919476472278026e-05,
+      "loss": 0.7121,
+      "mean_token_accuracy": 0.782238809466362,
+      "num_tokens": 25368089.0,
+      "step": 1150
+    },
+    {
+      "epoch": 0.8543901108889292,
+      "grad_norm": 0.9703376293182373,
+      "learning_rate": 7.915610070299001e-05,
+      "loss": 0.7073,
+      "mean_token_accuracy": 0.7827208504080773,
+      "num_tokens": 25912613.0,
+      "step": 1175
+    },
+    {
+      "epoch": 0.8725686238865661,
+      "grad_norm": 1.0707026720046997,
+      "learning_rate": 7.911653996630121e-05,
+      "loss": 0.7048,
+      "mean_token_accuracy": 0.7837258630990982,
+      "num_tokens": 26466244.0,
+      "step": 1200
+    },
+    {
+      "epoch": 0.8907471368842028,
+      "grad_norm": 1.0107282400131226,
+      "learning_rate": 7.90760834186973e-05,
+      "loss": 0.6945,
+      "mean_token_accuracy": 0.7874345901608467,
+      "num_tokens": 27009212.0,
+      "step": 1225
+    },
+    {
+      "epoch": 0.9089256498818397,
+      "grad_norm": 0.9038238525390625,
+      "learning_rate": 7.903473198667684e-05,
+      "loss": 0.7011,
+      "mean_token_accuracy": 0.7847383853793144,
+      "num_tokens": 27551777.0,
+      "step": 1250
+    },
+    {
+      "epoch": 0.9089256498818397,
+      "eval_loss": 0.6989394426345825,
+      "eval_mean_token_accuracy": 0.7840953680619694,
+      "eval_num_tokens": 27551777.0,
+      "eval_runtime": 113.5783,
+      "eval_samples_per_second": 43.054,
+      "eval_steps_per_second": 5.388,
+      "step": 1250
+    },
+    {
+      "epoch": 0.9271041628794765,
+      "grad_norm": 1.009592890739441,
+      "learning_rate": 7.899248661723218e-05,
+      "loss": 0.702,
+      "mean_token_accuracy": 0.7847409224510193,
+      "num_tokens": 28112347.0,
+      "step": 1275
+    },
+    {
+      "epoch": 0.9452826758771132,
+      "grad_norm": 1.1236053705215454,
+      "learning_rate": 7.894934827782781e-05,
+      "loss": 0.6992,
+      "mean_token_accuracy": 0.7860203450918197,
+      "num_tokens": 28650826.0,
+      "step": 1300
+    },
+    {
+      "epoch": 0.9634611888747501,
+      "grad_norm": 1.0439739227294922,
+      "learning_rate": 7.890531795637816e-05,
+      "loss": 0.6937,
+      "mean_token_accuracy": 0.7889308288693428,
+      "num_tokens": 29182993.0,
+      "step": 1325
+    },
+    {
+      "epoch": 0.9816397018723868,
+      "grad_norm": 1.2452131509780884,
+      "learning_rate": 7.886039666122508e-05,
+      "loss": 0.6941,
+      "mean_token_accuracy": 0.7871252146363258,
+      "num_tokens": 29740267.0,
+      "step": 1350
+    },
+    {
+      "epoch": 0.9998182148700236,
+      "grad_norm": 1.0504438877105713,
+      "learning_rate": 7.881458542111457e-05,
+      "loss": 0.6888,
+      "mean_token_accuracy": 0.7882778728008271,
+      "num_tokens": 30296922.0,
+      "step": 1375
+    },
+    {
+      "epoch": 1.0174513724777314,
+      "grad_norm": 0.949993908405304,
+      "learning_rate": 7.876788528517346e-05,
+      "loss": 0.6937,
+      "mean_token_accuracy": 0.7869942234349006,
+      "num_tokens": 30828543.0,
+      "step": 1400
+    },
+    {
+      "epoch": 1.035629885475368,
+      "grad_norm": 0.9186608195304871,
+      "learning_rate": 7.872029732288515e-05,
+      "loss": 0.6912,
+      "mean_token_accuracy": 0.7877016308903694,
+      "num_tokens": 31382166.0,
+      "step": 1425
+    },
+    {
+      "epoch": 1.053808398473005,
+      "grad_norm": 0.9520437121391296,
+      "learning_rate": 7.867182262406524e-05,
+      "loss": 0.6848,
+      "mean_token_accuracy": 0.789347026348114,
+      "num_tokens": 31936482.0,
+      "step": 1450
+    },
+    {
+      "epoch": 1.0719869114706417,
+      "grad_norm": 0.9007825255393982,
+      "learning_rate": 7.86224622988366e-05,
+      "loss": 0.6915,
+      "mean_token_accuracy": 0.7874828764796257,
+      "num_tokens": 32491054.0,
+      "step": 1475
+    },
+    {
+      "epoch": 1.0901654244682786,
+      "grad_norm": 0.9098522067070007,
+      "learning_rate": 7.857221747760384e-05,
+      "loss": 0.6791,
+      "mean_token_accuracy": 0.7907387048006058,
+      "num_tokens": 33049696.0,
+      "step": 1500
+    },
+    {
+      "epoch": 1.0901654244682786,
+      "eval_loss": 0.6834071278572083,
+      "eval_mean_token_accuracy": 0.788765495509104,
+      "eval_num_tokens": 33049696.0,
+      "eval_runtime": 113.3067,
+      "eval_samples_per_second": 43.157,
+      "eval_steps_per_second": 5.401,
+      "step": 1500
     }
   ],
   "logging_steps": 25,
@@ -436,7 +636,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 5.564104967821722e+16,
+  "total_flos": 8.33876308084654e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null