Training in progress, step 3000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48c8ccaea45630ddf9ea1b88f0156cd1fedd61905cf18898819db2276984fcb7
+oid sha256:e63c3db29c1618638ad881936caf0567d6bf237c2d8358081d7324f323992b88
 size 150625560

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7b50e0d348776f6bfde587ad7885adcff251381a86d03c221fe493591258125
+oid sha256:7c5d06df021b0676b1a33468e46c419e946ec39b40bd40d3bc089bc612bb03c6
 size 602335994

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddd99e2663b559b91167c1d3714d334dd46c05cbb133b69d7b52f7b883548de2
+oid sha256:841a17a497b009366c5a93d5dd6be560b5debec016825b3dd528f17c802bd947
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7286e34301379ab62676eaae8fceab28248cb123c9029e5c4370161a2e81133d
+oid sha256:5361224ea55f23c97a6ce6c66443aa545b9c0f839415b7a0921d0d516508616d
 size 1064

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.4828220951461291,
+  "epoch": 0.7242331427191937,
   "eval_steps": 500,
-  "global_step": 2000,
+  "global_step": 3000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -444,11 +444,229 @@
       "eval_steps_per_second": 21.193,
       "num_input_tokens_seen": 965894272,
       "step": 2000
+    },
+    {
+      "epoch": 0.49489264752478235,
+      "grad_norm": 0.55859375,
+      "learning_rate": 3.092939046469523e-05,
+      "loss": 2.3729,
+      "mean_token_accuracy": 0.5244628588855267,
+      "num_input_tokens_seen": 990100416,
+      "num_tokens": 417281294.0,
+      "step": 2050
+    },
+    {
+      "epoch": 0.5069631999034356,
+      "grad_norm": 0.46484375,
+      "learning_rate": 3.168376584188292e-05,
+      "loss": 2.365,
+      "mean_token_accuracy": 0.5261798568814993,
+      "num_input_tokens_seen": 1014101792,
+      "num_tokens": 427442066.0,
+      "step": 2100
+    },
+    {
+      "epoch": 0.5190337522820888,
+      "grad_norm": 0.4296875,
+      "learning_rate": 3.2438141219070614e-05,
+      "loss": 2.3679,
+      "mean_token_accuracy": 0.5253909171745181,
+      "num_input_tokens_seen": 1038463120,
+      "num_tokens": 437736421.0,
+      "step": 2150
+    },
+    {
+      "epoch": 0.531104304660742,
+      "grad_norm": 0.404296875,
+      "learning_rate": 3.31925165962583e-05,
+      "loss": 2.3664,
+      "mean_token_accuracy": 0.5255175845324993,
+      "num_input_tokens_seen": 1062548608,
+      "num_tokens": 447839969.0,
+      "step": 2200
+    },
+    {
+      "epoch": 0.5431748570393953,
+      "grad_norm": 0.376953125,
+      "learning_rate": 3.394689197344599e-05,
+      "loss": 2.3566,
+      "mean_token_accuracy": 0.5265485693141818,
+      "num_input_tokens_seen": 1086859296,
+      "num_tokens": 458069337.0,
+      "step": 2250
+    },
+    {
+      "epoch": 0.5552454094180485,
+      "grad_norm": 0.376953125,
+      "learning_rate": 3.4701267350633675e-05,
+      "loss": 2.3424,
+      "mean_token_accuracy": 0.529197344481945,
+      "num_input_tokens_seen": 1110954480,
+      "num_tokens": 468219132.0,
+      "step": 2300
+    },
+    {
+      "epoch": 0.5673159617967017,
+      "grad_norm": 0.376953125,
+      "learning_rate": 3.545564272782136e-05,
+      "loss": 2.3344,
+      "mean_token_accuracy": 0.5297643894702196,
+      "num_input_tokens_seen": 1135145632,
+      "num_tokens": 478383651.0,
+      "step": 2350
+    },
+    {
+      "epoch": 0.5793865141753549,
+      "grad_norm": 0.3984375,
+      "learning_rate": 3.621001810500905e-05,
+      "loss": 2.3403,
+      "mean_token_accuracy": 0.5286615265905857,
+      "num_input_tokens_seen": 1159223072,
+      "num_tokens": 488558846.0,
+      "step": 2400
+    },
+    {
+      "epoch": 0.5914570665540082,
+      "grad_norm": 0.38671875,
+      "learning_rate": 3.696439348219674e-05,
+      "loss": 2.3366,
+      "mean_token_accuracy": 0.5282911998406052,
+      "num_input_tokens_seen": 1183346592,
+      "num_tokens": 498693280.0,
+      "step": 2450
+    },
+    {
+      "epoch": 0.6035276189326614,
+      "grad_norm": 0.36328125,
+      "learning_rate": 3.771876885938443e-05,
+      "loss": 2.3361,
+      "num_input_tokens_seen": 1207430512,
+      "step": 2500
+    },
+    {
+      "epoch": 0.6035276189326614,
+      "eval_loss": 2.2252955436706543,
+      "eval_mean_token_accuracy": 0.5513953688186527,
+      "eval_num_tokens": 508899754.0,
+      "eval_runtime": 125.696,
+      "eval_samples_per_second": 85.221,
+      "eval_steps_per_second": 21.305,
+      "num_input_tokens_seen": 1207430512,
+      "step": 2500
+    },
+    {
+      "epoch": 0.6155981713113147,
+      "grad_norm": 0.33984375,
+      "learning_rate": 3.8473144236572124e-05,
+      "loss": 2.3191,
+      "mean_token_accuracy": 0.5302580918744206,
+      "num_input_tokens_seen": 1231510064,
+      "num_tokens": 519053973.0,
+      "step": 2550
+    },
+    {
+      "epoch": 0.6276687236899678,
+      "grad_norm": 0.40234375,
+      "learning_rate": 3.922751961375981e-05,
+      "loss": 2.3184,
+      "mean_token_accuracy": 0.5312636430934071,
+      "num_input_tokens_seen": 1255661376,
+      "num_tokens": 529292647.0,
+      "step": 2600
+    },
+    {
+      "epoch": 0.6397392760686211,
+      "grad_norm": 0.361328125,
+      "learning_rate": 3.99818949909475e-05,
+      "loss": 2.3149,
+      "mean_token_accuracy": 0.5317899576947093,
+      "num_input_tokens_seen": 1279915488,
+      "num_tokens": 539406430.0,
+      "step": 2650
+    },
+    {
+      "epoch": 0.6518098284472743,
+      "grad_norm": 0.37890625,
+      "learning_rate": 4.0736270368135186e-05,
+      "loss": 2.3226,
+      "mean_token_accuracy": 0.530639638863504,
+      "num_input_tokens_seen": 1303974000,
+      "num_tokens": 549560793.0,
+      "step": 2700
+    },
+    {
+      "epoch": 0.6638803808259276,
+      "grad_norm": 0.3046875,
+      "learning_rate": 4.149064574532287e-05,
+      "loss": 2.3027,
+      "mean_token_accuracy": 0.5339162700995803,
+      "num_input_tokens_seen": 1328099056,
+      "num_tokens": 559731643.0,
+      "step": 2750
+    },
+    {
+      "epoch": 0.6759509332045808,
+      "grad_norm": 0.353515625,
+      "learning_rate": 4.224502112251057e-05,
+      "loss": 2.3041,
+      "mean_token_accuracy": 0.5334529640898108,
+      "num_input_tokens_seen": 1352139632,
+      "num_tokens": 569834852.0,
+      "step": 2800
+    },
+    {
+      "epoch": 0.688021485583234,
+      "grad_norm": 0.333984375,
+      "learning_rate": 4.2999396499698254e-05,
+      "loss": 2.2978,
+      "mean_token_accuracy": 0.5343039923906326,
+      "num_input_tokens_seen": 1376232240,
+      "num_tokens": 579992251.0,
+      "step": 2850
+    },
+    {
+      "epoch": 0.7000920379618872,
+      "grad_norm": 0.330078125,
+      "learning_rate": 4.375377187688594e-05,
+      "loss": 2.2834,
+      "mean_token_accuracy": 0.5361328301951289,
+      "num_input_tokens_seen": 1400431104,
+      "num_tokens": 590199795.0,
+      "step": 2900
+    },
+    {
+      "epoch": 0.7121625903405404,
+      "grad_norm": 0.337890625,
+      "learning_rate": 4.450814725407363e-05,
+      "loss": 2.301,
+      "mean_token_accuracy": 0.5332023718208074,
+      "num_input_tokens_seen": 1424615616,
+      "num_tokens": 600407039.0,
+      "step": 2950
+    },
+    {
+      "epoch": 0.7242331427191937,
+      "grad_norm": 0.291015625,
+      "learning_rate": 4.5262522631261315e-05,
+      "loss": 2.2846,
+      "num_input_tokens_seen": 1448897616,
+      "step": 3000
+    },
+    {
+      "epoch": 0.7242331427191937,
+      "eval_loss": 2.174436569213867,
+      "eval_mean_token_accuracy": 0.5580886014145288,
+      "eval_num_tokens": 610496575.0,
+      "eval_runtime": 125.6585,
+      "eval_samples_per_second": 85.247,
+      "eval_steps_per_second": 21.312,
+      "num_input_tokens_seen": 1448897616,
+      "step": 3000
     }
   ],
   "logging_steps": 50,
   "max_steps": 16568,
-  "num_input_tokens_seen": 965894272,
+  "num_input_tokens_seen": 1448897616,
   "num_train_epochs": 4,
   "save_steps": 1000,
   "stateful_callbacks": {
@@ -463,7 +681,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.5838614488809472e+17,
+  "total_flos": 3.8759425352073216e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null