Upload folder using huggingface_hub

adapter_config.json

@@ -23,9 +23,9 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "qkv_proj",
-    "gate_up_proj",
     "o_proj",
+    "gate_up_proj",
+    "qkv_proj",
     "down_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05168bcf41d42782e54f2110629a07f6dfe491410f4a0c12e5c2ab3d90836271
+oid sha256:3c6fe0fc0a353acf52d13e5f2926d7ca2a170a6650f4c15060954cca6f1d4bef
 size 161515608

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd295a28962bd0b73c87cd25e49b90c33ee4984b4d89fb83894543530a2dc2f7
+oid sha256:3cac0002e7c7b8c2855bd74f0f11746b0f2de66ac0a81a0709e1713f3ce142e2
 size 323181259

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5b517d1b8e2b0f837c8b00170b154961d4d989feba4326ac25583df7a55c57a
+oid sha256:2534e434cd5abbb8f7668d3eab0549db0ef95d6a797a3efa86b712e8e32266a7
 size 14645

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ef3bc6f4d35776f47009c7dd438a2c3372feb65dbb821c4678645626f6d1776
+oid sha256:476926ce2c69ab4ccb2e05f1160789ff4fbbecc78d81f0f191944d21b8ccd89e
 size 1465

trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.245210727969349,
+  "epoch": 2.888888888888889,
   "eval_steps": 500,
-  "global_step": 140,
+  "global_step": 95,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -12,7 +12,7 @@
     {
       "entropy": 2.2323372662067413,
       "epoch": 0.03065134099616858,
-      "grad_norm": 23.125,
+      "grad_norm": 24.0,
       "learning_rate": 0.0,
       "loss": 2.7706,
       "mean_token_accuracy": 0.41634324193000793,
@@ -22,7 +22,7 @@
     {
       "entropy": 2.174584299325943,
       "epoch": 0.06130268199233716,
-      "grad_norm": 15.625,
+      "grad_norm": 16.125,
       "learning_rate": 2e-06,
       "loss": 2.4332,
       "mean_token_accuracy": 0.41893551871180534,
@@ -30,1390 +30,940 @@
       "step": 2
     },
     {
-      "entropy": 2.026109978556633,
+      "entropy": 2.026469185948372,
       "epoch": 0.09195402298850575,
-      "grad_norm": 13.625,
+      "grad_norm": 14.0625,
       "learning_rate": 4e-06,
-      "loss": 2.334,
-      "mean_token_accuracy": 0.4435151480138302,
+      "loss": 2.335,
+      "mean_token_accuracy": 0.4408150501549244,
       "num_tokens": 5582.0,
       "step": 3
     },
     {
-      "entropy": 2.292216345667839,
+      "entropy": 2.294757843017578,
       "epoch": 0.12260536398467432,
-      "grad_norm": 17.125,
+      "grad_norm": 17.625,
       "learning_rate": 6e-06,
-      "loss": 2.4632,
-      "mean_token_accuracy": 0.4366639256477356,
+      "loss": 2.458,
+      "mean_token_accuracy": 0.43590471893548965,
       "num_tokens": 7064.0,
       "step": 4
     },
     {
-      "entropy": 2.308288350701332,
+      "entropy": 2.3134160190820694,
       "epoch": 0.1532567049808429,
-      "grad_norm": 11.5625,
+      "grad_norm": 11.9375,
       "learning_rate": 8e-06,
-      "loss": 2.196,
-      "mean_token_accuracy": 0.43852633982896805,
+      "loss": 2.1904,
+      "mean_token_accuracy": 0.43868962302803993,
       "num_tokens": 9091.0,
       "step": 5
     },
     {
-      "entropy": 2.431372880935669,
+      "entropy": 2.4343303740024567,
       "epoch": 0.1839080459770115,
-      "grad_norm": 12.5625,
+      "grad_norm": 12.9375,
       "learning_rate": 9.999999999999999e-06,
-      "loss": 2.4038,
-      "mean_token_accuracy": 0.47505030781030655,
+      "loss": 2.3987,
+      "mean_token_accuracy": 0.47084228694438934,
       "num_tokens": 10556.0,
       "step": 6
     },
     {
-      "entropy": 2.442482829093933,
+      "entropy": 2.4412740767002106,
       "epoch": 0.21455938697318008,
-      "grad_norm": 10.625,
+      "grad_norm": 10.6875,
       "learning_rate": 1.2e-05,
-      "loss": 2.2507,
-      "mean_token_accuracy": 0.4729406535625458,
+      "loss": 2.2487,
+      "mean_token_accuracy": 0.472238015383482,
       "num_tokens": 12215.0,
       "step": 7
     },
     {
-      "entropy": 2.356838285923004,
+      "entropy": 2.3576506674289703,
       "epoch": 0.24521072796934865,
-      "grad_norm": 8.6875,
+      "grad_norm": 9.0625,
       "learning_rate": 1.4e-05,
-      "loss": 2.003,
-      "mean_token_accuracy": 0.5002904869616032,
+      "loss": 2.0025,
+      "mean_token_accuracy": 0.5058155283331871,
       "num_tokens": 13939.0,
       "step": 8
     },
     {
-      "entropy": 2.3505502343177795,
+      "entropy": 2.3528400361537933,
       "epoch": 0.27586206896551724,
-      "grad_norm": 7.0,
+      "grad_norm": 7.46875,
       "learning_rate": 1.6e-05,
-      "loss": 1.8798,
-      "mean_token_accuracy": 0.4973266273736954,
+      "loss": 1.885,
+      "mean_token_accuracy": 0.4919443018734455,
       "num_tokens": 15986.0,
       "step": 9
     },
     {
-      "entropy": 2.3207567632198334,
+      "entropy": 2.331418916583061,
       "epoch": 0.3065134099616858,
-      "grad_norm": 6.78125,
+      "grad_norm": 7.15625,
       "learning_rate": 1.8e-05,
-      "loss": 1.8644,
-      "mean_token_accuracy": 0.5209791846573353,
+      "loss": 1.8574,
+      "mean_token_accuracy": 0.5213944055140018,
       "num_tokens": 18444.0,
       "step": 10
     },
     {
-      "entropy": 2.071987748146057,
+      "entropy": 2.081568643450737,
       "epoch": 0.3371647509578544,
-      "grad_norm": 5.34375,
+      "grad_norm": 5.4375,
       "learning_rate": 1.9999999999999998e-05,
-      "loss": 1.6628,
-      "mean_token_accuracy": 0.56425766274333,
+      "loss": 1.6607,
+      "mean_token_accuracy": 0.5659460052847862,
       "num_tokens": 21127.0,
       "step": 11
     },
     {
-      "entropy": 2.291467070579529,
+      "entropy": 2.297955021262169,
       "epoch": 0.367816091954023,
-      "grad_norm": 6.40625,
+      "grad_norm": 6.59375,
       "learning_rate": 2.2e-05,
-      "loss": 1.9017,
-      "mean_token_accuracy": 0.5183870233595371,
+      "loss": 1.8978,
+      "mean_token_accuracy": 0.5127614960074425,
       "num_tokens": 23308.0,
       "step": 12
     },
     {
-      "entropy": 2.2638841718435287,
+      "entropy": 2.2664552181959152,
       "epoch": 0.39846743295019155,
-      "grad_norm": 6.5625,
+      "grad_norm": 6.8125,
       "learning_rate": 2.4e-05,
-      "loss": 1.975,
-      "mean_token_accuracy": 0.5189315676689148,
+      "loss": 1.9761,
+      "mean_token_accuracy": 0.5233316533267498,
       "num_tokens": 25072.0,
       "step": 13
     },
     {
-      "entropy": 2.312656208872795,
+      "entropy": 2.312450557947159,
       "epoch": 0.42911877394636017,
-      "grad_norm": 7.53125,
+      "grad_norm": 7.65625,
       "learning_rate": 2.6000000000000002e-05,
-      "loss": 1.9894,
-      "mean_token_accuracy": 0.5101005285978317,
+      "loss": 1.9805,
+      "mean_token_accuracy": 0.5128904804587364,
       "num_tokens": 26450.0,
       "step": 14
     },
     {
-      "entropy": 2.285459041595459,
+      "entropy": 2.2872647643089294,
       "epoch": 0.45977011494252873,
-      "grad_norm": 6.78125,
+      "grad_norm": 6.90625,
       "learning_rate": 2.8e-05,
-      "loss": 1.8031,
-      "mean_token_accuracy": 0.5333016850054264,
+      "loss": 1.7994,
+      "mean_token_accuracy": 0.527646865695715,
       "num_tokens": 27912.0,
       "step": 15
     },
     {
-      "entropy": 2.1630422174930573,
+      "entropy": 2.1604571640491486,
       "epoch": 0.4904214559386973,
-      "grad_norm": 6.4375,
+      "grad_norm": 6.53125,
       "learning_rate": 3e-05,
-      "loss": 1.6778,
-      "mean_token_accuracy": 0.5712834745645523,
+      "loss": 1.6737,
+      "mean_token_accuracy": 0.5715724155306816,
       "num_tokens": 29392.0,
       "step": 16
     },
     {
-      "entropy": 2.0271100252866745,
+      "entropy": 2.0225837230682373,
       "epoch": 0.5210727969348659,
-      "grad_norm": 4.84375,
-      "learning_rate": 2.9996710252122685e-05,
-      "loss": 1.4808,
-      "mean_token_accuracy": 0.5967330485582352,
+      "grad_norm": 4.96875,
+      "learning_rate": 2.998951057182598e-05,
+      "loss": 1.4747,
+      "mean_token_accuracy": 0.5897958800196648,
       "num_tokens": 31417.0,
       "step": 17
     },
     {
-      "entropy": 2.046408548951149,
+      "entropy": 2.0494654029607773,
       "epoch": 0.5517241379310345,
-      "grad_norm": 6.0,
-      "learning_rate": 2.9986842451482876e-05,
-      "loss": 1.7747,
-      "mean_token_accuracy": 0.5322429053485394,
+      "grad_norm": 6.1875,
+      "learning_rate": 2.99580569577177e-05,
+      "loss": 1.7777,
+      "mean_token_accuracy": 0.5307912044227123,
       "num_tokens": 33727.0,
       "step": 18
     },
     {
-      "entropy": 1.9437416046857834,
+      "entropy": 1.9438972026109695,
       "epoch": 0.5823754789272031,
-      "grad_norm": 5.71875,
-      "learning_rate": 2.9970400926424075e-05,
-      "loss": 1.5966,
-      "mean_token_accuracy": 0.5690519884228706,
+      "grad_norm": 5.90625,
+      "learning_rate": 2.9905683148398642e-05,
+      "loss": 1.5959,
+      "mean_token_accuracy": 0.5674299821257591,
       "num_tokens": 35836.0,
       "step": 19
     },
     {
-      "entropy": 2.019773244857788,
+      "entropy": 2.017078399658203,
       "epoch": 0.6130268199233716,
-      "grad_norm": 5.8125,
-      "learning_rate": 2.9947392888742566e-05,
-      "loss": 1.7132,
-      "mean_token_accuracy": 0.5585189834237099,
+      "grad_norm": 7.375,
+      "learning_rate": 2.9832462393376926e-05,
+      "loss": 1.7129,
+      "mean_token_accuracy": 0.5537029281258583,
       "num_tokens": 37639.0,
       "step": 20
     },
     {
-      "entropy": 2.1920904517173767,
+      "entropy": 2.1857858449220657,
       "epoch": 0.6436781609195402,
-      "grad_norm": 5.59375,
-      "learning_rate": 2.99178284305241e-05,
-      "loss": 1.7894,
-      "mean_token_accuracy": 0.5173998288810253,
+      "grad_norm": 5.71875,
+      "learning_rate": 2.9738497098499325e-05,
+      "loss": 1.7807,
+      "mean_token_accuracy": 0.5209250301122665,
       "num_tokens": 39351.0,
       "step": 21
     },
     {
-      "entropy": 1.9828892648220062,
+      "entropy": 1.9829543828964233,
       "epoch": 0.6743295019157088,
-      "grad_norm": 5.96875,
-      "learning_rate": 2.988172051971717e-05,
-      "loss": 1.5489,
-      "mean_token_accuracy": 0.5762875340878963,
+      "grad_norm": 6.125,
+      "learning_rate": 2.9623918682727355e-05,
+      "loss": 1.5466,
+      "mean_token_accuracy": 0.5822870768606663,
       "num_tokens": 41024.0,
       "step": 22
     },
     {
-      "entropy": 2.076566159725189,
+      "entropy": 2.069172888994217,
       "epoch": 0.7049808429118773,
-      "grad_norm": 5.96875,
-      "learning_rate": 2.983908499444483e-05,
-      "loss": 1.7828,
-      "mean_token_accuracy": 0.554260715842247,
+      "grad_norm": 6.34375,
+      "learning_rate": 2.9488887394336025e-05,
+      "loss": 1.7818,
+      "mean_token_accuracy": 0.553209200501442,
       "num_tokens": 42624.0,
       "step": 23
     },
     {
-      "entropy": 2.1858795881271362,
+      "entropy": 2.179490625858307,
       "epoch": 0.735632183908046,
-      "grad_norm": 7.75,
-      "learning_rate": 2.9789940556057574e-05,
-      "loss": 1.9101,
-      "mean_token_accuracy": 0.5420451276004314,
+      "grad_norm": 7.78125,
+      "learning_rate": 2.9333592086792113e-05,
+      "loss": 1.9207,
+      "mean_token_accuracy": 0.5399925448000431,
       "num_tokens": 43836.0,
       "step": 24
     },
     {
-      "entropy": 2.0579274892807007,
+      "entropy": 2.051348105072975,
       "epoch": 0.7662835249042146,
-      "grad_norm": 5.0625,
-      "learning_rate": 2.9734308760930333e-05,
-      "loss": 1.7436,
-      "mean_token_accuracy": 0.5261486992239952,
+      "grad_norm": 5.1875,
+      "learning_rate": 2.9158249954625514e-05,
+      "loss": 1.7447,
+      "mean_token_accuracy": 0.5221298187971115,
       "num_tokens": 45870.0,
       "step": 25
     },
     {
-      "entropy": 2.0463136434555054,
+      "entropy": 2.050755023956299,
       "epoch": 0.7969348659003831,
-      "grad_norm": 5.3125,
-      "learning_rate": 2.9672214011007087e-05,
-      "loss": 1.6874,
-      "mean_token_accuracy": 0.5605301037430763,
+      "grad_norm": 5.40625,
+      "learning_rate": 2.8963106229663064e-05,
+      "loss": 1.6868,
+      "mean_token_accuracy": 0.563789501786232,
       "num_tokens": 47664.0,
       "step": 26
     },
     {
-      "entropy": 2.0209241062402725,
+      "entropy": 2.0225460678339005,
       "epoch": 0.8275862068965517,
-      "grad_norm": 5.125,
-      "learning_rate": 2.9603683543097406e-05,
-      "loss": 1.6662,
-      "mean_token_accuracy": 0.5574814975261688,
+      "grad_norm": 6.15625,
+      "learning_rate": 2.8748433838049642e-05,
+      "loss": 1.6656,
+      "mean_token_accuracy": 0.5637699663639069,
       "num_tokens": 49646.0,
       "step": 27
     },
     {
-      "entropy": 2.006110742688179,
+      "entropy": 2.0079710334539413,
       "epoch": 0.8582375478927203,
-      "grad_norm": 5.15625,
-      "learning_rate": 2.9528747416929467e-05,
-      "loss": 1.5072,
-      "mean_token_accuracy": 0.5874482207000256,
+      "grad_norm": 5.28125,
+      "learning_rate": 2.8514533018536286e-05,
+      "loss": 1.5144,
+      "mean_token_accuracy": 0.5907510407269001,
       "num_tokens": 51235.0,
       "step": 28
     },
     {
-      "entropy": 2.0461973696947098,
+      "entropy": 2.0421831011772156,
       "epoch": 0.8888888888888888,
-      "grad_norm": 5.40625,
-      "learning_rate": 2.9447438501964873e-05,
-      "loss": 1.6601,
-      "mean_token_accuracy": 0.5673006623983383,
+      "grad_norm": 5.625,
+      "learning_rate": 2.8261730902569146e-05,
+      "loss": 1.6633,
+      "mean_token_accuracy": 0.5682153329253197,
       "num_tokens": 53037.0,
       "step": 29
     },
     {
-      "entropy": 2.0841187089681625,
+      "entropy": 2.089518904685974,
       "epoch": 0.9195402298850575,
-      "grad_norm": 4.96875,
-      "learning_rate": 2.9359792462981007e-05,
-      "loss": 1.6926,
-      "mean_token_accuracy": 0.5491850897669792,
+      "grad_norm": 5.0625,
+      "learning_rate": 2.7990381056766583e-05,
+      "loss": 1.6961,
+      "mean_token_accuracy": 0.5529678054153919,
       "num_tokens": 54826.0,
       "step": 30
     },
     {
-      "entropy": 2.110024333000183,
+      "entropy": 2.1116303354501724,
       "epoch": 0.9501915708812261,
-      "grad_norm": 5.0,
-      "learning_rate": 2.9265847744427305e-05,
-      "loss": 1.6511,
-      "mean_token_accuracy": 0.5542715303599834,
+      "grad_norm": 5.09375,
+      "learning_rate": 2.770086298842426e-05,
+      "loss": 1.652,
+      "mean_token_accuracy": 0.5529469661414623,
       "num_tokens": 56737.0,
       "step": 31
     },
     {
-      "entropy": 2.1065904200077057,
+      "entropy": 2.109420418739319,
       "epoch": 0.9808429118773946,
-      "grad_norm": 6.1875,
-      "learning_rate": 2.9165645553562215e-05,
-      "loss": 1.6936,
-      "mean_token_accuracy": 0.5561113506555557,
+      "grad_norm": 6.15625,
+      "learning_rate": 2.7393581614739924e-05,
+      "loss": 1.694,
+      "mean_token_accuracy": 0.5546637140214443,
       "num_tokens": 58084.0,
       "step": 32
     },
     {
-      "entropy": 1.8917590856552124,
+      "entropy": 1.8913073062896728,
       "epoch": 1.0,
-      "grad_norm": 7.78125,
-      "learning_rate": 2.9059229842378373e-05,
-      "loss": 1.6156,
-      "mean_token_accuracy": 0.5620729327201843,
+      "grad_norm": 7.28125,
+      "learning_rate": 2.7068966696500025e-05,
+      "loss": 1.6355,
+      "mean_token_accuracy": 0.5572340369224549,
       "num_tokens": 59142.0,
       "step": 33
     },
     {
-      "entropy": 1.823348343372345,
+      "entropy": 1.8269917368888855,
       "epoch": 1.0306513409961686,
-      "grad_norm": 4.5,
-      "learning_rate": 2.894664728832377e-05,
-      "loss": 1.2857,
-      "mean_token_accuracy": 0.6347417011857033,
+      "grad_norm": 4.625,
+      "learning_rate": 2.672747223702045e-05,
+      "loss": 1.2993,
+      "mean_token_accuracy": 0.6277801990509033,
       "num_tokens": 60897.0,
       "step": 34
     },
     {
-      "entropy": 1.9907677173614502,
+      "entropy": 1.9859961122274399,
       "epoch": 1.0613026819923372,
-      "grad_norm": 5.5,
-      "learning_rate": 2.8827947273827508e-05,
-      "loss": 1.5295,
-      "mean_token_accuracy": 0.6137617453932762,
+      "grad_norm": 5.625,
+      "learning_rate": 2.6369575847181795e-05,
+      "loss": 1.5442,
+      "mean_token_accuracy": 0.6044492051005363,
       "num_tokens": 62325.0,
       "step": 35
     },
     {
-      "entropy": 1.802370086312294,
+      "entropy": 1.8101668655872345,
       "epoch": 1.0919540229885056,
-      "grad_norm": 4.46875,
-      "learning_rate": 2.8703181864639013e-05,
-      "loss": 1.2187,
-      "mean_token_accuracy": 0.645849384367466,
+      "grad_norm": 4.625,
+      "learning_rate": 2.5995778077447393e-05,
+      "loss": 1.2377,
+      "mean_token_accuracy": 0.6392937749624252,
       "num_tokens": 64163.0,
       "step": 36
     },
     {
-      "entropy": 1.6892547607421875,
+      "entropy": 1.7033060044050217,
       "epoch": 1.1226053639846743,
-      "grad_norm": 4.375,
-      "learning_rate": 2.8572405786990293e-05,
-      "loss": 1.2733,
-      "mean_token_accuracy": 0.6611653044819832,
+      "grad_norm": 4.46875,
+      "learning_rate": 2.5606601717798212e-05,
+      "loss": 1.2791,
+      "mean_token_accuracy": 0.6602629125118256,
       "num_tokens": 66168.0,
       "step": 37
     },
     {
-      "entropy": 1.7277430444955826,
+      "entropy": 1.7478309273719788,
       "epoch": 1.1532567049808429,
-      "grad_norm": 4.28125,
-      "learning_rate": 2.8435676403591193e-05,
-      "loss": 1.2477,
-      "mean_token_accuracy": 0.6409317329525948,
+      "grad_norm": 4.40625,
+      "learning_rate": 2.520259106656379e-05,
+      "loss": 1.2589,
+      "mean_token_accuracy": 0.6367502361536026,
       "num_tokens": 68398.0,
       "step": 38
     },
     {
-      "entropy": 1.6852583438158035,
+      "entropy": 1.7075601518154144,
       "epoch": 1.1839080459770115,
-      "grad_norm": 4.8125,
-      "learning_rate": 2.829305368846822e-05,
-      "loss": 1.2649,
-      "mean_token_accuracy": 0.6238643229007721,
+      "grad_norm": 4.78125,
+      "learning_rate": 2.4784311169171818e-05,
+      "loss": 1.2696,
+      "mean_token_accuracy": 0.619562640786171,
       "num_tokens": 70548.0,
       "step": 39
     },
     {
-      "entropy": 1.6644471734762192,
+      "entropy": 1.6914921998977661,
       "epoch": 1.21455938697318,
-      "grad_norm": 5.40625,
-      "learning_rate": 2.8144600200657953e-05,
-      "loss": 1.2255,
-      "mean_token_accuracy": 0.6601320579648018,
+      "grad_norm": 5.125,
+      "learning_rate": 2.4352347027881003e-05,
+      "loss": 1.2324,
+      "mean_token_accuracy": 0.6510230824351311,
       "num_tokens": 72463.0,
       "step": 40
     },
     {
-      "entropy": 1.5217097848653793,
+      "entropy": 1.5418914556503296,
       "epoch": 1.2452107279693487,
       "grad_norm": 5.71875,
-      "learning_rate": 2.7990381056766583e-05,
-      "loss": 1.1483,
-      "mean_token_accuracy": 0.6674150452017784,
+      "learning_rate": 2.3907302783602522e-05,
+      "loss": 1.1592,
+      "mean_token_accuracy": 0.6608721315860748,
       "num_tokens": 74061.0,
       "step": 41
     },
     {
-      "entropy": 1.5630977302789688,
+      "entropy": 1.585049256682396,
       "epoch": 1.2758620689655173,
       "grad_norm": 4.8125,
-      "learning_rate": 2.78304639024076e-05,
-      "loss": 1.2095,
-      "mean_token_accuracy": 0.6475186944007874,
+      "learning_rate": 2.344980087095433e-05,
+      "loss": 1.2178,
+      "mean_token_accuracy": 0.6534182205796242,
       "num_tokens": 76130.0,
       "step": 42
     },
     {
-      "entropy": 1.5357515215873718,
+      "entropy": 1.5578415244817734,
       "epoch": 1.3065134099616857,
-      "grad_norm": 5.59375,
-      "learning_rate": 2.7664918882530227e-05,
-      "loss": 1.2226,
-      "mean_token_accuracy": 0.651666060090065,
+      "grad_norm": 5.40625,
+      "learning_rate": 2.298048114773005e-05,
+      "loss": 1.2307,
+      "mean_token_accuracy": 0.6517133414745331,
       "num_tokens": 77912.0,
       "step": 43
     },
     {
-      "entropy": 1.447281613945961,
+      "entropy": 1.4642669260501862,
       "epoch": 1.3371647509578544,
-      "grad_norm": 4.8125,
-      "learning_rate": 2.7493818610651493e-05,
-      "loss": 1.1148,
-      "mean_token_accuracy": 0.6739721521735191,
+      "grad_norm": 5.28125,
+      "learning_rate": 2.25e-05,
+      "loss": 1.1424,
+      "mean_token_accuracy": 0.6686440855264664,
       "num_tokens": 79873.0,
       "step": 44
     },
     {
-      "entropy": 1.5578174591064453,
+      "entropy": 1.5738479793071747,
       "epoch": 1.367816091954023,
-      "grad_norm": 5.46875,
-      "learning_rate": 2.731723813700556e-05,
-      "loss": 1.1376,
-      "mean_token_accuracy": 0.652525044977665,
+      "grad_norm": 5.25,
+      "learning_rate": 2.200902942409593e-05,
+      "loss": 1.1464,
+      "mean_token_accuracy": 0.6602782011032104,
       "num_tokens": 81708.0,
       "step": 45
     },
     {
-      "entropy": 1.509346380829811,
+      "entropy": 1.5143821835517883,
       "epoch": 1.3984674329501916,
-      "grad_norm": 6.09375,
-      "learning_rate": 2.7135254915624213e-05,
-      "loss": 1.2312,
-      "mean_token_accuracy": 0.6451430022716522,
+      "grad_norm": 6.03125,
+      "learning_rate": 2.1508256086763372e-05,
+      "loss": 1.2378,
+      "mean_token_accuracy": 0.6555924639105797,
       "num_tokens": 83479.0,
       "step": 46
     },
     {
-      "entropy": 1.4848076701164246,
+      "entropy": 1.4974510818719864,
       "epoch": 1.4291187739463602,
-      "grad_norm": 6.125,
-      "learning_rate": 2.6947948770362945e-05,
-      "loss": 1.2792,
-      "mean_token_accuracy": 0.6499253809452057,
+      "grad_norm": 6.21875,
+      "learning_rate": 2.0998380364796112e-05,
+      "loss": 1.2925,
+      "mean_token_accuracy": 0.6449481919407845,
       "num_tokens": 85091.0,
       "step": 47
     },
     {
-      "entropy": 1.5493847280740738,
+      "entropy": 1.546600267291069,
       "epoch": 1.4597701149425286,
-      "grad_norm": 5.03125,
-      "learning_rate": 2.6755401859887598e-05,
-      "loss": 1.152,
-      "mean_token_accuracy": 0.6553128436207771,
+      "grad_norm": 5.125,
+      "learning_rate": 2.0480115365495928e-05,
+      "loss": 1.1719,
+      "mean_token_accuracy": 0.6519531160593033,
       "num_tokens": 87067.0,
       "step": 48
     },
     {
-      "entropy": 1.4846443086862564,
+      "entropy": 1.4868987947702408,
       "epoch": 1.4904214559386972,
-      "grad_norm": 4.4375,
-      "learning_rate": 2.655769864163684e-05,
-      "loss": 1.0865,
-      "mean_token_accuracy": 0.6880389750003815,
+      "grad_norm": 4.625,
+      "learning_rate": 1.995418592932751e-05,
+      "loss": 1.0959,
+      "mean_token_accuracy": 0.6849717572331429,
       "num_tokens": 89257.0,
       "step": 49
     },
     {
-      "entropy": 1.5293567329645157,
+      "entropy": 1.5329422056674957,
       "epoch": 1.5210727969348659,
-      "grad_norm": 5.5,
-      "learning_rate": 2.6354925834776346e-05,
-      "loss": 1.2067,
-      "mean_token_accuracy": 0.6626974493265152,
+      "grad_norm": 5.625,
+      "learning_rate": 1.9421327616163564e-05,
+      "loss": 1.2158,
+      "mean_token_accuracy": 0.6467685103416443,
       "num_tokens": 91129.0,
       "step": 50
     },
     {
-      "entropy": 1.55410997569561,
+      "entropy": 1.5554772466421127,
       "epoch": 1.5517241379310345,
       "grad_norm": 5.15625,
-      "learning_rate": 2.6147172382160913e-05,
-      "loss": 1.1197,
-      "mean_token_accuracy": 0.6779765337705612,
+      "learning_rate": 1.888228567653781e-05,
+      "loss": 1.1291,
+      "mean_token_accuracy": 0.6766190454363823,
       "num_tokens": 93217.0,
       "step": 51
     },
     {
-      "entropy": 1.5660214275121689,
+      "entropy": 1.5629148334264755,
       "epoch": 1.582375478927203,
-      "grad_norm": 5.59375,
-      "learning_rate": 2.5934529411321174e-05,
-      "loss": 1.0945,
-      "mean_token_accuracy": 0.660072922706604,
+      "grad_norm": 5.71875,
+      "learning_rate": 1.8337814009344716e-05,
+      "loss": 1.1132,
+      "mean_token_accuracy": 0.6620798110961914,
       "num_tokens": 94882.0,
       "step": 52
     },
     {
-      "entropy": 1.6198417395353317,
+      "entropy": 1.6230137348175049,
       "epoch": 1.6130268199233715,
-      "grad_norm": 6.34375,
-      "learning_rate": 2.571709019449205e-05,
-      "loss": 1.1716,
-      "mean_token_accuracy": 0.6578115820884705,
+      "grad_norm": 6.46875,
+      "learning_rate": 1.778867410744372e-05,
+      "loss": 1.1935,
+      "mean_token_accuracy": 0.6662464067339897,
       "num_tokens": 96226.0,
       "step": 53
     },
     {
-      "entropy": 1.5572579056024551,
+      "entropy": 1.5616475045681,
       "epoch": 1.6436781609195403,
-      "grad_norm": 5.5625,
-      "learning_rate": 2.5494950107700482e-05,
-      "loss": 1.138,
-      "mean_token_accuracy": 0.642712339758873,
+      "grad_norm": 5.78125,
+      "learning_rate": 1.7235633992642615e-05,
+      "loss": 1.1494,
+      "mean_token_accuracy": 0.6327598392963409,
       "num_tokens": 98064.0,
       "step": 54
     },
     {
-      "entropy": 1.6491648256778717,
+      "entropy": 1.6592390835285187,
       "epoch": 1.6743295019157087,
-      "grad_norm": 5.125,
-      "learning_rate": 2.5268206588930332e-05,
-      "loss": 1.1748,
-      "mean_token_accuracy": 0.6431401446461678,
+      "grad_norm": 5.25,
+      "learning_rate": 1.667946714154962e-05,
+      "loss": 1.2152,
+      "mean_token_accuracy": 0.6306749582290649,
       "num_tokens": 99875.0,
       "step": 55
     },
     {
-      "entropy": 1.45476695895195,
+      "entropy": 1.454154759645462,
       "epoch": 1.7049808429118773,
-      "grad_norm": 4.625,
-      "learning_rate": 2.5036959095382875e-05,
-      "loss": 1.1288,
-      "mean_token_accuracy": 0.6704870611429214,
+      "grad_norm": 4.65625,
+      "learning_rate": 1.6120951403796367e-05,
+      "loss": 1.1444,
+      "mean_token_accuracy": 0.653047151863575,
       "num_tokens": 102303.0,
       "step": 56
     },
     {
-      "entropy": 1.5417988002300262,
+      "entropy": 1.556449756026268,
       "epoch": 1.735632183908046,
-      "grad_norm": 5.875,
-      "learning_rate": 2.4801309059851586e-05,
-      "loss": 1.1504,
-      "mean_token_accuracy": 0.6553002744913101,
+      "grad_norm": 5.96875,
+      "learning_rate": 1.5560867914144887e-05,
+      "loss": 1.1629,
+      "mean_token_accuracy": 0.6586132422089577,
       "num_tokens": 103806.0,
       "step": 57
     },
     {
-      "entropy": 1.611013650894165,
+      "entropy": 1.625523403286934,
       "epoch": 1.7662835249042146,
-      "grad_norm": 7.0,
-      "learning_rate": 2.4561359846230346e-05,
-      "loss": 1.2064,
-      "mean_token_accuracy": 0.6713609769940376,
+      "grad_norm": 7.09375,
+      "learning_rate": 1.5e-05,
+      "loss": 1.2231,
+      "mean_token_accuracy": 0.6647503152489662,
       "num_tokens": 105012.0,
       "step": 58
     },
     {
-      "entropy": 1.683053657412529,
+      "entropy": 1.6887281984090805,
       "epoch": 1.7969348659003832,
-      "grad_norm": 6.15625,
-      "learning_rate": 2.4317216704174656e-05,
-      "loss": 1.2315,
-      "mean_token_accuracy": 0.6650961637496948,
+      "grad_norm": 6.40625,
+      "learning_rate": 1.4439132085855117e-05,
+      "loss": 1.2619,
+      "mean_token_accuracy": 0.6570713892579079,
       "num_tokens": 106373.0,
       "step": 59
     },
     {
-      "entropy": 1.6171066462993622,
+      "entropy": 1.637197270989418,
       "epoch": 1.8275862068965516,
-      "grad_norm": 5.5625,
-      "learning_rate": 2.4068986722935625e-05,
-      "loss": 1.1285,
-      "mean_token_accuracy": 0.6854713410139084,
+      "grad_norm": 5.84375,
+      "learning_rate": 1.3879048596203637e-05,
+      "loss": 1.1806,
+      "mean_token_accuracy": 0.6695754155516624,
       "num_tokens": 107938.0,
       "step": 60
     },
     {
-      "entropy": 1.5850916504859924,
+      "entropy": 1.5931589007377625,
       "epoch": 1.8582375478927204,
-      "grad_norm": 5.625,
-      "learning_rate": 2.3816778784387097e-05,
-      "loss": 1.1496,
-      "mean_token_accuracy": 0.6636858507990837,
+      "grad_norm": 6.0,
+      "learning_rate": 1.3320532858450382e-05,
+      "loss": 1.1908,
+      "mean_token_accuracy": 0.6473727226257324,
       "num_tokens": 109587.0,
       "step": 61
     },
     {
-      "entropy": 1.6505294740200043,
+      "entropy": 1.6761258989572525,
       "epoch": 1.8888888888888888,
-      "grad_norm": 6.46875,
-      "learning_rate": 2.356070351526648e-05,
-      "loss": 1.4317,
-      "mean_token_accuracy": 0.6071530058979988,
+      "grad_norm": 6.59375,
+      "learning_rate": 1.2764366007357382e-05,
+      "loss": 1.4479,
+      "mean_token_accuracy": 0.6152776852250099,
       "num_tokens": 111253.0,
       "step": 62
     },
     {
-      "entropy": 1.5908723175525665,
+      "entropy": 1.6198187619447708,
       "epoch": 1.9195402298850575,
-      "grad_norm": 6.34375,
-      "learning_rate": 2.3300873238650163e-05,
-      "loss": 1.2333,
-      "mean_token_accuracy": 0.6693138256669044,
+      "grad_norm": 6.09375,
+      "learning_rate": 1.2211325892556282e-05,
+      "loss": 1.2462,
+      "mean_token_accuracy": 0.6710881665349007,
       "num_tokens": 112833.0,
       "step": 63
     },
     {
-      "entropy": 1.5132576674222946,
+      "entropy": 1.5342242866754532,
       "epoch": 1.950191570881226,
-      "grad_norm": 5.84375,
-      "learning_rate": 2.303740192468495e-05,
-      "loss": 1.2285,
-      "mean_token_accuracy": 0.651925578713417,
+      "grad_norm": 5.75,
+      "learning_rate": 1.1662185990655285e-05,
+      "loss": 1.2428,
+      "mean_token_accuracy": 0.6555797532200813,
       "num_tokens": 114573.0,
       "step": 64
     },
     {
-      "entropy": 1.5014029294252396,
+      "entropy": 1.5250054597854614,
       "epoch": 1.9808429118773945,
-      "grad_norm": 4.9375,
-      "learning_rate": 2.2770405140596954e-05,
-      "loss": 1.1041,
-      "mean_token_accuracy": 0.6395103111863136,
+      "grad_norm": 4.90625,
+      "learning_rate": 1.1117714323462188e-05,
+      "loss": 1.1106,
+      "mean_token_accuracy": 0.6406564339995384,
       "num_tokens": 116981.0,
       "step": 65
     },
     {
-      "entropy": 1.5801513910293579,
+      "entropy": 1.5915982723236084,
       "epoch": 2.0,
-      "grad_norm": 7.21875,
-      "learning_rate": 2.25e-05,
-      "loss": 1.387,
-      "mean_token_accuracy": 0.6241851329803467,
+      "grad_norm": 9.25,
+      "learning_rate": 1.0578672383836437e-05,
+      "loss": 1.3903,
+      "mean_token_accuracy": 0.6222738027572632,
       "num_tokens": 118284.0,
       "step": 66
     },
     {
-      "entropy": 1.5819022357463837,
+      "entropy": 1.6206194460391998,
       "epoch": 2.0306513409961684,
-      "grad_norm": 5.5,
-      "learning_rate": 2.222630511152573e-05,
-      "loss": 0.8186,
-      "mean_token_accuracy": 0.7697352692484856,
+      "grad_norm": 5.8125,
+      "learning_rate": 1.0045814070672498e-05,
+      "loss": 0.9269,
+      "mean_token_accuracy": 0.7339452728629112,
       "num_tokens": 119663.0,
       "step": 67
     },
     {
-      "entropy": 1.473861888051033,
+      "entropy": 1.520051747560501,
       "epoch": 2.0613026819923372,
-      "grad_norm": 4.46875,
-      "learning_rate": 2.1949440526797928e-05,
-      "loss": 0.7233,
-      "mean_token_accuracy": 0.7909102663397789,
+      "grad_norm": 4.625,
+      "learning_rate": 9.519884634504074e-06,
+      "loss": 0.8334,
+      "mean_token_accuracy": 0.7491423934698105,
       "num_tokens": 121476.0,
       "step": 68
     },
     {
-      "entropy": 1.4712600708007812,
+      "entropy": 1.5426668524742126,
       "epoch": 2.0919540229885056,
-      "grad_norm": 4.15625,
-      "learning_rate": 2.166952768777391e-05,
-      "loss": 0.7592,
-      "mean_token_accuracy": 0.7729948312044144,
+      "grad_norm": 4.25,
+      "learning_rate": 9.001619635203889e-06,
+      "loss": 0.8431,
+      "mean_token_accuracy": 0.7333525344729424,
       "num_tokens": 123792.0,
       "step": 69
     },
     {
-      "entropy": 1.4692800045013428,
+      "entropy": 1.5813151746988297,
       "epoch": 2.1226053639846745,
-      "grad_norm": 4.96875,
-      "learning_rate": 2.138668937347609e-05,
-      "loss": 0.7338,
-      "mean_token_accuracy": 0.7888433858752251,
+      "grad_norm": 5.25,
+      "learning_rate": 8.491743913236629e-06,
+      "loss": 0.8621,
+      "mean_token_accuracy": 0.7509580850601196,
       "num_tokens": 125329.0,
       "step": 70
     },
     {
-      "entropy": 1.4230495542287827,
+      "entropy": 1.5605921745300293,
       "epoch": 2.153256704980843,
-      "grad_norm": 5.4375,
-      "learning_rate": 2.1101049646137008e-05,
-      "loss": 0.7094,
-      "mean_token_accuracy": 0.7864886075258255,
+      "grad_norm": 5.5,
+      "learning_rate": 7.99097057590407e-06,
+      "loss": 0.825,
+      "mean_token_accuracy": 0.7649728059768677,
       "num_tokens": 126654.0,
       "step": 71
     },
     {
-      "entropy": 1.2409694343805313,
+      "entropy": 1.404028445482254,
       "epoch": 2.1839080459770113,
-      "grad_norm": 4.6875,
-      "learning_rate": 2.0812733796781544e-05,
-      "loss": 0.6902,
-      "mean_token_accuracy": 0.7980181872844696,
+      "grad_norm": 4.65625,
+      "learning_rate": 7.500000000000004e-06,
+      "loss": 0.8113,
+      "mean_token_accuracy": 0.7664826065301895,
       "num_tokens": 128629.0,
       "step": 72
     },
     {
-      "entropy": 1.241740956902504,
+      "entropy": 1.4577097594738007,
       "epoch": 2.21455938697318,
-      "grad_norm": 5.90625,
-      "learning_rate": 2.052186829027017e-05,
-      "loss": 0.8967,
-      "mean_token_accuracy": 0.7425272911787033,
+      "grad_norm": 5.53125,
+      "learning_rate": 7.019518852269953e-06,
+      "loss": 1.0503,
+      "mean_token_accuracy": 0.7050014063715935,
       "num_tokens": 130344.0,
       "step": 73
     },
     {
-      "entropy": 1.15965236723423,
+      "entropy": 1.4058507531881332,
       "epoch": 2.2452107279693485,
-      "grad_norm": 5.625,
-      "learning_rate": 2.022858070982723e-05,
-      "loss": 0.728,
-      "mean_token_accuracy": 0.792903944849968,
+      "grad_norm": 5.03125,
+      "learning_rate": 6.55019912904567e-06,
+      "loss": 0.8932,
+      "mean_token_accuracy": 0.7476209998130798,
       "num_tokens": 132152.0,
       "step": 74
     },
     {
-      "entropy": 1.1466698423027992,
+      "entropy": 1.4188005179166794,
       "epoch": 2.2758620689655173,
-      "grad_norm": 5.28125,
-      "learning_rate": 1.993299970107875e-05,
-      "loss": 0.6783,
-      "mean_token_accuracy": 0.7755344063043594,
+      "grad_norm": 4.5625,
+      "learning_rate": 6.092697216397478e-06,
+      "loss": 0.8157,
+      "mean_token_accuracy": 0.7511886954307556,
       "num_tokens": 134144.0,
       "step": 75
     },
     {
-      "entropy": 1.0256517007946968,
+      "entropy": 1.332677274942398,
       "epoch": 2.3065134099616857,
-      "grad_norm": 5.65625,
-      "learning_rate": 1.963525491562421e-05,
-      "loss": 0.7747,
-      "mean_token_accuracy": 0.7711224406957626,
+      "grad_norm": 4.34375,
+      "learning_rate": 5.647652972118998e-06,
+      "loss": 0.9148,
+      "mean_token_accuracy": 0.7313786372542381,
       "num_tokens": 136715.0,
       "step": 76
     },
     {
-      "entropy": 1.0880693718791008,
+      "entropy": 1.3922066241502762,
       "epoch": 2.3371647509578546,
-      "grad_norm": 5.96875,
-      "learning_rate": 1.9335476954167073e-05,
-      "loss": 0.7357,
-      "mean_token_accuracy": 0.7778737023472786,
+      "grad_norm": 4.625,
+      "learning_rate": 5.2156888308281875e-06,
+      "loss": 0.8858,
+      "mean_token_accuracy": 0.7430698126554489,
       "num_tokens": 138907.0,
       "step": 77
     },
     {
-      "entropy": 1.0792594850063324,
+      "entropy": 1.3953599333763123,
       "epoch": 2.367816091954023,
-      "grad_norm": 6.875,
-      "learning_rate": 1.9033797309228984e-05,
-      "loss": 0.622,
-      "mean_token_accuracy": 0.8023911938071251,
+      "grad_norm": 5.0625,
+      "learning_rate": 4.797408933436207e-06,
+      "loss": 0.812,
+      "mean_token_accuracy": 0.7497159391641617,
       "num_tokens": 140536.0,
       "step": 78
     },
     {
-      "entropy": 1.047565944492817,
+      "entropy": 1.3784821927547455,
       "epoch": 2.3984674329501914,
-      "grad_norm": 7.90625,
-      "learning_rate": 1.8730348307472828e-05,
-      "loss": 0.6963,
-      "mean_token_accuracy": 0.8018106818199158,
+      "grad_norm": 5.6875,
+      "learning_rate": 4.393398282201788e-06,
+      "loss": 0.9021,
+      "mean_token_accuracy": 0.7352318316698074,
       "num_tokens": 142205.0,
       "step": 79
     },
     {
-      "entropy": 1.0534571334719658,
+      "entropy": 1.3571110516786575,
       "epoch": 2.42911877394636,
-      "grad_norm": 7.09375,
-      "learning_rate": 1.8425263051659838e-05,
-      "loss": 0.6769,
-      "mean_token_accuracy": 0.7829258516430855,
+      "grad_norm": 5.46875,
+      "learning_rate": 4.004221922552608e-06,
+      "loss": 0.8427,
+      "mean_token_accuracy": 0.7509503439068794,
       "num_tokens": 143937.0,
       "step": 80
     },
     {
-      "entropy": 1.06615050137043,
+      "entropy": 1.342074140906334,
       "epoch": 2.4597701149425286,
-      "grad_norm": 6.96875,
-      "learning_rate": 1.8118675362266388e-05,
-      "loss": 0.724,
-      "mean_token_accuracy": 0.7941268905997276,
+      "grad_norm": 5.125,
+      "learning_rate": 3.630424152818203e-06,
+      "loss": 0.8727,
+      "mean_token_accuracy": 0.7591300755739212,
       "num_tokens": 145867.0,
       "step": 81
     },
     {
-      "entropy": 1.076167106628418,
+      "entropy": 1.3443543463945389,
       "epoch": 2.4904214559386975,
-      "grad_norm": 7.71875,
-      "learning_rate": 1.781071971878587e-05,
-      "loss": 0.7821,
-      "mean_token_accuracy": 0.774849958717823,
+      "grad_norm": 5.6875,
+      "learning_rate": 3.272527762979553e-06,
+      "loss": 0.92,
+      "mean_token_accuracy": 0.7423544898629189,
       "num_tokens": 147522.0,
       "step": 82
     },
     {
-      "entropy": 1.1518089175224304,
+      "entropy": 1.4216418862342834,
       "epoch": 2.521072796934866,
-      "grad_norm": 8.3125,
-      "learning_rate": 1.7501531200741536e-05,
-      "loss": 0.7973,
-      "mean_token_accuracy": 0.7875072434544563,
+      "grad_norm": 6.0625,
+      "learning_rate": 2.931033303499975e-06,
+      "loss": 0.946,
+      "mean_token_accuracy": 0.7276971638202667,
       "num_tokens": 149154.0,
       "step": 83
     },
     {
-      "entropy": 1.036544032394886,
+      "entropy": 1.2710229381918907,
       "epoch": 2.5517241379310347,
-      "grad_norm": 5.9375,
-      "learning_rate": 1.7191245428436175e-05,
-      "loss": 0.7165,
-      "mean_token_accuracy": 0.8045910075306892,
+      "grad_norm": 4.84375,
+      "learning_rate": 2.60641838526008e-06,
+      "loss": 0.868,
+      "mean_token_accuracy": 0.7580988109111786,
       "num_tokens": 151443.0,
       "step": 84
     },
     {
-      "entropy": 1.1293774545192719,
+      "entropy": 1.3590690642595291,
       "epoch": 2.582375478927203,
-      "grad_norm": 6.125,
-      "learning_rate": 1.6879998503464565e-05,
-      "loss": 0.6904,
-      "mean_token_accuracy": 0.8081322461366653,
+      "grad_norm": 5.21875,
+      "learning_rate": 2.2991370115757383e-06,
+      "loss": 0.8524,
+      "mean_token_accuracy": 0.7542793154716492,
       "num_tokens": 153346.0,
       "step": 85
     },
     {
-      "entropy": 1.1728527545928955,
+      "entropy": 1.3945435881614685,
       "epoch": 2.6130268199233715,
-      "grad_norm": 6.46875,
-      "learning_rate": 1.6567926949014805e-05,
-      "loss": 0.6986,
-      "mean_token_accuracy": 0.7757782712578773,
+      "grad_norm": 7.4375,
+      "learning_rate": 2.0096189432334194e-06,
+      "loss": 0.8715,
+      "mean_token_accuracy": 0.7236102521419525,
       "num_tokens": 155041.0,
       "step": 86
     },
     {
-      "entropy": 1.0829114317893982,
+      "entropy": 1.2569653242826462,
       "epoch": 2.6436781609195403,
-      "grad_norm": 6.21875,
-      "learning_rate": 1.6255167649984738e-05,
-      "loss": 0.6176,
-      "mean_token_accuracy": 0.8206781521439552,
+      "grad_norm": 5.71875,
+      "learning_rate": 1.7382690974308551e-06,
+      "loss": 0.7554,
+      "mean_token_accuracy": 0.7710509598255157,
       "num_tokens": 156508.0,
       "step": 87
     },
     {
-      "entropy": 1.1221561282873154,
+      "entropy": 1.3096674382686615,
       "epoch": 2.6743295019157087,
-      "grad_norm": 5.71875,
-      "learning_rate": 1.5941857792939702e-05,
-      "loss": 0.6748,
-      "mean_token_accuracy": 0.7972528710961342,
+      "grad_norm": 5.1875,
+      "learning_rate": 1.4854669814637145e-06,
+      "loss": 0.8363,
+      "mean_token_accuracy": 0.7428526133298874,
       "num_tokens": 158506.0,
       "step": 88
     },
     {
-      "entropy": 1.1622852012515068,
+      "entropy": 1.3830756843090057,
       "epoch": 2.704980842911877,
-      "grad_norm": 5.4375,
-      "learning_rate": 1.5628134805937995e-05,
-      "loss": 0.6891,
-      "mean_token_accuracy": 0.7869931757450104,
+      "grad_norm": 5.0,
+      "learning_rate": 1.2515661619503572e-06,
+      "loss": 0.8451,
+      "mean_token_accuracy": 0.7292191162705421,
       "num_tokens": 160511.0,
       "step": 89
     },
     {
-      "entropy": 1.1033327355980873,
+      "entropy": 1.3252490162849426,
       "epoch": 2.735632183908046,
-      "grad_norm": 17.0,
-      "learning_rate": 1.5314136298250355e-05,
-      "loss": 0.6848,
-      "mean_token_accuracy": 0.7839160785079002,
+      "grad_norm": 5.15625,
+      "learning_rate": 1.036893770336938e-06,
+      "loss": 0.877,
+      "mean_token_accuracy": 0.7445427924394608,
       "num_tokens": 162548.0,
       "step": 90
     },
     {
-      "entropy": 1.0911096036434174,
+      "entropy": 1.290112927556038,
       "epoch": 2.766283524904215,
-      "grad_norm": 5.03125,
-      "learning_rate": 1.5e-05,
-      "loss": 0.7139,
-      "mean_token_accuracy": 0.7690701261162758,
+      "grad_norm": 4.59375,
+      "learning_rate": 8.417500453744864e-07,
+      "loss": 0.8187,
+      "mean_token_accuracy": 0.7387886121869087,
       "num_tokens": 164874.0,
       "step": 91
     },
     {
-      "entropy": 1.0845600441098213,
+      "entropy": 1.28690767288208,
       "epoch": 2.796934865900383,
-      "grad_norm": 5.9375,
-      "learning_rate": 1.4685863701749648e-05,
-      "loss": 0.6626,
-      "mean_token_accuracy": 0.8035678789019585,
+      "grad_norm": 5.6875,
+      "learning_rate": 6.664079132078881e-07,
+      "loss": 0.832,
+      "mean_token_accuracy": 0.7535636648535728,
       "num_tokens": 166614.0,
       "step": 92
     },
     {
-      "entropy": 1.1177793815732002,
+      "entropy": 1.3516953140497208,
       "epoch": 2.8275862068965516,
-      "grad_norm": 7.0,
-      "learning_rate": 1.4371865194062009e-05,
-      "loss": 0.776,
-      "mean_token_accuracy": 0.7617417871952057,
+      "grad_norm": 6.46875,
+      "learning_rate": 5.11112605663977e-07,
+      "loss": 0.9553,
+      "mean_token_accuracy": 0.7194050028920174,
       "num_tokens": 168220.0,
       "step": 93
     },
     {
-      "entropy": 1.1276764497160912,
+      "entropy": 1.3391648083925247,
       "epoch": 2.8582375478927204,
-      "grad_norm": 8.8125,
-      "learning_rate": 1.40581422070603e-05,
-      "loss": 0.6741,
-      "mean_token_accuracy": 0.7929504066705704,
+      "grad_norm": 6.9375,
+      "learning_rate": 3.760813172726457e-07,
+      "loss": 0.8756,
+      "mean_token_accuracy": 0.7469412162899971,
       "num_tokens": 169540.0,
       "step": 94
     },
     {
-      "entropy": 1.072372280061245,
+      "entropy": 1.256810650229454,
       "epoch": 2.888888888888889,
-      "grad_norm": 4.84375,
-      "learning_rate": 1.3744832350015271e-05,
-      "loss": 0.6173,
-      "mean_token_accuracy": 0.8033151254057884,
+      "grad_norm": 4.71875,
+      "learning_rate": 2.6150290150067593e-07,
+      "loss": 0.7591,
+      "mean_token_accuracy": 0.7443727105855942,
       "num_tokens": 171709.0,
       "step": 95
-    },
-    {
-      "entropy": 1.0452390983700752,
-      "epoch": 2.9195402298850572,
-      "grad_norm": 6.0,
-      "learning_rate": 1.3432073050985201e-05,
-      "loss": 0.6426,
-      "mean_token_accuracy": 0.8162983357906342,
-      "num_tokens": 173432.0,
-      "step": 96
-    },
-    {
-      "entropy": 1.0664306953549385,
-      "epoch": 2.950191570881226,
-      "grad_norm": 7.125,
-      "learning_rate": 1.3120001496535434e-05,
-      "loss": 0.6244,
-      "mean_token_accuracy": 0.8141586259007454,
-      "num_tokens": 174962.0,
-      "step": 97
-    },
-    {
-      "entropy": 1.0960277691483498,
-      "epoch": 2.9808429118773945,
-      "grad_norm": 6.6875,
-      "learning_rate": 1.2808754571563827e-05,
-      "loss": 0.7344,
-      "mean_token_accuracy": 0.7819336354732513,
-      "num_tokens": 176611.0,
-      "step": 98
-    },
-    {
-      "entropy": 1.068971049785614,
-      "epoch": 3.0,
-      "grad_norm": 8.5,
-      "learning_rate": 1.2498468799258468e-05,
-      "loss": 0.5678,
-      "mean_token_accuracy": 0.8390557408332825,
-      "num_tokens": 177426.0,
-      "step": 99
-    },
-    {
-      "entropy": 1.0476091727614403,
-      "epoch": 3.0306513409961684,
-      "grad_norm": 5.21875,
-      "learning_rate": 1.2189280281214128e-05,
-      "loss": 0.4454,
-      "mean_token_accuracy": 0.8829296976327896,
-      "num_tokens": 179048.0,
-      "step": 100
-    },
-    {
-      "entropy": 1.0911573767662048,
-      "epoch": 3.0613026819923372,
-      "grad_norm": 4.71875,
-      "learning_rate": 1.1881324637733613e-05,
-      "loss": 0.4992,
-      "mean_token_accuracy": 0.8481416404247284,
-      "num_tokens": 181115.0,
-      "step": 101
-    },
-    {
-      "entropy": 1.1437290906906128,
-      "epoch": 3.0919540229885056,
-      "grad_norm": 5.625,
-      "learning_rate": 1.1574736948340163e-05,
-      "loss": 0.5052,
-      "mean_token_accuracy": 0.8615396842360497,
-      "num_tokens": 182578.0,
-      "step": 102
-    },
-    {
-      "entropy": 1.0376266986131668,
-      "epoch": 3.1226053639846745,
-      "grad_norm": 5.5,
-      "learning_rate": 1.1269651692527181e-05,
-      "loss": 0.4683,
-      "mean_token_accuracy": 0.870830163359642,
-      "num_tokens": 184376.0,
-      "step": 103
-    },
-    {
-      "entropy": 0.9589341431856155,
-      "epoch": 3.153256704980843,
-      "grad_norm": 4.46875,
-      "learning_rate": 1.0966202690771015e-05,
-      "loss": 0.4558,
-      "mean_token_accuracy": 0.8758394122123718,
-      "num_tokens": 186551.0,
-      "step": 104
-    },
-    {
-      "entropy": 0.9614228904247284,
-      "epoch": 3.1839080459770113,
-      "grad_norm": 5.34375,
-      "learning_rate": 1.0664523045832923e-05,
-      "loss": 0.3656,
-      "mean_token_accuracy": 0.9130250364542007,
-      "num_tokens": 187965.0,
-      "step": 105
-    },
-    {
-      "entropy": 0.927314467728138,
-      "epoch": 3.21455938697318,
-      "grad_norm": 4.8125,
-      "learning_rate": 1.036474508437579e-05,
-      "loss": 0.3549,
-      "mean_token_accuracy": 0.9026911705732346,
-      "num_tokens": 189763.0,
-      "step": 106
-    },
-    {
-      "entropy": 0.8468578010797501,
-      "epoch": 3.2452107279693485,
-      "grad_norm": 5.375,
-      "learning_rate": 1.0067000298921251e-05,
-      "loss": 0.3705,
-      "mean_token_accuracy": 0.8972815871238708,
-      "num_tokens": 191586.0,
-      "step": 107
-    },
-    {
-      "entropy": 0.847766101360321,
-      "epoch": 3.2758620689655173,
-      "grad_norm": 5.53125,
-      "learning_rate": 9.771419290172776e-06,
-      "loss": 0.3128,
-      "mean_token_accuracy": 0.9080770388245583,
-      "num_tokens": 193154.0,
-      "step": 108
-    },
-    {
-      "entropy": 0.8512594848871231,
-      "epoch": 3.3065134099616857,
-      "grad_norm": 6.59375,
-      "learning_rate": 9.478131709729831e-06,
-      "loss": 0.4468,
-      "mean_token_accuracy": 0.8701968789100647,
-      "num_tokens": 194813.0,
-      "step": 109
-    },
-    {
-      "entropy": 0.85344647616148,
-      "epoch": 3.3371647509578546,
-      "grad_norm": 5.125,
-      "learning_rate": 9.187266203218457e-06,
-      "loss": 0.4125,
-      "mean_token_accuracy": 0.8799988329410553,
-      "num_tokens": 196957.0,
-      "step": 110
-    },
-    {
-      "entropy": 0.8800590857863426,
-      "epoch": 3.367816091954023,
-      "grad_norm": 7.53125,
-      "learning_rate": 8.898950353863e-06,
-      "loss": 0.3593,
-      "mean_token_accuracy": 0.8787510544061661,
-      "num_tokens": 198398.0,
-      "step": 111
-    },
-    {
-      "entropy": 0.7779126092791557,
-      "epoch": 3.3984674329501914,
-      "grad_norm": 6.21875,
-      "learning_rate": 8.61331062652391e-06,
-      "loss": 0.3267,
-      "mean_token_accuracy": 0.9116306900978088,
-      "num_tokens": 200136.0,
-      "step": 112
-    },
-    {
-      "entropy": 0.7932204827666283,
-      "epoch": 3.42911877394636,
-      "grad_norm": 6.96875,
-      "learning_rate": 8.330472312226091e-06,
-      "loss": 0.3873,
-      "mean_token_accuracy": 0.887348584830761,
-      "num_tokens": 201989.0,
-      "step": 113
-    },
-    {
-      "entropy": 0.6965708956122398,
-      "epoch": 3.4597701149425286,
-      "grad_norm": 13.0,
-      "learning_rate": 8.050559473202078e-06,
-      "loss": 0.3972,
-      "mean_token_accuracy": 0.890875369310379,
-      "num_tokens": 203992.0,
-      "step": 114
-    },
-    {
-      "entropy": 0.7952002212405205,
-      "epoch": 3.4904214559386975,
-      "grad_norm": 10.0,
-      "learning_rate": 7.773694888474268e-06,
-      "loss": 0.4792,
-      "mean_token_accuracy": 0.8768180310726166,
-      "num_tokens": 205854.0,
-      "step": 115
-    },
-    {
-      "entropy": 0.7920583859086037,
-      "epoch": 3.521072796934866,
-      "grad_norm": 7.25,
-      "learning_rate": 7.500000000000004e-06,
-      "loss": 0.3583,
-      "mean_token_accuracy": 0.8890090510249138,
-      "num_tokens": 207815.0,
-      "step": 116
-    },
-    {
-      "entropy": 0.7490416169166565,
-      "epoch": 3.5517241379310347,
-      "grad_norm": 6.09375,
-      "learning_rate": 7.229594859403049e-06,
-      "loss": 0.3908,
-      "mean_token_accuracy": 0.8729168474674225,
-      "num_tokens": 209918.0,
-      "step": 117
-    },
-    {
-      "entropy": 0.6982008591294289,
-      "epoch": 3.582375478927203,
-      "grad_norm": 7.9375,
-      "learning_rate": 6.962598075315047e-06,
-      "loss": 0.3882,
-      "mean_token_accuracy": 0.8720036372542381,
-      "num_tokens": 211772.0,
-      "step": 118
-    },
-    {
-      "entropy": 0.7215415760874748,
-      "epoch": 3.6130268199233715,
-      "grad_norm": 7.3125,
-      "learning_rate": 6.699126761349841e-06,
-      "loss": 0.3885,
-      "mean_token_accuracy": 0.8832474499940872,
-      "num_tokens": 213561.0,
-      "step": 119
-    },
-    {
-      "entropy": 0.7576938644051552,
-      "epoch": 3.6436781609195403,
-      "grad_norm": 6.4375,
-      "learning_rate": 6.439296484733526e-06,
-      "loss": 0.3824,
-      "mean_token_accuracy": 0.8854040205478668,
-      "num_tokens": 215335.0,
-      "step": 120
-    },
-    {
-      "entropy": 0.6329802498221397,
-      "epoch": 3.6743295019157087,
-      "grad_norm": 4.5,
-      "learning_rate": 6.1832212156129045e-06,
-      "loss": 0.3047,
-      "mean_token_accuracy": 0.9093970954418182,
-      "num_tokens": 218204.0,
-      "step": 121
-    },
-    {
-      "entropy": 0.7503926008939743,
-      "epoch": 3.704980842911877,
-      "grad_norm": 6.28125,
-      "learning_rate": 5.931013277064377e-06,
-      "loss": 0.3702,
-      "mean_token_accuracy": 0.8855115100741386,
-      "num_tokens": 220090.0,
-      "step": 122
-    },
-    {
-      "entropy": 0.7402322217822075,
-      "epoch": 3.735632183908046,
-      "grad_norm": 9.125,
-      "learning_rate": 5.682783295825346e-06,
-      "loss": 0.3685,
-      "mean_token_accuracy": 0.8846122622489929,
-      "num_tokens": 221667.0,
-      "step": 123
-    },
-    {
-      "entropy": 0.8042828142642975,
-      "epoch": 3.766283524904215,
-      "grad_norm": 8.375,
-      "learning_rate": 5.438640153769654e-06,
-      "loss": 0.3649,
-      "mean_token_accuracy": 0.9067123532295227,
-      "num_tokens": 222962.0,
-      "step": 124
-    },
-    {
-      "entropy": 0.7551866173744202,
-      "epoch": 3.796934865900383,
-      "grad_norm": 5.875,
-      "learning_rate": 5.198690940148418e-06,
-      "loss": 0.3998,
-      "mean_token_accuracy": 0.8821066841483116,
-      "num_tokens": 224965.0,
-      "step": 125
-    },
-    {
-      "entropy": 0.8327518254518509,
-      "epoch": 3.8275862068965516,
-      "grad_norm": 6.78125,
-      "learning_rate": 4.963040904617131e-06,
-      "loss": 0.3778,
-      "mean_token_accuracy": 0.8923891186714172,
-      "num_tokens": 226522.0,
-      "step": 126
-    },
-    {
-      "entropy": 0.7820432782173157,
-      "epoch": 3.8582375478927204,
-      "grad_norm": 5.1875,
-      "learning_rate": 4.731793411069669e-06,
-      "loss": 0.4173,
-      "mean_token_accuracy": 0.8775044828653336,
-      "num_tokens": 229172.0,
-      "step": 127
-    },
-    {
-      "entropy": 0.8462389037013054,
-      "epoch": 3.888888888888889,
-      "grad_norm": 6.53125,
-      "learning_rate": 4.505049892299517e-06,
-      "loss": 0.2995,
-      "mean_token_accuracy": 0.9146698862314224,
-      "num_tokens": 230464.0,
-      "step": 128
-    },
-    {
-      "entropy": 0.8228934556245804,
-      "epoch": 3.9195402298850572,
-      "grad_norm": 6.75,
-      "learning_rate": 4.282909805507953e-06,
-      "loss": 0.3327,
-      "mean_token_accuracy": 0.9099200069904327,
-      "num_tokens": 231921.0,
-      "step": 129
-    },
-    {
-      "entropy": 0.8176594227552414,
-      "epoch": 3.950191570881226,
-      "grad_norm": 8.1875,
-      "learning_rate": 4.06547058867883e-06,
-      "loss": 0.3909,
-      "mean_token_accuracy": 0.8916172161698341,
-      "num_tokens": 233200.0,
-      "step": 130
-    },
-    {
-      "entropy": 0.7947382852435112,
-      "epoch": 3.9808429118773945,
-      "grad_norm": 4.90625,
-      "learning_rate": 3.852827617839085e-06,
-      "loss": 0.3452,
-      "mean_token_accuracy": 0.8946295380592346,
-      "num_tokens": 235153.0,
-      "step": 131
-    },
-    {
-      "entropy": 0.7858668446540833,
-      "epoch": 4.0,
-      "grad_norm": 7.125,
-      "learning_rate": 3.645074165223656e-06,
-      "loss": 0.5253,
-      "mean_token_accuracy": 0.8538177251815796,
-      "num_tokens": 236568.0,
-      "step": 132
-    },
-    {
-      "entropy": 0.8421337753534317,
-      "epoch": 4.030651340996169,
-      "grad_norm": 5.03125,
-      "learning_rate": 3.442301358363163e-06,
-      "loss": 0.2725,
-      "mean_token_accuracy": 0.9301287308335304,
-      "num_tokens": 238229.0,
-      "step": 133
-    },
-    {
-      "entropy": 0.8355702385306358,
-      "epoch": 4.061302681992337,
-      "grad_norm": 5.03125,
-      "learning_rate": 3.244598140112404e-06,
-      "loss": 0.2914,
-      "mean_token_accuracy": 0.9221316650509834,
-      "num_tokens": 239958.0,
-      "step": 134
-    },
-    {
-      "entropy": 0.7881455570459366,
-      "epoch": 4.091954022988506,
-      "grad_norm": 4.84375,
-      "learning_rate": 3.052051229637056e-06,
-      "loss": 0.3885,
-      "mean_token_accuracy": 0.9120407700538635,
-      "num_tokens": 242097.0,
-      "step": 135
-    },
-    {
-      "entropy": 0.8195698857307434,
-      "epoch": 4.1226053639846745,
-      "grad_norm": 5.40625,
-      "learning_rate": 2.86474508437579e-06,
-      "loss": 0.2115,
-      "mean_token_accuracy": 0.9468159675598145,
-      "num_tokens": 243375.0,
-      "step": 136
-    },
-    {
-      "entropy": 0.7908449321985245,
-      "epoch": 4.153256704980843,
-      "grad_norm": 3.890625,
-      "learning_rate": 2.6827618629944394e-06,
-      "loss": 0.2748,
-      "mean_token_accuracy": 0.9350386410951614,
-      "num_tokens": 245859.0,
-      "step": 137
-    },
-    {
-      "entropy": 0.7695152685046196,
-      "epoch": 4.183908045977011,
-      "grad_norm": 4.53125,
-      "learning_rate": 2.5061813893485087e-06,
-      "loss": 0.2897,
-      "mean_token_accuracy": 0.9324667826294899,
-      "num_tokens": 248006.0,
-      "step": 138
-    },
-    {
-      "entropy": 0.816169910132885,
-      "epoch": 4.21455938697318,
-      "grad_norm": 5.125,
-      "learning_rate": 2.335081117469777e-06,
-      "loss": 0.2809,
-      "mean_token_accuracy": 0.9280546009540558,
-      "num_tokens": 249615.0,
-      "step": 139
-    },
-    {
-      "entropy": 0.8056535497307777,
-      "epoch": 4.245210727969349,
-      "grad_norm": 5.375,
-      "learning_rate": 2.169536097592401e-06,
-      "loss": 0.2502,
-      "mean_token_accuracy": 0.9439713805913925,
-      "num_tokens": 251133.0,
-      "step": 140
     }
   ],
   "logging_steps": 1,
-  "max_steps": 165,
+  "max_steps": 99,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 3,
   "save_steps": 5,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -1427,7 +977,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6665368235175936.0,
+  "total_flos": 4559405580668928.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:827ef2be292e1537aee3e07967732029beae9cb85d6335a905d123bebbf9cbca
+oid sha256:28a22a0d804c1fd72a92439f81a35c0aabfe6c7861fb91d64ea135d1b443db6e
 size 6353