Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/mingzeli/PreThink_MemAgent/runs/umrk1n8k) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/mingzeli/PreThink_MemAgent/runs/88u8xb90) 
 
 
 This model was trained with SFT.

all_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 2.7972111648056934e+17,
-    "train_loss": 1.0216455256655104,
-    "train_runtime": 2780.2712,
-    "train_samples": 5375,
-    "train_samples_per_second": 9.666,
-    "train_steps_per_second": 0.076
+    "total_flos": 5.614755170526167e+17,
+    "train_loss": 1.1423310605036159,
+    "train_runtime": 3470.2952,
+    "train_samples": 12431,
+    "train_samples_per_second": 10.746,
+    "train_steps_per_second": 0.042
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 2.7972111648056934e+17,
-    "train_loss": 1.0216455256655104,
-    "train_runtime": 2780.2712,
-    "train_samples": 5375,
-    "train_samples_per_second": 9.666,
-    "train_steps_per_second": 0.076
+    "total_flos": 5.614755170526167e+17,
+    "train_loss": 1.1423310605036159,
+    "train_runtime": 3470.2952,
+    "train_samples": 12431,
+    "train_samples_per_second": 10.746,
+    "train_steps_per_second": 0.042
 }

trainer_state.json

@@ -2,1707 +2,1221 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
-  "eval_steps": 500,
-  "global_step": 210,
+  "epoch": 3.0,
+  "eval_steps": 51,
+  "global_step": 147,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.023809523809523808,
-      "grad_norm": 6.538216213930027,
+      "epoch": 0.02056555269922879,
+      "grad_norm": 7.168717487255555,
       "learning_rate": 0.0,
-      "loss": 1.9338,
-      "num_tokens": 301499.0,
+      "loss": 1.6868,
+      "num_tokens": 809024.0,
       "step": 1
     },
     {
-      "epoch": 0.047619047619047616,
-      "grad_norm": 6.773412169837184,
-      "learning_rate": 4.2857142857142855e-06,
-      "loss": 1.9236,
-      "num_tokens": 603383.0,
+      "epoch": 0.04113110539845758,
+      "grad_norm": 7.291622078201161,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 1.6858,
+      "num_tokens": 1592788.0,
       "step": 2
     },
     {
-      "epoch": 0.07142857142857142,
-      "grad_norm": 4.749957858339624,
-      "learning_rate": 8.571428571428571e-06,
-      "loss": 1.8766,
-      "num_tokens": 863622.0,
+      "epoch": 0.061696658097686374,
+      "grad_norm": 6.633388372265764,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 1.6593,
+      "num_tokens": 2354703.0,
       "step": 3
     },
     {
-      "epoch": 0.09523809523809523,
-      "grad_norm": 2.2238600419377277,
-      "learning_rate": 1.2857142857142857e-05,
-      "loss": 1.7506,
-      "num_tokens": 1179346.0,
+      "epoch": 0.08226221079691516,
+      "grad_norm": 3.5110553987567465,
+      "learning_rate": 6e-06,
+      "loss": 1.5722,
+      "num_tokens": 3143138.0,
       "step": 4
     },
     {
-      "epoch": 0.11904761904761904,
-      "grad_norm": 3.7552483574413404,
-      "learning_rate": 1.7142857142857142e-05,
-      "loss": 1.7277,
-      "num_tokens": 1490946.0,
+      "epoch": 0.10282776349614396,
+      "grad_norm": 12.715495674379813,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 1.5493,
+      "num_tokens": 3888034.0,
       "step": 5
     },
     {
-      "epoch": 0.14285714285714285,
-      "grad_norm": 4.15755928790991,
-      "learning_rate": 2.1428571428571428e-05,
-      "loss": 1.658,
-      "num_tokens": 1719799.0,
+      "epoch": 0.12339331619537275,
+      "grad_norm": 24.03794663222114,
+      "learning_rate": 1e-05,
+      "loss": 1.5373,
+      "num_tokens": 4682804.0,
       "step": 6
     },
     {
-      "epoch": 0.16666666666666666,
-      "grad_norm": 2.245417974821929,
-      "learning_rate": 2.5714285714285714e-05,
-      "loss": 1.6608,
-      "num_tokens": 2009958.0,
+      "epoch": 0.14395886889460155,
+      "grad_norm": 19.871112801805985,
+      "learning_rate": 9.998898745083595e-06,
+      "loss": 1.4966,
+      "num_tokens": 5508918.0,
       "step": 7
     },
     {
-      "epoch": 0.19047619047619047,
-      "grad_norm": 3.1872785546255233,
-      "learning_rate": 3e-05,
-      "loss": 1.6032,
-      "num_tokens": 2334131.0,
+      "epoch": 0.16452442159383032,
+      "grad_norm": 12.796838534757285,
+      "learning_rate": 9.995595519339882e-06,
+      "loss": 1.418,
+      "num_tokens": 6259535.0,
       "step": 8
     },
     {
-      "epoch": 0.21428571428571427,
-      "grad_norm": 2.062217429493514,
-      "learning_rate": 2.999838339925525e-05,
-      "loss": 1.6093,
-      "num_tokens": 2659726.0,
+      "epoch": 0.18508997429305912,
+      "grad_norm": 5.341991620474575,
+      "learning_rate": 9.99009193952157e-06,
+      "loss": 1.4099,
+      "num_tokens": 6992386.0,
       "step": 9
     },
     {
-      "epoch": 0.23809523809523808,
-      "grad_norm": 30.952811910237916,
-      "learning_rate": 2.9993533984191048e-05,
-      "loss": 1.6153,
-      "num_tokens": 2997515.0,
+      "epoch": 0.20565552699228792,
+      "grad_norm": 4.762493826344396,
+      "learning_rate": 9.982390699337253e-06,
+      "loss": 1.3722,
+      "num_tokens": 7728349.0,
       "step": 10
     },
     {
-      "epoch": 0.2619047619047619,
-      "grad_norm": 5.437630168475735,
-      "learning_rate": 2.9985452916224895e-05,
-      "loss": 1.6272,
-      "num_tokens": 3298571.0,
+      "epoch": 0.2262210796915167,
+      "grad_norm": 4.232985415108636,
+      "learning_rate": 9.97249556813298e-06,
+      "loss": 1.3674,
+      "num_tokens": 8421223.0,
       "step": 11
     },
     {
-      "epoch": 0.2857142857142857,
-      "grad_norm": 9.067784859415095,
-      "learning_rate": 2.9974142130743523e-05,
-      "loss": 1.6427,
-      "num_tokens": 3621523.0,
+      "epoch": 0.2467866323907455,
+      "grad_norm": 4.678412892018652,
+      "learning_rate": 9.960411389047366e-06,
+      "loss": 1.3579,
+      "num_tokens": 9252435.0,
       "step": 12
     },
     {
-      "epoch": 0.30952380952380953,
-      "grad_norm": 8.297445832624902,
-      "learning_rate": 2.9959604336639406e-05,
-      "loss": 1.6557,
-      "num_tokens": 3918652.0,
+      "epoch": 0.26735218508997427,
+      "grad_norm": 4.981692703295692,
+      "learning_rate": 9.946144076641138e-06,
+      "loss": 1.3257,
+      "num_tokens": 10072798.0,
       "step": 13
     },
     {
-      "epoch": 0.3333333333333333,
-      "grad_norm": 18.611016423707234,
-      "learning_rate": 2.9941843015662005e-05,
-      "loss": 1.6543,
-      "num_tokens": 4189760.0,
+      "epoch": 0.2879177377892031,
+      "grad_norm": 2.8431033488668582,
+      "learning_rate": 9.929700614002265e-06,
+      "loss": 1.3241,
+      "num_tokens": 10915700.0,
       "step": 14
     },
     {
-      "epoch": 0.35714285714285715,
-      "grad_norm": 37.588129897999515,
-      "learning_rate": 2.9920862421583856e-05,
-      "loss": 1.6303,
-      "num_tokens": 4543297.0,
+      "epoch": 0.30848329048843187,
+      "grad_norm": 6.237403081050188,
+      "learning_rate": 9.911089049328113e-06,
+      "loss": 1.2926,
+      "num_tokens": 11690475.0,
       "step": 15
     },
     {
-      "epoch": 0.38095238095238093,
-      "grad_norm": 4.123366988000153,
-      "learning_rate": 2.9896667579181855e-05,
-      "loss": 1.6285,
-      "num_tokens": 4829061.0,
+      "epoch": 0.32904884318766064,
+      "grad_norm": 3.3556394601847654,
+      "learning_rate": 9.890318491986282e-06,
+      "loss": 1.3084,
+      "num_tokens": 12454353.0,
       "step": 16
     },
     {
-      "epoch": 0.40476190476190477,
-      "grad_norm": 11.611181067813334,
-      "learning_rate": 2.9869264283033827e-05,
-      "loss": 1.5812,
-      "num_tokens": 5089606.0,
+      "epoch": 0.3496143958868895,
+      "grad_norm": 3.4957166279626715,
+      "learning_rate": 9.867399108056054e-06,
+      "loss": 1.2421,
+      "num_tokens": 13239119.0,
       "step": 17
     },
     {
-      "epoch": 0.42857142857142855,
-      "grad_norm": 2.2261338342365398,
-      "learning_rate": 2.9838659096130715e-05,
-      "loss": 1.5597,
-      "num_tokens": 5390312.0,
+      "epoch": 0.37017994858611825,
+      "grad_norm": 2.531289170109121,
+      "learning_rate": 9.842342115352647e-06,
+      "loss": 1.2823,
+      "num_tokens": 14021595.0,
       "step": 18
     },
     {
-      "epoch": 0.4523809523809524,
-      "grad_norm": 2.1851134690595377,
-      "learning_rate": 2.9804859348304814e-05,
-      "loss": 1.5541,
-      "num_tokens": 5637391.0,
+      "epoch": 0.390745501285347,
+      "grad_norm": 1.6632412232515026,
+      "learning_rate": 9.815159777936682e-06,
+      "loss": 1.2517,
+      "num_tokens": 14783289.0,
       "step": 19
     },
     {
-      "epoch": 0.47619047619047616,
-      "grad_norm": 19.42274392919098,
-      "learning_rate": 2.976787313447427e-05,
-      "loss": 1.5582,
-      "num_tokens": 5900436.0,
+      "epoch": 0.41131105398457585,
+      "grad_norm": 5.91844570481689,
+      "learning_rate": 9.785865400111593e-06,
+      "loss": 1.2599,
+      "num_tokens": 15628684.0,
       "step": 20
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 2.743718856278709,
-      "learning_rate": 2.9727709312704392e-05,
-      "loss": 1.5783,
-      "num_tokens": 6208557.0,
+      "epoch": 0.4318766066838046,
+      "grad_norm": 7.193576968428877,
+      "learning_rate": 9.754473319911866e-06,
+      "loss": 1.252,
+      "num_tokens": 16396270.0,
       "step": 21
     },
     {
-      "epoch": 0.5238095238095238,
-      "grad_norm": 1.270567239695732,
-      "learning_rate": 2.968437750208617e-05,
-      "loss": 1.571,
-      "num_tokens": 6508211.0,
+      "epoch": 0.4524421593830334,
+      "grad_norm": 5.504834853944556,
+      "learning_rate": 9.720998902085354e-06,
+      "loss": 1.2151,
+      "num_tokens": 17186501.0,
       "step": 22
     },
     {
-      "epoch": 0.5476190476190477,
-      "grad_norm": 1.3295343234029675,
-      "learning_rate": 2.963788808043254e-05,
-      "loss": 1.5558,
-      "num_tokens": 6815212.0,
+      "epoch": 0.4730077120822622,
+      "grad_norm": 1.264259592512111,
+      "learning_rate": 9.685458530573035e-06,
+      "loss": 1.2377,
+      "num_tokens": 17997203.0,
       "step": 23
     },
     {
-      "epoch": 0.5714285714285714,
-      "grad_norm": 1.0645487974082066,
-      "learning_rate": 2.9588252181792933e-05,
-      "loss": 1.5579,
-      "num_tokens": 7165341.0,
+      "epoch": 0.493573264781491,
+      "grad_norm": 3.713191417559263,
+      "learning_rate": 9.647869600489954e-06,
+      "loss": 1.2181,
+      "num_tokens": 18817970.0,
       "step": 24
     },
     {
-      "epoch": 0.5952380952380952,
-      "grad_norm": 0.8618665557232967,
-      "learning_rate": 2.953548169378672e-05,
-      "loss": 1.5311,
-      "num_tokens": 7446406.0,
+      "epoch": 0.5141388174807198,
+      "grad_norm": 3.1765726456521626,
+      "learning_rate": 9.60825050961125e-06,
+      "loss": 1.227,
+      "num_tokens": 19633513.0,
       "step": 25
     },
     {
-      "epoch": 0.6190476190476191,
-      "grad_norm": 0.8833172294647831,
-      "learning_rate": 2.9479589254756145e-05,
-      "loss": 1.5395,
-      "num_tokens": 7716401.0,
+      "epoch": 0.5347043701799485,
+      "grad_norm": 1.1060798252537263,
+      "learning_rate": 9.566620649367418e-06,
+      "loss": 1.2185,
+      "num_tokens": 20390710.0,
       "step": 26
     },
     {
-      "epoch": 0.6428571428571429,
-      "grad_norm": 0.8730876262810777,
-      "learning_rate": 2.9420588250739514e-05,
-      "loss": 1.5083,
-      "num_tokens": 7991093.0,
+      "epoch": 0.5552699228791774,
+      "grad_norm": 2.8920027077334307,
+      "learning_rate": 9.523000395353242e-06,
+      "loss": 1.2012,
+      "num_tokens": 21154825.0,
       "step": 27
     },
     {
-      "epoch": 0.6666666666666666,
-      "grad_norm": 9.830926234059927,
-      "learning_rate": 2.9358492812265286e-05,
-      "loss": 1.562,
-      "num_tokens": 8326807.0,
+      "epoch": 0.5758354755784062,
+      "grad_norm": 3.0008962834932205,
+      "learning_rate": 9.477411097355025e-06,
+      "loss": 1.1862,
+      "num_tokens": 21973044.0,
       "step": 28
     },
     {
-      "epoch": 0.6904761904761905,
-      "grad_norm": 3.439648574877625,
-      "learning_rate": 2.9293317810967836e-05,
-      "loss": 1.5434,
-      "num_tokens": 8608746.0,
+      "epoch": 0.596401028277635,
+      "grad_norm": 0.8612249291394044,
+      "learning_rate": 9.429875068901018e-06,
+      "loss": 1.2349,
+      "num_tokens": 22763384.0,
       "step": 29
     },
     {
-      "epoch": 0.7142857142857143,
-      "grad_norm": 1.4403398373980578,
-      "learning_rate": 2.9225078856025776e-05,
-      "loss": 1.55,
-      "num_tokens": 8887444.0,
+      "epoch": 0.6169665809768637,
+      "grad_norm": 1.5747643898508015,
+      "learning_rate": 9.380415576340127e-06,
+      "loss": 1.2552,
+      "num_tokens": 23624743.0,
       "step": 30
     },
     {
-      "epoch": 0.7380952380952381,
-      "grad_norm": 0.9256924750760002,
-      "learning_rate": 2.915379229042361e-05,
-      "loss": 1.5387,
-      "num_tokens": 9158119.0,
+      "epoch": 0.6375321336760925,
+      "grad_norm": 1.0439033686337735,
+      "learning_rate": 9.329056827454279e-06,
+      "loss": 1.2262,
+      "num_tokens": 24445122.0,
       "step": 31
     },
     {
-      "epoch": 0.7619047619047619,
-      "grad_norm": 0.9693333317012788,
-      "learning_rate": 2.9079475187037645e-05,
-      "loss": 1.5525,
-      "num_tokens": 9455380.0,
+      "epoch": 0.6580976863753213,
+      "grad_norm": 1.1431587522720466,
+      "learning_rate": 9.275823959610019e-06,
+      "loss": 1.1802,
+      "num_tokens": 25269608.0,
       "step": 32
     },
     {
-      "epoch": 0.7857142857142857,
-      "grad_norm": 4.418422539715986,
-      "learning_rate": 2.900214534454708e-05,
-      "loss": 1.562,
-      "num_tokens": 9819963.0,
+      "epoch": 0.6786632390745502,
+      "grad_norm": 0.9682411879222097,
+      "learning_rate": 9.22074302745509e-06,
+      "loss": 1.2086,
+      "num_tokens": 26077592.0,
       "step": 33
     },
     {
-      "epoch": 0.8095238095238095,
-      "grad_norm": 1.4976653598911862,
-      "learning_rate": 2.8921821283171333e-05,
-      "loss": 1.5134,
-      "num_tokens": 10120795.0,
+      "epoch": 0.699228791773779,
+      "grad_norm": 2.0072489147760595,
+      "learning_rate": 9.163840990166085e-06,
+      "loss": 1.2066,
+      "num_tokens": 26890903.0,
       "step": 34
     },
     {
-      "epoch": 0.8333333333333334,
-      "grad_norm": 0.9155113033240181,
-      "learning_rate": 2.883852224023446e-05,
-      "loss": 1.5485,
-      "num_tokens": 10448963.0,
+      "epoch": 0.7197943444730077,
+      "grad_norm": 1.4975825224817174,
+      "learning_rate": 9.105145698253367e-06,
+      "loss": 1.1348,
+      "num_tokens": 27707185.0,
       "step": 35
     },
     {
-      "epoch": 0.8571428571428571,
-      "grad_norm": 1.078789553362349,
-      "learning_rate": 2.875226816555792e-05,
-      "loss": 1.5239,
-      "num_tokens": 10726463.0,
+      "epoch": 0.7403598971722365,
+      "grad_norm": 0.9880199998455155,
+      "learning_rate": 9.044685879929734e-06,
+      "loss": 1.2621,
+      "num_tokens": 28536664.0,
       "step": 36
     },
     {
-      "epoch": 0.8809523809523809,
-      "grad_norm": 0.8912072449466584,
-      "learning_rate": 2.8663079716682657e-05,
-      "loss": 1.5328,
-      "num_tokens": 11034561.0,
+      "epoch": 0.7609254498714653,
+      "grad_norm": 2.668219422837114,
+      "learning_rate": 8.982491127049482e-06,
+      "loss": 1.1934,
+      "num_tokens": 29362087.0,
       "step": 37
     },
     {
-      "epoch": 0.9047619047619048,
-      "grad_norm": 0.8137040759182133,
-      "learning_rate": 2.8570978253921695e-05,
-      "loss": 1.5173,
-      "num_tokens": 11357514.0,
+      "epoch": 0.781491002570694,
+      "grad_norm": 1.1933259433753431,
+      "learning_rate": 8.918591880624783e-06,
+      "loss": 1.1959,
+      "num_tokens": 30117101.0,
       "step": 38
     },
     {
-      "epoch": 0.9285714285714286,
-      "grad_norm": 0.9066560545161888,
-      "learning_rate": 2.8475985835244395e-05,
-      "loss": 1.5032,
-      "num_tokens": 11618431.0,
+      "epoch": 0.8020565552699229,
+      "grad_norm": 3.820068913066678,
+      "learning_rate": 8.85301941592643e-06,
+      "loss": 1.1953,
+      "num_tokens": 30947725.0,
       "step": 39
     },
     {
-      "epoch": 0.9523809523809523,
-      "grad_norm": 0.789402718151679,
-      "learning_rate": 2.8378125210993705e-05,
-      "loss": 1.5219,
-      "num_tokens": 11938221.0,
+      "epoch": 0.8226221079691517,
+      "grad_norm": 3.2235005439130355,
+      "learning_rate": 8.785805827176256e-06,
+      "loss": 1.1483,
+      "num_tokens": 31745760.0,
       "step": 40
     },
     {
-      "epoch": 0.9761904761904762,
-      "grad_norm": 0.984207528765468,
-      "learning_rate": 2.8277419818437477e-05,
-      "loss": 1.536,
-      "num_tokens": 12260374.0,
+      "epoch": 0.8431876606683805,
+      "grad_norm": 1.3876292035353346,
+      "learning_rate": 8.716984011838731e-06,
+      "loss": 1.1777,
+      "num_tokens": 32569405.0,
       "step": 41
     },
     {
-      "epoch": 1.0,
-      "grad_norm": 0.7141475421535004,
-      "learning_rate": 2.817389377615535e-05,
-      "loss": 1.5138,
-      "num_tokens": 12557020.0,
+      "epoch": 0.8637532133676092,
+      "grad_norm": 1.8637438280292022,
+      "learning_rate": 8.646587654519413e-06,
+      "loss": 1.1803,
+      "num_tokens": 33393264.0,
       "step": 42
     },
     {
-      "epoch": 1.0238095238095237,
-      "grad_norm": 0.9363198060842514,
-      "learning_rate": 2.8067571878262455e-05,
-      "loss": 1.3474,
-      "num_tokens": 12816238.0,
+      "epoch": 0.884318766066838,
+      "grad_norm": 1.195519553673873,
+      "learning_rate": 8.574651210478121e-06,
+      "loss": 1.211,
+      "num_tokens": 34224768.0,
       "step": 43
     },
     {
-      "epoch": 1.0476190476190477,
-      "grad_norm": 0.6926582015960785,
-      "learning_rate": 2.7958479588471274e-05,
-      "loss": 1.3267,
-      "num_tokens": 13122015.0,
+      "epoch": 0.9048843187660668,
+      "grad_norm": 0.8382740981864941,
+      "learning_rate": 8.501209888764928e-06,
+      "loss": 1.1545,
+      "num_tokens": 35047872.0,
       "step": 44
     },
     {
-      "epoch": 1.0714285714285714,
-      "grad_norm": 0.9454215939995514,
-      "learning_rate": 2.784664303399321e-05,
-      "loss": 1.339,
-      "num_tokens": 13397622.0,
+      "epoch": 0.9254498714652957,
+      "grad_norm": 1.760524895981456,
+      "learning_rate": 8.426299634987217e-06,
+      "loss": 1.1532,
+      "num_tokens": 35875180.0,
       "step": 45
     },
     {
-      "epoch": 1.0952380952380953,
-      "grad_norm": 0.7721777219877292,
-      "learning_rate": 2.7732088999281182e-05,
-      "loss": 1.3203,
-      "num_tokens": 13699593.0,
+      "epoch": 0.9460154241645244,
+      "grad_norm": 3.878430468083381,
+      "learning_rate": 8.349957113716213e-06,
+      "loss": 1.1812,
+      "num_tokens": 36683234.0,
       "step": 46
     },
     {
-      "epoch": 1.119047619047619,
-      "grad_norm": 0.8317013732600135,
-      "learning_rate": 2.7614844919614878e-05,
-      "loss": 1.3393,
-      "num_tokens": 14012389.0,
+      "epoch": 0.9665809768637532,
+      "grad_norm": 3.037524297964966,
+      "learning_rate": 8.272219690541639e-06,
+      "loss": 1.2384,
+      "num_tokens": 37448006.0,
       "step": 47
     },
     {
-      "epoch": 1.1428571428571428,
-      "grad_norm": 0.8420079333202011,
-      "learning_rate": 2.749493887453007e-05,
-      "loss": 1.321,
-      "num_tokens": 14310791.0,
+      "epoch": 0.987146529562982,
+      "grad_norm": 1.4759778040121787,
+      "learning_rate": 8.19312541378326e-06,
+      "loss": 1.1638,
+      "num_tokens": 38273059.0,
       "step": 48
     },
     {
-      "epoch": 1.1666666666666667,
-      "grad_norm": 0.808825236941867,
-      "learning_rate": 2.7372399581093692e-05,
-      "loss": 1.3444,
-      "num_tokens": 14658237.0,
+      "epoch": 1.0,
+      "grad_norm": 1.4759778040121787,
+      "learning_rate": 8.112712995868244e-06,
+      "loss": 1.181,
+      "num_tokens": 38793091.0,
       "step": 49
     },
     {
-      "epoch": 1.1904761904761905,
-      "grad_norm": 0.9423947902880564,
-      "learning_rate": 2.724725638702619e-05,
-      "loss": 1.3208,
-      "num_tokens": 14932524.0,
+      "epoch": 1.0205655526992288,
+      "grad_norm": 3.8386221295253287,
+      "learning_rate": 8.031021794383513e-06,
+      "loss": 1.1309,
+      "num_tokens": 39621652.0,
       "step": 50
     },
     {
-      "epoch": 1.2142857142857142,
-      "grad_norm": 0.8920368672000416,
-      "learning_rate": 2.7119539263672863e-05,
-      "loss": 1.3663,
-      "num_tokens": 15221313.0,
+      "epoch": 1.0411311053984575,
+      "grad_norm": 3.3524575596264032,
+      "learning_rate": 7.948091792812296e-06,
+      "loss": 1.0887,
+      "num_tokens": 40386138.0,
+      "step": 51
+    },
+    {
+      "epoch": 1.0411311053984575,
+      "eval_loss": 0.9421821236610413,
+      "eval_num_tokens": 40386138.0,
+      "eval_runtime": 31.2484,
+      "eval_samples_per_second": 44.226,
+      "eval_steps_per_second": 5.536,
       "step": 51
     },
     {
-      "epoch": 1.2380952380952381,
-      "grad_norm": 1.037715092791166,
-      "learning_rate": 2.698927879882581e-05,
-      "loss": 1.3133,
-      "num_tokens": 15503438.0,
+      "epoch": 1.0616966580976863,
+      "grad_norm": 2.5943580288080614,
+      "learning_rate": 7.863963580964344e-06,
+      "loss": 1.2202,
+      "num_tokens": 41267887.0,
       "step": 52
     },
     {
-      "epoch": 1.2619047619047619,
-      "grad_norm": 0.9566394653382752,
-      "learning_rate": 2.6856506189398304e-05,
-      "loss": 1.3418,
-      "num_tokens": 15812209.0,
+      "epoch": 1.082262210796915,
+      "grad_norm": 1.0418412079638837,
+      "learning_rate": 7.778678335109399e-06,
+      "loss": 1.1243,
+      "num_tokens": 42061937.0,
       "step": 53
     },
     {
-      "epoch": 1.2857142857142856,
-      "grad_norm": 0.7773982206501678,
-      "learning_rate": 2.672125323395319e-05,
-      "loss": 1.3214,
-      "num_tokens": 16133281.0,
+      "epoch": 1.1028277634961439,
+      "grad_norm": 1.7618278256873732,
+      "learning_rate": 7.692277797823585e-06,
+      "loss": 1.0871,
+      "num_tokens": 42839063.0,
       "step": 54
     },
     {
-      "epoch": 1.3095238095238095,
-      "grad_norm": 0.7309052587570527,
-      "learning_rate": 2.6583552325087277e-05,
-      "loss": 1.3106,
-      "num_tokens": 16469546.0,
+      "epoch": 1.1233933161953726,
+      "grad_norm": 1.6933944438474042,
+      "learning_rate": 7.6048042575586675e-06,
+      "loss": 1.1047,
+      "num_tokens": 43584262.0,
       "step": 55
     },
     {
-      "epoch": 1.3333333333333333,
-      "grad_norm": 0.731975052614095,
-      "learning_rate": 2.644343644167344e-05,
-      "loss": 1.315,
-      "num_tokens": 16740412.0,
+      "epoch": 1.1439588688946016,
+      "grad_norm": 1.0074442814131914,
+      "learning_rate": 7.516300527944104e-06,
+      "loss": 1.0769,
+      "num_tokens": 44448960.0,
       "step": 56
     },
     {
-      "epoch": 1.3571428571428572,
-      "grad_norm": 0.888533248470318,
-      "learning_rate": 2.6300939140962265e-05,
-      "loss": 1.3347,
-      "num_tokens": 17069209.0,
+      "epoch": 1.1645244215938304,
+      "grad_norm": 1.614591216728204,
+      "learning_rate": 7.426809926832049e-06,
+      "loss": 1.1515,
+      "num_tokens": 45212858.0,
       "step": 57
     },
     {
-      "epoch": 1.380952380952381,
-      "grad_norm": 0.8174152731954014,
-      "learning_rate": 2.615609455054523e-05,
-      "loss": 1.3252,
-      "num_tokens": 17390230.0,
+      "epoch": 1.1850899742930592,
+      "grad_norm": 1.2719018890548137,
+      "learning_rate": 7.336376255095592e-06,
+      "loss": 1.1,
+      "num_tokens": 45985576.0,
       "step": 58
     },
     {
-      "epoch": 1.4047619047619047,
-      "grad_norm": 0.7703407162710431,
-      "learning_rate": 2.6008937360181247e-05,
-      "loss": 1.333,
-      "num_tokens": 17751056.0,
+      "epoch": 1.205655526992288,
+      "grad_norm": 0.7493358958267934,
+      "learning_rate": 7.24504377519056e-06,
+      "loss": 1.1753,
+      "num_tokens": 46750204.0,
       "step": 59
     },
     {
-      "epoch": 1.4285714285714286,
-      "grad_norm": 0.8093422656169592,
-      "learning_rate": 2.5859502813488634e-05,
-      "loss": 1.2688,
-      "num_tokens": 18029513.0,
+      "epoch": 1.2262210796915167,
+      "grad_norm": 2.1100937351617874,
+      "learning_rate": 7.152857189491406e-06,
+      "loss": 1.1719,
+      "num_tokens": 47534251.0,
       "step": 60
     },
     {
-      "epoch": 1.4523809523809523,
-      "grad_norm": 0.7448128748019907,
-      "learning_rate": 2.570782669950435e-05,
-      "loss": 1.2866,
-      "num_tokens": 18342199.0,
+      "epoch": 1.2467866323907455,
+      "grad_norm": 2.292323991944126,
+      "learning_rate": 7.059861618411789e-06,
+      "loss": 1.0608,
+      "num_tokens": 48344753.0,
       "step": 61
     },
     {
-      "epoch": 1.4761904761904763,
-      "grad_norm": 0.797247555706446,
-      "learning_rate": 2.5553945344112646e-05,
-      "loss": 1.3474,
-      "num_tokens": 18654129.0,
+      "epoch": 1.2673521850899743,
+      "grad_norm": 0.8232509355203044,
+      "learning_rate": 6.966102578320531e-06,
+      "loss": 1.123,
+      "num_tokens": 49162942.0,
       "step": 62
     },
     {
-      "epoch": 1.5,
-      "grad_norm": 0.8270748581084111,
-      "learning_rate": 2.539789560134521e-05,
-      "loss": 1.3099,
-      "num_tokens": 18959464.0,
+      "epoch": 1.287917737789203,
+      "grad_norm": 3.585688770272051,
+      "learning_rate": 6.87162595926379e-06,
+      "loss": 1.1799,
+      "num_tokens": 49945314.0,
       "step": 63
     },
     {
-      "epoch": 1.5238095238095237,
-      "grad_norm": 0.813301315394994,
-      "learning_rate": 2.5239714844554674e-05,
-      "loss": 1.3041,
-      "num_tokens": 19214541.0,
+      "epoch": 1.3084832904884318,
+      "grad_norm": 4.4862385716926605,
+      "learning_rate": 6.776478002504335e-06,
+      "loss": 1.1507,
+      "num_tokens": 50765754.0,
       "step": 64
     },
     {
-      "epoch": 1.5476190476190477,
-      "grad_norm": 0.951995926855497,
-      "learning_rate": 2.5079440957463894e-05,
-      "loss": 1.2476,
-      "num_tokens": 19491950.0,
+      "epoch": 1.3290488431876606,
+      "grad_norm": 2.963644902210829,
+      "learning_rate": 6.680705277888907e-06,
+      "loss": 1.1385,
+      "num_tokens": 51580961.0,
       "step": 65
     },
     {
-      "epoch": 1.5714285714285714,
-      "grad_norm": 0.7172356466200617,
-      "learning_rate": 2.4917112325092904e-05,
-      "loss": 1.2979,
-      "num_tokens": 19789279.0,
+      "epoch": 1.3496143958868894,
+      "grad_norm": 0.9625449299027389,
+      "learning_rate": 6.584354661054765e-06,
+      "loss": 1.0606,
+      "num_tokens": 52327855.0,
       "step": 66
     },
     {
-      "epoch": 1.5952380952380953,
-      "grad_norm": 0.842364452878783,
-      "learning_rate": 2.475276782456585e-05,
-      "loss": 1.2964,
-      "num_tokens": 20050385.0,
+      "epoch": 1.3701799485861184,
+      "grad_norm": 3.402724405424802,
+      "learning_rate": 6.487473310486574e-06,
+      "loss": 1.1162,
+      "num_tokens": 53102088.0,
       "step": 67
     },
     {
-      "epoch": 1.619047619047619,
-      "grad_norm": 0.8775202665218085,
-      "learning_rate": 2.4586446815800056e-05,
-      "loss": 1.2306,
-      "num_tokens": 20293468.0,
+      "epoch": 1.390745501285347,
+      "grad_norm": 1.9902964062133557,
+      "learning_rate": 6.390108644434828e-06,
+      "loss": 1.0108,
+      "num_tokens": 53877061.0,
       "step": 68
     },
     {
-      "epoch": 1.6428571428571428,
-      "grad_norm": 0.8077565121484018,
-      "learning_rate": 2.441818913207947e-05,
-      "loss": 1.2124,
-      "num_tokens": 20548563.0,
+      "epoch": 1.411311053984576,
+      "grad_norm": 0.8883185890120039,
+      "learning_rate": 6.292308317707163e-06,
+      "loss": 1.1061,
+      "num_tokens": 54680526.0,
       "step": 69
     },
     {
-      "epoch": 1.6666666666666665,
-      "grad_norm": 0.7891520078797498,
-      "learning_rate": 2.4248035070514732e-05,
-      "loss": 1.31,
-      "num_tokens": 20899914.0,
+      "epoch": 1.4318766066838047,
+      "grad_norm": 3.1753697126676976,
+      "learning_rate": 6.19412019834386e-06,
+      "loss": 1.1154,
+      "num_tokens": 55519612.0,
       "step": 70
     },
     {
-      "epoch": 1.6904761904761905,
-      "grad_norm": 0.8935642914804406,
-      "learning_rate": 2.4076025382392166e-05,
-      "loss": 1.2904,
-      "num_tokens": 21207831.0,
+      "epoch": 1.4524421593830334,
+      "grad_norm": 1.8774651070058619,
+      "learning_rate": 6.095592344189002e-06,
+      "loss": 1.0765,
+      "num_tokens": 56336361.0,
       "step": 71
     },
     {
-      "epoch": 1.7142857142857144,
-      "grad_norm": 0.7791700089762551,
-      "learning_rate": 2.3902201263413968e-05,
-      "loss": 1.3009,
-      "num_tokens": 21535523.0,
+      "epoch": 1.4730077120822622,
+      "grad_norm": 0.7156088232150171,
+      "learning_rate": 5.996772979368715e-06,
+      "loss": 1.1009,
+      "num_tokens": 57166304.0,
       "step": 72
     },
     {
-      "epoch": 1.7380952380952381,
-      "grad_norm": 0.6803499303828192,
-      "learning_rate": 2.372660434383203e-05,
-      "loss": 1.3084,
-      "num_tokens": 21866131.0,
+      "epoch": 1.493573264781491,
+      "grad_norm": 1.594991892592125,
+      "learning_rate": 5.897710470688042e-06,
+      "loss": 1.0415,
+      "num_tokens": 57940003.0,
       "step": 73
     },
     {
-      "epoch": 1.7619047619047619,
-      "grad_norm": 0.8712043390870515,
-      "learning_rate": 2.3549276678477605e-05,
-      "loss": 1.2974,
-      "num_tokens": 22201523.0,
+      "epoch": 1.5141388174807198,
+      "grad_norm": 1.2034191805379335,
+      "learning_rate": 5.798453303957968e-06,
+      "loss": 1.0978,
+      "num_tokens": 58721507.0,
       "step": 74
     },
     {
-      "epoch": 1.7857142857142856,
-      "grad_norm": 0.8755800517621928,
-      "learning_rate": 2.337026073668934e-05,
-      "loss": 1.2914,
-      "num_tokens": 22488112.0,
+      "epoch": 1.5347043701799485,
+      "grad_norm": 2.43542768552457,
+      "learning_rate": 5.6990500602642056e-06,
+      "loss": 1.1483,
+      "num_tokens": 59499744.0,
       "step": 75
     },
     {
-      "epoch": 1.8095238095238095,
-      "grad_norm": 0.6269216466100511,
-      "learning_rate": 2.3189599392142027e-05,
-      "loss": 1.3065,
-      "num_tokens": 22813071.0,
+      "epoch": 1.5552699228791775,
+      "grad_norm": 1.0836491203509007,
+      "learning_rate": 5.5995493921893415e-06,
+      "loss": 1.1097,
+      "num_tokens": 60318482.0,
       "step": 76
     },
     {
-      "epoch": 1.8333333333333335,
-      "grad_norm": 0.8732441594396502,
-      "learning_rate": 2.3007335912578496e-05,
-      "loss": 1.2387,
-      "num_tokens": 23107173.0,
+      "epoch": 1.575835475578406,
+      "grad_norm": 0.7098795195940786,
+      "learning_rate": 5.500000000000001e-06,
+      "loss": 1.0371,
+      "num_tokens": 61070625.0,
       "step": 77
     },
     {
-      "epoch": 1.8571428571428572,
-      "grad_norm": 0.8321205173092046,
-      "learning_rate": 2.282351394944717e-05,
-      "loss": 1.3084,
-      "num_tokens": 23422105.0,
+      "epoch": 1.596401028277635,
+      "grad_norm": 0.7002726718932227,
+      "learning_rate": 5.40045060781066e-06,
+      "loss": 1.1046,
+      "num_tokens": 61874321.0,
       "step": 78
     },
     {
-      "epoch": 1.880952380952381,
-      "grad_norm": 0.6201826285098905,
-      "learning_rate": 2.263817752744767e-05,
-      "loss": 1.2937,
-      "num_tokens": 23788463.0,
+      "epoch": 1.6169665809768636,
+      "grad_norm": 0.5013188744398371,
+      "learning_rate": 5.300949939735797e-06,
+      "loss": 1.0814,
+      "num_tokens": 62747678.0,
       "step": 79
     },
     {
-      "epoch": 1.9047619047619047,
-      "grad_norm": 0.864663931903862,
-      "learning_rate": 2.2451371033987086e-05,
-      "loss": 1.2315,
-      "num_tokens": 24016057.0,
+      "epoch": 1.6375321336760926,
+      "grad_norm": 0.930487154531265,
+      "learning_rate": 5.201546696042033e-06,
+      "loss": 1.1056,
+      "num_tokens": 63600515.0,
       "step": 80
     },
     {
-      "epoch": 1.9285714285714286,
-      "grad_norm": 0.8151031466504542,
-      "learning_rate": 2.226313920854934e-05,
-      "loss": 1.2065,
-      "num_tokens": 24286443.0,
+      "epoch": 1.6580976863753212,
+      "grad_norm": 0.5762398846789,
+      "learning_rate": 5.10228952931196e-06,
+      "loss": 1.1254,
+      "num_tokens": 64367564.0,
       "step": 81
     },
     {
-      "epoch": 1.9523809523809523,
-      "grad_norm": 0.69377209773933,
-      "learning_rate": 2.207352713198024e-05,
-      "loss": 1.2522,
-      "num_tokens": 24564564.0,
+      "epoch": 1.6786632390745502,
+      "grad_norm": 0.4494307449174887,
+      "learning_rate": 5.003227020631287e-06,
+      "loss": 1.067,
+      "num_tokens": 65129369.0,
       "step": 82
     },
     {
-      "epoch": 1.9761904761904763,
-      "grad_norm": 0.7749074972232748,
-      "learning_rate": 2.1882580215690765e-05,
-      "loss": 1.3075,
-      "num_tokens": 24856761.0,
+      "epoch": 1.699228791773779,
+      "grad_norm": 0.5249083910569988,
+      "learning_rate": 4.904407655811e-06,
+      "loss": 1.0899,
+      "num_tokens": 65908690.0,
       "step": 83
     },
     {
-      "epoch": 2.0,
-      "grad_norm": 0.836947163785287,
-      "learning_rate": 2.169034419078124e-05,
-      "loss": 1.2472,
-      "num_tokens": 25119155.0,
+      "epoch": 1.7197943444730077,
+      "grad_norm": 1.20880116726979,
+      "learning_rate": 4.80587980165614e-06,
+      "loss": 1.1388,
+      "num_tokens": 66729195.0,
       "step": 84
     },
     {
-      "epoch": 2.0238095238095237,
-      "grad_norm": 1.302298749181937,
-      "learning_rate": 2.1496865097088846e-05,
-      "loss": 1.0524,
-      "num_tokens": 25461378.0,
+      "epoch": 1.7403598971722365,
+      "grad_norm": 1.057560348088788,
+      "learning_rate": 4.707691682292838e-06,
+      "loss": 1.1105,
+      "num_tokens": 67475591.0,
       "step": 85
     },
     {
-      "epoch": 2.0476190476190474,
-      "grad_norm": 0.9938997123428022,
-      "learning_rate": 2.130218927216128e-05,
-      "loss": 0.9867,
-      "num_tokens": 25757738.0,
+      "epoch": 1.7609254498714653,
+      "grad_norm": 0.6863002803853495,
+      "learning_rate": 4.609891355565172e-06,
+      "loss": 1.0767,
+      "num_tokens": 68256417.0,
       "step": 86
     },
     {
-      "epoch": 2.0714285714285716,
-      "grad_norm": 2.239246853283492,
-      "learning_rate": 2.110636334015907e-05,
-      "loss": 0.9849,
-      "num_tokens": 26050090.0,
+      "epoch": 1.781491002570694,
+      "grad_norm": 0.6819115141106705,
+      "learning_rate": 4.5125266895134275e-06,
+      "loss": 1.0941,
+      "num_tokens": 69028462.0,
       "step": 87
     },
     {
-      "epoch": 2.0952380952380953,
-      "grad_norm": 2.255240802193634,
-      "learning_rate": 2.0909434200689264e-05,
-      "loss": 1.0166,
-      "num_tokens": 26323746.0,
+      "epoch": 1.8020565552699228,
+      "grad_norm": 1.4347921037396176,
+      "learning_rate": 4.415645338945236e-06,
+      "loss": 1.091,
+      "num_tokens": 69808820.0,
       "step": 88
     },
     {
-      "epoch": 2.119047619047619,
-      "grad_norm": 1.0279577082424196,
-      "learning_rate": 2.0711449017573122e-05,
-      "loss": 1.016,
-      "num_tokens": 26607004.0,
+      "epoch": 1.8226221079691518,
+      "grad_norm": 1.4693715473624118,
+      "learning_rate": 4.3192947221110956e-06,
+      "loss": 1.0714,
+      "num_tokens": 70645343.0,
       "step": 89
     },
     {
-      "epoch": 2.142857142857143,
-      "grad_norm": 0.8902435973371925,
-      "learning_rate": 2.0512455207550557e-05,
-      "loss": 0.9812,
-      "num_tokens": 26898427.0,
+      "epoch": 1.8431876606683804,
+      "grad_norm": 0.6582970773448227,
+      "learning_rate": 4.223521997495665e-06,
+      "loss": 1.0957,
+      "num_tokens": 71458144.0,
       "step": 90
     },
     {
-      "epoch": 2.1666666666666665,
-      "grad_norm": 0.9183593054714583,
-      "learning_rate": 2.0312500428924023e-05,
-      "loss": 1.0121,
-      "num_tokens": 27205357.0,
+      "epoch": 1.8637532133676094,
+      "grad_norm": 1.1825217028469224,
+      "learning_rate": 4.128374040736211e-06,
+      "loss": 1.1132,
+      "num_tokens": 72282385.0,
       "step": 91
     },
     {
-      "epoch": 2.1904761904761907,
-      "grad_norm": 0.9376600266290521,
-      "learning_rate": 2.0111632570144487e-05,
-      "loss": 0.988,
-      "num_tokens": 27496694.0,
+      "epoch": 1.884318766066838,
+      "grad_norm": 1.1298665815485143,
+      "learning_rate": 4.033897421679472e-06,
+      "loss": 1.0389,
+      "num_tokens": 73088829.0,
       "step": 92
     },
     {
-      "epoch": 2.2142857142857144,
-      "grad_norm": 0.7220165463532561,
-      "learning_rate": 1.9909899738342324e-05,
-      "loss": 1.0119,
-      "num_tokens": 27794697.0,
+      "epoch": 1.904884318766067,
+      "grad_norm": 1.3570211954476283,
+      "learning_rate": 3.940138381588214e-06,
+      "loss": 1.1499,
+      "num_tokens": 73938027.0,
       "step": 93
     },
     {
-      "epoch": 2.238095238095238,
-      "grad_norm": 0.9053661036106623,
-      "learning_rate": 1.9707350247805862e-05,
-      "loss": 1.0194,
-      "num_tokens": 28115378.0,
+      "epoch": 1.9254498714652957,
+      "grad_norm": 1.280064760591942,
+      "learning_rate": 3.847142810508596e-06,
+      "loss": 1.0316,
+      "num_tokens": 74712404.0,
       "step": 94
     },
     {
-      "epoch": 2.261904761904762,
-      "grad_norm": 0.8729413169116081,
-      "learning_rate": 1.9504032608410243e-05,
-      "loss": 0.9866,
-      "num_tokens": 28404844.0,
+      "epoch": 1.9460154241645244,
+      "grad_norm": 1.446677445007892,
+      "learning_rate": 3.7549562248094396e-06,
+      "loss": 1.0729,
+      "num_tokens": 75476326.0,
       "step": 95
     },
     {
-      "epoch": 2.2857142857142856,
-      "grad_norm": 0.7627072905875467,
-      "learning_rate": 1.9299995513999512e-05,
-      "loss": 1.0105,
-      "num_tokens": 28717491.0,
+      "epoch": 1.9665809768637532,
+      "grad_norm": 0.7546075726383373,
+      "learning_rate": 3.6636237449044077e-06,
+      "loss": 1.0677,
+      "num_tokens": 76280080.0,
       "step": 96
     },
     {
-      "epoch": 2.3095238095238093,
-      "grad_norm": 0.7347382833732581,
-      "learning_rate": 1.90952878307246e-05,
-      "loss": 0.9667,
-      "num_tokens": 28969175.0,
+      "epoch": 1.987146529562982,
+      "grad_norm": 0.6418441333219697,
+      "learning_rate": 3.5731900731679525e-06,
+      "loss": 1.0859,
+      "num_tokens": 77079865.0,
       "step": 97
     },
     {
-      "epoch": 2.3333333333333335,
-      "grad_norm": 0.8054230888254635,
-      "learning_rate": 1.888995858534005e-05,
-      "loss": 1.0089,
-      "num_tokens": 29278525.0,
+      "epoch": 2.0,
+      "grad_norm": 1.1588829155309626,
+      "learning_rate": 3.483699472055897e-06,
+      "loss": 1.0656,
+      "num_tokens": 77568391.0,
       "step": 98
     },
     {
-      "epoch": 2.357142857142857,
-      "grad_norm": 0.7107149711009891,
-      "learning_rate": 1.8684056953462328e-05,
-      "loss": 0.9881,
-      "num_tokens": 29589990.0,
+      "epoch": 2.020565552699229,
+      "grad_norm": 0.6238842093615174,
+      "learning_rate": 3.3951957424413327e-06,
+      "loss": 1.0429,
+      "num_tokens": 78366813.0,
       "step": 99
     },
     {
-      "epoch": 2.380952380952381,
-      "grad_norm": 0.709536600984241,
-      "learning_rate": 1.8477632247792365e-05,
-      "loss": 0.9676,
-      "num_tokens": 29851176.0,
+      "epoch": 2.0411311053984575,
+      "grad_norm": 0.5553403632072313,
+      "learning_rate": 3.307722202176417e-06,
+      "loss": 1.0947,
+      "num_tokens": 79224595.0,
       "step": 100
     },
     {
-      "epoch": 2.4047619047619047,
-      "grad_norm": 0.7059346761233823,
-      "learning_rate": 1.827073390630542e-05,
-      "loss": 0.9866,
-      "num_tokens": 30150417.0,
+      "epoch": 2.0616966580976865,
+      "grad_norm": 0.5248026371177484,
+      "learning_rate": 3.2213216648906037e-06,
+      "loss": 1.0545,
+      "num_tokens": 80031127.0,
       "step": 101
     },
     {
-      "epoch": 2.4285714285714284,
-      "grad_norm": 0.6475677168167608,
-      "learning_rate": 1.806341148041082e-05,
-      "loss": 0.9679,
-      "num_tokens": 30476868.0,
+      "epoch": 2.082262210796915,
+      "grad_norm": 0.5022337615857637,
+      "learning_rate": 3.136036419035656e-06,
+      "loss": 1.0642,
+      "num_tokens": 80790581.0,
       "step": 102
     },
     {
-      "epoch": 2.4523809523809526,
-      "grad_norm": 0.6574153556079124,
-      "learning_rate": 1.785571462308458e-05,
-      "loss": 0.961,
-      "num_tokens": 30720675.0,
+      "epoch": 2.082262210796915,
+      "eval_loss": 0.8722440600395203,
+      "eval_num_tokens": 80790581.0,
+      "eval_runtime": 30.28,
+      "eval_samples_per_second": 45.641,
+      "eval_steps_per_second": 5.713,
+      "step": 102
+    },
+    {
+      "epoch": 2.102827763496144,
+      "grad_norm": 0.9132791992130541,
+      "learning_rate": 3.051908207187705e-06,
+      "loss": 1.0463,
+      "num_tokens": 81572710.0,
       "step": 103
     },
     {
-      "epoch": 2.4761904761904763,
-      "grad_norm": 0.782684876535666,
-      "learning_rate": 1.764769307697769e-05,
-      "loss": 0.9449,
-      "num_tokens": 30943687.0,
+      "epoch": 2.1233933161953726,
+      "grad_norm": 0.49676184487454156,
+      "learning_rate": 2.9689782056164874e-06,
+      "loss": 1.0363,
+      "num_tokens": 82392421.0,
       "step": 104
     },
     {
-      "epoch": 2.5,
-      "grad_norm": 0.6959657002281912,
-      "learning_rate": 1.7439396662502947e-05,
-      "loss": 0.9588,
-      "num_tokens": 31226585.0,
+      "epoch": 2.1439588688946016,
+      "grad_norm": 0.41641122076638964,
+      "learning_rate": 2.887287004131757e-06,
+      "loss": 1.0129,
+      "num_tokens": 83197018.0,
       "step": 105
     },
     {
-      "epoch": 2.5238095238095237,
-      "grad_norm": 0.6638836519525753,
-      "learning_rate": 1.723087526590314e-05,
-      "loss": 0.9657,
-      "num_tokens": 31536165.0,
+      "epoch": 2.16452442159383,
+      "grad_norm": 0.6305455673369688,
+      "learning_rate": 2.8068745862167423e-06,
+      "loss": 1.0604,
+      "num_tokens": 84017921.0,
       "step": 106
     },
     {
-      "epoch": 2.5476190476190474,
-      "grad_norm": 0.5543755171176452,
-      "learning_rate": 1.702217882730345e-05,
-      "loss": 0.9605,
-      "num_tokens": 31859352.0,
+      "epoch": 2.185089974293059,
+      "grad_norm": 0.45877001918075977,
+      "learning_rate": 2.727780309458361e-06,
+      "loss": 1.0573,
+      "num_tokens": 84789176.0,
       "step": 107
     },
     {
-      "epoch": 2.571428571428571,
-      "grad_norm": 0.7519329941062585,
-      "learning_rate": 1.6813357328751003e-05,
-      "loss": 0.951,
-      "num_tokens": 32141589.0,
+      "epoch": 2.2056555269922877,
+      "grad_norm": 0.6286596726421219,
+      "learning_rate": 2.6500428862837878e-06,
+      "loss": 0.9936,
+      "num_tokens": 85561595.0,
       "step": 108
     },
     {
-      "epoch": 2.5952380952380953,
-      "grad_norm": 0.6384580732849452,
-      "learning_rate": 1.660446078224433e-05,
-      "loss": 0.9641,
-      "num_tokens": 32415823.0,
+      "epoch": 2.2262210796915167,
+      "grad_norm": 0.7376081520239532,
+      "learning_rate": 2.5737003650127834e-06,
+      "loss": 0.9638,
+      "num_tokens": 86334568.0,
       "step": 109
     },
     {
-      "epoch": 2.619047619047619,
-      "grad_norm": 0.6375498198212793,
-      "learning_rate": 1.6395539217755673e-05,
-      "loss": 0.9767,
-      "num_tokens": 32738959.0,
+      "epoch": 2.2467866323907453,
+      "grad_norm": 1.0507755635770515,
+      "learning_rate": 2.498790111235072e-06,
+      "loss": 1.0676,
+      "num_tokens": 87089590.0,
       "step": 110
     },
     {
-      "epoch": 2.642857142857143,
-      "grad_norm": 0.5909380929927559,
-      "learning_rate": 1.6186642671249e-05,
-      "loss": 0.9553,
-      "num_tokens": 33046758.0,
+      "epoch": 2.2673521850899743,
+      "grad_norm": 0.751812284582568,
+      "learning_rate": 2.4253487895218796e-06,
+      "loss": 1.003,
+      "num_tokens": 87870458.0,
       "step": 111
     },
     {
-      "epoch": 2.6666666666666665,
-      "grad_norm": 0.6627084816829975,
-      "learning_rate": 1.597782117269655e-05,
-      "loss": 0.9918,
-      "num_tokens": 33356268.0,
+      "epoch": 2.2879177377892033,
+      "grad_norm": 0.7258055709708501,
+      "learning_rate": 2.353412345480587e-06,
+      "loss": 1.0957,
+      "num_tokens": 88697761.0,
       "step": 112
     },
     {
-      "epoch": 2.6904761904761907,
-      "grad_norm": 0.6209028590481098,
-      "learning_rate": 1.5769124734096862e-05,
-      "loss": 0.9946,
-      "num_tokens": 33669521.0,
+      "epoch": 2.308483290488432,
+      "grad_norm": 0.9702567291939189,
+      "learning_rate": 2.2830159881612695e-06,
+      "loss": 0.9815,
+      "num_tokens": 89498858.0,
       "step": 113
     },
     {
-      "epoch": 2.7142857142857144,
-      "grad_norm": 0.5682420553998002,
-      "learning_rate": 1.5560603337497055e-05,
-      "loss": 0.9809,
-      "num_tokens": 34006046.0,
+      "epoch": 2.329048843187661,
+      "grad_norm": 1.706294287936,
+      "learning_rate": 2.2141941728237467e-06,
+      "loss": 1.02,
+      "num_tokens": 90207780.0,
       "step": 114
     },
     {
-      "epoch": 2.738095238095238,
-      "grad_norm": 0.6102593452284443,
-      "learning_rate": 1.5352306923022314e-05,
-      "loss": 0.9949,
-      "num_tokens": 34329314.0,
+      "epoch": 2.3496143958868894,
+      "grad_norm": 0.7383798216635169,
+      "learning_rate": 2.146980584073573e-06,
+      "loss": 1.0215,
+      "num_tokens": 90929907.0,
       "step": 115
     },
     {
-      "epoch": 2.761904761904762,
-      "grad_norm": 0.5963721089952566,
-      "learning_rate": 1.5144285376915424e-05,
-      "loss": 0.9496,
-      "num_tokens": 34649593.0,
+      "epoch": 2.3701799485861184,
+      "grad_norm": 0.8092138470162481,
+      "learning_rate": 2.081408119375219e-06,
+      "loss": 1.0648,
+      "num_tokens": 91776397.0,
       "step": 116
     },
     {
-      "epoch": 2.7857142857142856,
-      "grad_norm": 0.6334479812477452,
-      "learning_rate": 1.4936588519589182e-05,
-      "loss": 0.9865,
-      "num_tokens": 34968486.0,
+      "epoch": 2.390745501285347,
+      "grad_norm": 0.8621943427931472,
+      "learning_rate": 2.0175088729505206e-06,
+      "loss": 1.0926,
+      "num_tokens": 92513206.0,
       "step": 117
     },
     {
-      "epoch": 2.8095238095238093,
-      "grad_norm": 0.579179313504508,
-      "learning_rate": 1.4729266093694578e-05,
-      "loss": 1.0112,
-      "num_tokens": 35335674.0,
+      "epoch": 2.411311053984576,
+      "grad_norm": 1.787970295069879,
+      "learning_rate": 1.955314120070269e-06,
+      "loss": 0.9965,
+      "num_tokens": 93306104.0,
       "step": 118
     },
     {
-      "epoch": 2.8333333333333335,
-      "grad_norm": 0.6547466538591331,
-      "learning_rate": 1.4522367752207636e-05,
-      "loss": 0.9476,
-      "num_tokens": 35620162.0,
+      "epoch": 2.4318766066838045,
+      "grad_norm": 1.479554722978421,
+      "learning_rate": 1.8948543017466337e-06,
+      "loss": 1.0851,
+      "num_tokens": 94155482.0,
       "step": 119
     },
     {
-      "epoch": 2.857142857142857,
-      "grad_norm": 0.6519356419157716,
-      "learning_rate": 1.4315943046537676e-05,
-      "loss": 0.9574,
-      "num_tokens": 35888853.0,
+      "epoch": 2.4524421593830334,
+      "grad_norm": 0.49335005217781264,
+      "learning_rate": 1.8361590098339168e-06,
+      "loss": 1.037,
+      "num_tokens": 94936066.0,
       "step": 120
     },
     {
-      "epoch": 2.880952380952381,
-      "grad_norm": 0.6056113333547287,
-      "learning_rate": 1.411004141465995e-05,
-      "loss": 0.9453,
-      "num_tokens": 36190378.0,
+      "epoch": 2.4730077120822624,
+      "grad_norm": 0.6721016458518935,
+      "learning_rate": 1.7792569725449114e-06,
+      "loss": 1.0131,
+      "num_tokens": 95818245.0,
       "step": 121
     },
     {
-      "epoch": 2.9047619047619047,
-      "grad_norm": 0.5999985950614662,
-      "learning_rate": 1.3904712169275403e-05,
-      "loss": 0.9878,
-      "num_tokens": 36499565.0,
+      "epoch": 2.493573264781491,
+      "grad_norm": 1.0956037260235203,
+      "learning_rate": 1.724176040389982e-06,
+      "loss": 1.0814,
+      "num_tokens": 96635430.0,
       "step": 122
     },
     {
-      "epoch": 2.928571428571429,
-      "grad_norm": 0.6170461063881166,
-      "learning_rate": 1.3700004486000488e-05,
-      "loss": 1.003,
-      "num_tokens": 36819385.0,
+      "epoch": 2.5141388174807195,
+      "grad_norm": 1.107512998866383,
+      "learning_rate": 1.6709431725457203e-06,
+      "loss": 1.0615,
+      "num_tokens": 97438156.0,
       "step": 123
     },
     {
-      "epoch": 2.9523809523809526,
-      "grad_norm": 0.5609993983165859,
-      "learning_rate": 1.3495967391589758e-05,
-      "loss": 0.9709,
-      "num_tokens": 37126003.0,
+      "epoch": 2.5347043701799485,
+      "grad_norm": 0.5872510671862957,
+      "learning_rate": 1.619584423659875e-06,
+      "loss": 1.042,
+      "num_tokens": 98247321.0,
       "step": 124
     },
     {
-      "epoch": 2.9761904761904763,
-      "grad_norm": 0.686886397394039,
-      "learning_rate": 1.3292649752194144e-05,
-      "loss": 0.9514,
-      "num_tokens": 37385037.0,
+      "epoch": 2.5552699228791775,
+      "grad_norm": 0.9339232970249086,
+      "learning_rate": 1.5701249310989827e-06,
+      "loss": 1.0533,
+      "num_tokens": 99018604.0,
       "step": 125
     },
     {
-      "epoch": 3.0,
-      "grad_norm": 0.6489847174148443,
-      "learning_rate": 1.309010026165768e-05,
-      "loss": 0.9919,
-      "num_tokens": 37681802.0,
+      "epoch": 2.575835475578406,
+      "grad_norm": 0.9907046236471151,
+      "learning_rate": 1.5225889026449754e-06,
+      "loss": 1.0472,
+      "num_tokens": 99814782.0,
       "step": 126
     },
     {
-      "epoch": 3.0238095238095237,
-      "grad_norm": 1.3407025275092483,
-      "learning_rate": 1.288836742985552e-05,
-      "loss": 0.7691,
-      "num_tokens": 37984864.0,
+      "epoch": 2.596401028277635,
+      "grad_norm": 0.42612961753245276,
+      "learning_rate": 1.4769996046467599e-06,
+      "loss": 1.1264,
+      "num_tokens": 100614217.0,
       "step": 127
     },
     {
-      "epoch": 3.0476190476190474,
-      "grad_norm": 1.1240882365427978,
-      "learning_rate": 1.2687499571075978e-05,
-      "loss": 0.7481,
-      "num_tokens": 38255245.0,
+      "epoch": 2.6169665809768636,
+      "grad_norm": 0.9091136187984498,
+      "learning_rate": 1.4333793506325832e-06,
+      "loss": 1.0193,
+      "num_tokens": 101396655.0,
       "step": 128
     },
     {
-      "epoch": 3.0714285714285716,
-      "grad_norm": 0.7591608642049189,
-      "learning_rate": 1.2487544792449443e-05,
-      "loss": 0.7418,
-      "num_tokens": 38525517.0,
+      "epoch": 2.6375321336760926,
+      "grad_norm": 0.7892032837871268,
+      "learning_rate": 1.3917494903887516e-06,
+      "loss": 0.9758,
+      "num_tokens": 102215959.0,
       "step": 129
     },
     {
-      "epoch": 3.0952380952380953,
-      "grad_norm": 1.4687837390641727,
-      "learning_rate": 1.2288550982426879e-05,
-      "loss": 0.7639,
-      "num_tokens": 38849309.0,
+      "epoch": 2.658097686375321,
+      "grad_norm": 0.5128890053331989,
+      "learning_rate": 1.3521303995100479e-06,
+      "loss": 1.057,
+      "num_tokens": 103010668.0,
       "step": 130
     },
     {
-      "epoch": 3.119047619047619,
-      "grad_norm": 1.6813633030275168,
-      "learning_rate": 1.2090565799310738e-05,
-      "loss": 0.7419,
-      "num_tokens": 39220621.0,
+      "epoch": 2.67866323907455,
+      "grad_norm": 1.0590105248836392,
+      "learning_rate": 1.3145414694269671e-06,
+      "loss": 1.0391,
+      "num_tokens": 103845595.0,
       "step": 131
     },
     {
-      "epoch": 3.142857142857143,
-      "grad_norm": 1.5003572460299635,
-      "learning_rate": 1.1893636659840927e-05,
-      "loss": 0.7561,
-      "num_tokens": 39564336.0,
+      "epoch": 2.6992287917737787,
+      "grad_norm": 0.8126843666598889,
+      "learning_rate": 1.2790010979146467e-06,
+      "loss": 0.9984,
+      "num_tokens": 104715999.0,
       "step": 132
     },
     {
-      "epoch": 3.1666666666666665,
-      "grad_norm": 1.0050005240430444,
-      "learning_rate": 1.169781072783872e-05,
-      "loss": 0.7044,
-      "num_tokens": 39847434.0,
+      "epoch": 2.7197943444730077,
+      "grad_norm": 0.47820659260495135,
+      "learning_rate": 1.2455266800881342e-06,
+      "loss": 1.0,
+      "num_tokens": 105488928.0,
       "step": 133
     },
     {
-      "epoch": 3.1904761904761907,
-      "grad_norm": 0.8969397973329072,
-      "learning_rate": 1.1503134902911155e-05,
-      "loss": 0.722,
-      "num_tokens": 40157495.0,
+      "epoch": 2.7403598971722367,
+      "grad_norm": 0.5317897918745331,
+      "learning_rate": 1.2141345998884092e-06,
+      "loss": 1.0498,
+      "num_tokens": 106186987.0,
       "step": 134
     },
     {
-      "epoch": 3.2142857142857144,
-      "grad_norm": 0.8424389883668345,
-      "learning_rate": 1.1309655809218764e-05,
-      "loss": 0.7033,
-      "num_tokens": 40469966.0,
+      "epoch": 2.7609254498714653,
+      "grad_norm": 0.9341792757623867,
+      "learning_rate": 1.1848402220633198e-06,
+      "loss": 0.9566,
+      "num_tokens": 106967242.0,
       "step": 135
     },
     {
-      "epoch": 3.238095238095238,
-      "grad_norm": 0.8450270760954434,
-      "learning_rate": 1.1117419784309236e-05,
-      "loss": 0.7047,
-      "num_tokens": 40771634.0,
+      "epoch": 2.781491002570694,
+      "grad_norm": 1.4588060292984133,
+      "learning_rate": 1.1576578846473558e-06,
+      "loss": 1.0252,
+      "num_tokens": 107817871.0,
       "step": 136
     },
     {
-      "epoch": 3.261904761904762,
-      "grad_norm": 0.7538567142823028,
-      "learning_rate": 1.0926472868019764e-05,
-      "loss": 0.715,
-      "num_tokens": 41087014.0,
+      "epoch": 2.802056555269923,
+      "grad_norm": 0.8215750745057664,
+      "learning_rate": 1.1326008919439476e-06,
+      "loss": 1.0712,
+      "num_tokens": 108629125.0,
       "step": 137
     },
     {
-      "epoch": 3.2857142857142856,
-      "grad_norm": 0.799941449070327,
-      "learning_rate": 1.0736860791450658e-05,
-      "loss": 0.7477,
-      "num_tokens": 41435947.0,
+      "epoch": 2.822622107969152,
+      "grad_norm": 0.38163574145212953,
+      "learning_rate": 1.1096815080137196e-06,
+      "loss": 1.0421,
+      "num_tokens": 109428401.0,
       "step": 138
     },
     {
-      "epoch": 3.3095238095238093,
-      "grad_norm": 0.821321998423862,
-      "learning_rate": 1.0548628966012917e-05,
-      "loss": 0.7057,
-      "num_tokens": 41733553.0,
+      "epoch": 2.8431876606683804,
+      "grad_norm": 1.0563512897683773,
+      "learning_rate": 1.0889109506718873e-06,
+      "loss": 0.961,
+      "num_tokens": 110292149.0,
       "step": 139
     },
     {
-      "epoch": 3.3333333333333335,
-      "grad_norm": 0.8039593601908853,
-      "learning_rate": 1.036182247255233e-05,
-      "loss": 0.6771,
-      "num_tokens": 41998508.0,
+      "epoch": 2.8637532133676094,
+      "grad_norm": 0.6548608506755417,
+      "learning_rate": 1.070299385997735e-06,
+      "loss": 0.9862,
+      "num_tokens": 111130565.0,
       "step": 140
     },
     {
-      "epoch": 3.357142857142857,
-      "grad_norm": 0.7486362398650765,
-      "learning_rate": 1.0176486050552834e-05,
-      "loss": 0.7025,
-      "num_tokens": 42328033.0,
+      "epoch": 2.884318766066838,
+      "grad_norm": 0.7840147082599769,
+      "learning_rate": 1.053855923358862e-06,
+      "loss": 1.101,
+      "num_tokens": 111859971.0,
       "step": 141
     },
     {
-      "epoch": 3.380952380952381,
-      "grad_norm": 0.7435262398434506,
-      "learning_rate": 9.99266408742151e-06,
-      "loss": 0.6721,
-      "num_tokens": 42576679.0,
+      "epoch": 2.904884318766067,
+      "grad_norm": 0.8638836969151383,
+      "learning_rate": 1.0395886109526346e-06,
+      "loss": 1.0626,
+      "num_tokens": 112658866.0,
       "step": 142
     },
     {
-      "epoch": 3.4047619047619047,
-      "grad_norm": 0.640772240723941,
-      "learning_rate": 9.810400607857975e-06,
-      "loss": 0.7008,
-      "num_tokens": 42881848.0,
+      "epoch": 2.925449871465296,
+      "grad_norm": 0.5112313921441834,
+      "learning_rate": 1.0275044318670219e-06,
+      "loss": 1.0679,
+      "num_tokens": 113427101.0,
       "step": 143
     },
     {
-      "epoch": 3.4285714285714284,
-      "grad_norm": 0.6750307967979159,
-      "learning_rate": 9.629739263310663e-06,
-      "loss": 0.6576,
-      "num_tokens": 43180370.0,
+      "epoch": 2.9460154241645244,
+      "grad_norm": 1.1240389638051913,
+      "learning_rate": 1.0176093006627485e-06,
+      "loss": 0.9497,
+      "num_tokens": 114240330.0,
       "step": 144
     },
     {
-      "epoch": 3.4523809523809526,
-      "grad_norm": 0.6349593868773542,
-      "learning_rate": 9.4507233215224e-06,
-      "loss": 0.6895,
-      "num_tokens": 43485194.0,
+      "epoch": 2.966580976863753,
+      "grad_norm": 1.2175079530173951,
+      "learning_rate": 1.0099080604784303e-06,
+      "loss": 0.9541,
+      "num_tokens": 115051111.0,
       "step": 145
     },
     {
-      "epoch": 3.4761904761904763,
-      "grad_norm": 0.7007765296778146,
-      "learning_rate": 9.273395656167974e-06,
-      "loss": 0.6616,
-      "num_tokens": 43735211.0,
+      "epoch": 2.987146529562982,
+      "grad_norm": 0.6533936705036852,
+      "learning_rate": 1.0044044806601188e-06,
+      "loss": 1.0145,
+      "num_tokens": 115853478.0,
       "step": 146
     },
     {
-      "epoch": 3.5,
-      "grad_norm": 0.6838183617031265,
-      "learning_rate": 9.097798736586033e-06,
-      "loss": 0.6899,
-      "num_tokens": 44018672.0,
+      "epoch": 3.0,
+      "grad_norm": 0.6533936705036852,
+      "learning_rate": 1.0011012549164066e-06,
+      "loss": 1.0527,
+      "num_tokens": 116344566.0,
       "step": 147
     },
     {
-      "epoch": 3.5238095238095237,
-      "grad_norm": 0.6632040105335824,
-      "learning_rate": 8.92397461760784e-06,
-      "loss": 0.6846,
-      "num_tokens": 44328901.0,
-      "step": 148
-    },
-    {
-      "epoch": 3.5476190476190474,
-      "grad_norm": 0.6283627701359418,
-      "learning_rate": 8.751964929485264e-06,
-      "loss": 0.7087,
-      "num_tokens": 44639895.0,
-      "step": 149
-    },
-    {
-      "epoch": 3.571428571428571,
-      "grad_norm": 0.6949482066022805,
-      "learning_rate": 8.58181086792053e-06,
-      "loss": 0.7229,
-      "num_tokens": 44912674.0,
-      "step": 150
-    },
-    {
-      "epoch": 3.5952380952380953,
-      "grad_norm": 0.6161930464924805,
-      "learning_rate": 8.413553184199947e-06,
-      "loss": 0.7318,
-      "num_tokens": 45239369.0,
-      "step": 151
-    },
-    {
-      "epoch": 3.619047619047619,
-      "grad_norm": 0.6775157046640012,
-      "learning_rate": 8.247232175434151e-06,
-      "loss": 0.69,
-      "num_tokens": 45487078.0,
-      "step": 152
-    },
-    {
-      "epoch": 3.642857142857143,
-      "grad_norm": 0.6598743917563118,
-      "learning_rate": 8.082887674907099e-06,
-      "loss": 0.6823,
-      "num_tokens": 45764060.0,
-      "step": 153
-    },
-    {
-      "epoch": 3.6666666666666665,
-      "grad_norm": 0.6389058360603095,
-      "learning_rate": 7.92055904253611e-06,
-      "loss": 0.7304,
-      "num_tokens": 46055608.0,
-      "step": 154
-    },
-    {
-      "epoch": 3.6904761904761907,
-      "grad_norm": 0.6077489003813343,
-      "learning_rate": 7.760285155445328e-06,
-      "loss": 0.7093,
-      "num_tokens": 46359953.0,
-      "step": 155
-    },
-    {
-      "epoch": 3.7142857142857144,
-      "grad_norm": 0.5967397524054626,
-      "learning_rate": 7.602104398654793e-06,
-      "loss": 0.7066,
-      "num_tokens": 46668754.0,
-      "step": 156
-    },
-    {
-      "epoch": 3.738095238095238,
-      "grad_norm": 0.6350745338711014,
-      "learning_rate": 7.446054655887351e-06,
-      "loss": 0.7024,
-      "num_tokens": 46984092.0,
-      "step": 157
-    },
-    {
-      "epoch": 3.761904761904762,
-      "grad_norm": 0.654479549343766,
-      "learning_rate": 7.292173300495655e-06,
-      "loss": 0.7024,
-      "num_tokens": 47279370.0,
-      "step": 158
-    },
-    {
-      "epoch": 3.7857142857142856,
-      "grad_norm": 0.6241512933913473,
-      "learning_rate": 7.140497186511365e-06,
-      "loss": 0.6782,
-      "num_tokens": 47607126.0,
-      "step": 159
-    },
-    {
-      "epoch": 3.8095238095238093,
-      "grad_norm": 0.6892381674511956,
-      "learning_rate": 6.99106263981875e-06,
-      "loss": 0.6708,
-      "num_tokens": 47854285.0,
-      "step": 160
-    },
-    {
-      "epoch": 3.8333333333333335,
-      "grad_norm": 0.6054186909041117,
-      "learning_rate": 6.843905449454775e-06,
-      "loss": 0.7098,
-      "num_tokens": 48172146.0,
-      "step": 161
-    },
-    {
-      "epoch": 3.857142857142857,
-      "grad_norm": 0.5956432849446107,
-      "learning_rate": 6.699060859037737e-06,
-      "loss": 0.7313,
-      "num_tokens": 48512170.0,
-      "step": 162
-    },
-    {
-      "epoch": 3.880952380952381,
-      "grad_norm": 0.6733062415864223,
-      "learning_rate": 6.556563558326561e-06,
-      "loss": 0.7001,
-      "num_tokens": 48798674.0,
-      "step": 163
-    },
-    {
-      "epoch": 3.9047619047619047,
-      "grad_norm": 0.6484125556278886,
-      "learning_rate": 6.416447674912726e-06,
-      "loss": 0.7064,
-      "num_tokens": 49078950.0,
-      "step": 164
-    },
-    {
-      "epoch": 3.928571428571429,
-      "grad_norm": 0.6651238762004111,
-      "learning_rate": 6.278746766046815e-06,
-      "loss": 0.6613,
-      "num_tokens": 49321513.0,
-      "step": 165
-    },
-    {
-      "epoch": 3.9523809523809526,
-      "grad_norm": 0.6123780278745324,
-      "learning_rate": 6.143493810601696e-06,
-      "loss": 0.6737,
-      "num_tokens": 49622007.0,
-      "step": 166
-    },
-    {
-      "epoch": 3.9761904761904763,
-      "grad_norm": 0.6147575021892541,
-      "learning_rate": 6.010721201174187e-06,
-      "loss": 0.7213,
-      "num_tokens": 49949396.0,
-      "step": 167
-    },
-    {
-      "epoch": 4.0,
-      "grad_norm": 0.6525996500721447,
-      "learning_rate": 5.880460736327139e-06,
-      "loss": 0.6751,
-      "num_tokens": 50240879.0,
-      "step": 168
-    },
-    {
-      "epoch": 4.023809523809524,
-      "grad_norm": 1.1945930661064657,
-      "learning_rate": 5.7527436129738094e-06,
-      "loss": 0.5781,
-      "num_tokens": 50522550.0,
-      "step": 169
-    },
-    {
-      "epoch": 4.0476190476190474,
-      "grad_norm": 1.0060804692820782,
-      "learning_rate": 5.62760041890631e-06,
-      "loss": 0.5498,
-      "num_tokens": 50813206.0,
-      "step": 170
-    },
-    {
-      "epoch": 4.071428571428571,
-      "grad_norm": 0.751291910016424,
-      "learning_rate": 5.505061125469931e-06,
-      "loss": 0.5308,
-      "num_tokens": 51062794.0,
-      "step": 171
-    },
-    {
-      "epoch": 4.095238095238095,
-      "grad_norm": 0.7527801055673746,
-      "learning_rate": 5.385155080385125e-06,
-      "loss": 0.5223,
-      "num_tokens": 51335547.0,
-      "step": 172
-    },
-    {
-      "epoch": 4.119047619047619,
-      "grad_norm": 0.9363987669688705,
-      "learning_rate": 5.267911000718819e-06,
-      "loss": 0.5258,
-      "num_tokens": 51637656.0,
-      "step": 173
-    },
-    {
-      "epoch": 4.142857142857143,
-      "grad_norm": 1.1015381943904987,
-      "learning_rate": 5.153356966006791e-06,
-      "loss": 0.5196,
-      "num_tokens": 51953140.0,
-      "step": 174
-    },
-    {
-      "epoch": 4.166666666666667,
-      "grad_norm": 1.0393498678201005,
-      "learning_rate": 5.041520411528727e-06,
-      "loss": 0.55,
-      "num_tokens": 52262499.0,
-      "step": 175
-    },
-    {
-      "epoch": 4.190476190476191,
-      "grad_norm": 0.8250332666446762,
-      "learning_rate": 4.932428121737547e-06,
-      "loss": 0.5226,
-      "num_tokens": 52568631.0,
-      "step": 176
-    },
-    {
-      "epoch": 4.214285714285714,
-      "grad_norm": 0.7655426183040523,
-      "learning_rate": 4.826106223844648e-06,
-      "loss": 0.5127,
-      "num_tokens": 52834140.0,
-      "step": 177
-    },
-    {
-      "epoch": 4.238095238095238,
-      "grad_norm": 0.6995684804290214,
-      "learning_rate": 4.722580181562527e-06,
-      "loss": 0.494,
-      "num_tokens": 53156720.0,
-      "step": 178
-    },
-    {
-      "epoch": 4.261904761904762,
-      "grad_norm": 0.7748083731595724,
-      "learning_rate": 4.621874789006295e-06,
-      "loss": 0.5381,
-      "num_tokens": 53491051.0,
-      "step": 179
-    },
-    {
-      "epoch": 4.285714285714286,
-      "grad_norm": 0.8131952113654631,
-      "learning_rate": 4.524014164755603e-06,
-      "loss": 0.4968,
-      "num_tokens": 53787155.0,
-      "step": 180
-    },
-    {
-      "epoch": 4.309523809523809,
-      "grad_norm": 0.7367042370774411,
-      "learning_rate": 4.429021746078311e-06,
-      "loss": 0.4861,
-      "num_tokens": 54049978.0,
-      "step": 181
-    },
-    {
-      "epoch": 4.333333333333333,
-      "grad_norm": 0.7194704571250619,
-      "learning_rate": 4.336920283317344e-06,
-      "loss": 0.5212,
-      "num_tokens": 54305304.0,
-      "step": 182
-    },
-    {
-      "epoch": 4.357142857142857,
-      "grad_norm": 0.6527660461115374,
-      "learning_rate": 4.247731834442083e-06,
-      "loss": 0.5263,
-      "num_tokens": 54632438.0,
-      "step": 183
-    },
-    {
-      "epoch": 4.380952380952381,
-      "grad_norm": 0.7266514800309486,
-      "learning_rate": 4.161477759765542e-06,
-      "loss": 0.5202,
-      "num_tokens": 54954887.0,
-      "step": 184
-    },
-    {
-      "epoch": 4.404761904761905,
-      "grad_norm": 0.761296256236184,
-      "learning_rate": 4.078178716828666e-06,
-      "loss": 0.5379,
-      "num_tokens": 55245795.0,
-      "step": 185
-    },
-    {
-      "epoch": 4.428571428571429,
-      "grad_norm": 0.7945856951237223,
-      "learning_rate": 3.9978546554529185e-06,
-      "loss": 0.5176,
-      "num_tokens": 55546573.0,
-      "step": 186
-    },
-    {
-      "epoch": 4.4523809523809526,
-      "grad_norm": 0.7468660256983087,
-      "learning_rate": 3.920524812962359e-06,
-      "loss": 0.5096,
-      "num_tokens": 55843166.0,
-      "step": 187
-    },
-    {
-      "epoch": 4.476190476190476,
-      "grad_norm": 0.7193745095120948,
-      "learning_rate": 3.846207709576389e-06,
-      "loss": 0.5205,
-      "num_tokens": 56138218.0,
-      "step": 188
-    },
-    {
-      "epoch": 4.5,
-      "grad_norm": 0.6655820277052442,
-      "learning_rate": 3.7749211439742257e-06,
-      "loss": 0.5041,
-      "num_tokens": 56417291.0,
-      "step": 189
-    },
-    {
-      "epoch": 4.523809523809524,
-      "grad_norm": 0.7239518179183401,
-      "learning_rate": 3.7066821890321687e-06,
-      "loss": 0.4846,
-      "num_tokens": 56692919.0,
-      "step": 190
-    },
-    {
-      "epoch": 4.5476190476190474,
-      "grad_norm": 0.6477616515263318,
-      "learning_rate": 3.6415071877347165e-06,
-      "loss": 0.539,
-      "num_tokens": 57031682.0,
-      "step": 191
-    },
-    {
-      "epoch": 4.571428571428571,
-      "grad_norm": 0.6640615854897481,
-      "learning_rate": 3.579411749260486e-06,
-      "loss": 0.4944,
-      "num_tokens": 57344445.0,
-      "step": 192
-    },
-    {
-      "epoch": 4.595238095238095,
-      "grad_norm": 0.6848846128939778,
-      "learning_rate": 3.520410745243858e-06,
-      "loss": 0.5121,
-      "num_tokens": 57613210.0,
-      "step": 193
-    },
-    {
-      "epoch": 4.619047619047619,
-      "grad_norm": 0.6422019750084208,
-      "learning_rate": 3.4645183062132826e-06,
-      "loss": 0.5193,
-      "num_tokens": 57942367.0,
-      "step": 194
-    },
-    {
-      "epoch": 4.642857142857143,
-      "grad_norm": 0.7657789249156284,
-      "learning_rate": 3.411747818207066e-06,
-      "loss": 0.4979,
-      "num_tokens": 58201641.0,
-      "step": 195
-    },
-    {
-      "epoch": 4.666666666666667,
-      "grad_norm": 0.6739307543671282,
-      "learning_rate": 3.3621119195674597e-06,
-      "loss": 0.4888,
-      "num_tokens": 58517521.0,
-      "step": 196
-    },
-    {
-      "epoch": 4.690476190476191,
-      "grad_norm": 0.6865319310088059,
-      "learning_rate": 3.3156224979138306e-06,
-      "loss": 0.494,
-      "num_tokens": 58816395.0,
-      "step": 197
-    },
-    {
-      "epoch": 4.714285714285714,
-      "grad_norm": 0.6584126661804732,
-      "learning_rate": 3.2722906872956088e-06,
-      "loss": 0.5181,
-      "num_tokens": 59120541.0,
-      "step": 198
-    },
-    {
-      "epoch": 4.738095238095238,
-      "grad_norm": 0.6099916205741043,
-      "learning_rate": 3.232126865525731e-06,
-      "loss": 0.5293,
-      "num_tokens": 59461204.0,
-      "step": 199
-    },
-    {
-      "epoch": 4.761904761904762,
-      "grad_norm": 0.667910578763116,
-      "learning_rate": 3.1951406516951874e-06,
-      "loss": 0.5116,
-      "num_tokens": 59751334.0,
-      "step": 200
-    },
-    {
-      "epoch": 4.785714285714286,
-      "grad_norm": 0.6141509992681458,
-      "learning_rate": 3.1613409038692853e-06,
-      "loss": 0.5124,
-      "num_tokens": 60077027.0,
-      "step": 201
-    },
-    {
-      "epoch": 4.809523809523809,
-      "grad_norm": 0.6302007739092258,
-      "learning_rate": 3.130735716966174e-06,
-      "loss": 0.5405,
-      "num_tokens": 60408543.0,
-      "step": 202
-    },
-    {
-      "epoch": 4.833333333333333,
-      "grad_norm": 0.6664208365053094,
-      "learning_rate": 3.1033324208181422e-06,
-      "loss": 0.5142,
-      "num_tokens": 60702718.0,
-      "step": 203
-    },
-    {
-      "epoch": 4.857142857142857,
-      "grad_norm": 0.6536331401194854,
-      "learning_rate": 3.0791375784161456e-06,
-      "loss": 0.5193,
-      "num_tokens": 61018938.0,
-      "step": 204
-    },
-    {
-      "epoch": 4.880952380952381,
-      "grad_norm": 0.6969216972121804,
-      "learning_rate": 3.058156984337999e-06,
-      "loss": 0.5061,
-      "num_tokens": 61279305.0,
-      "step": 205
-    },
-    {
-      "epoch": 4.904761904761905,
-      "grad_norm": 0.6535453587519624,
-      "learning_rate": 3.0403956633605944e-06,
-      "loss": 0.4729,
-      "num_tokens": 61565633.0,
-      "step": 206
-    },
-    {
-      "epoch": 4.928571428571429,
-      "grad_norm": 0.6753649990944919,
-      "learning_rate": 3.0258578692564804e-06,
-      "loss": 0.4772,
-      "num_tokens": 61841335.0,
-      "step": 207
-    },
-    {
-      "epoch": 4.9523809523809526,
-      "grad_norm": 0.6358980505171887,
-      "learning_rate": 3.014547083775106e-06,
-      "loss": 0.5398,
-      "num_tokens": 62169402.0,
-      "step": 208
-    },
-    {
-      "epoch": 4.976190476190476,
-      "grad_norm": 0.5805504932681311,
-      "learning_rate": 3.0064660158089516e-06,
-      "loss": 0.524,
-      "num_tokens": 62538856.0,
-      "step": 209
-    },
-    {
-      "epoch": 5.0,
-      "grad_norm": 0.6817532772499482,
-      "learning_rate": 3.001616600744755e-06,
-      "loss": 0.5121,
-      "num_tokens": 62797894.0,
-      "step": 210
-    },
-    {
-      "epoch": 5.0,
-      "step": 210,
-      "total_flos": 2.7972111648056934e+17,
-      "train_loss": 1.0216455256655104,
-      "train_runtime": 2780.2712,
-      "train_samples_per_second": 9.666,
-      "train_steps_per_second": 0.076
+      "epoch": 3.0,
+      "step": 147,
+      "total_flos": 5.614755170526167e+17,
+      "train_loss": 1.1423310605036159,
+      "train_runtime": 3470.2952,
+      "train_samples_per_second": 10.746,
+      "train_steps_per_second": 0.042
     }
   ],
   "logging_steps": 1,
-  "max_steps": 210,
+  "max_steps": 147,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 3,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -1716,8 +1230,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.7972111648056934e+17,
-  "train_batch_size": 2,
+  "total_flos": 5.614755170526167e+17,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }