diff --git "a/checkpoint-12588/trainer_state.json" "b/checkpoint-12588/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-12588/trainer_state.json"
@@ -0,0 +1,90907 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "global_step": 12588,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-06,
+      "loss": 1.2938,
+      "theoretical_loss": 3.515360179132646,
+      "tokens_seen": 1650130944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 1.2607,
+      "theoretical_loss": 3.515336204772745,
+      "tokens_seen": 1650262016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1904761904761905e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.5153122328500483,
+      "tokens_seen": 1650393088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015872e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.5152882633641145,
+      "tokens_seen": 1650524160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.515264296314503,
+      "tokens_seen": 1650655232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.380952380952381e-05,
+      "loss": 1.3137,
+      "theoretical_loss": 3.515240331700772,
+      "tokens_seen": 1650786304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.7777777777777776e-05,
+      "loss": 1.3073,
+      "theoretical_loss": 3.5152163695224807,
+      "tokens_seen": 1650917376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 1.3057,
+      "theoretical_loss": 3.515192409779189,
+      "tokens_seen": 1651048448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.571428571428571e-05,
+      "loss": 1.2436,
+      "theoretical_loss": 3.515168452470455,
+      "tokens_seen": 1651179520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.5151444975958386,
+      "tokens_seen": 1651310592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.365079365079365e-05,
+      "loss": 1.1714,
+      "theoretical_loss": 3.5151205451548995,
+      "tokens_seen": 1651441664
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.5150965951471975,
+      "tokens_seen": 1651572736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.1587301587301586e-05,
+      "loss": 1.2982,
+      "theoretical_loss": 3.5150726475722918,
+      "tokens_seen": 1651703808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.555555555555555e-05,
+      "loss": 1.2801,
+      "theoretical_loss": 3.515048702429743,
+      "tokens_seen": 1651834880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.9523809523809524e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.5150247597191098,
+      "tokens_seen": 1651965952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.5150008194399534,
+      "tokens_seen": 1652097024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.746031746031745e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.5149768815918336,
+      "tokens_seen": 1652228096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.142857142857142e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.5149529461743105,
+      "tokens_seen": 1652359168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.53968253968254e-05,
+      "loss": 1.3312,
+      "theoretical_loss": 3.5149290131869453,
+      "tokens_seen": 1652490240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507937e-05,
+      "loss": 1.3053,
+      "theoretical_loss": 3.514905082629298,
+      "tokens_seen": 1652621312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.333333333333333e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.5148811545009293,
+      "tokens_seen": 1652752384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.73015873015873e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.5148572288013993,
+      "tokens_seen": 1652883456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.126984126984126e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.51483330553027,
+      "tokens_seen": 1653014528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.5148093846871022,
+      "tokens_seen": 1653145600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-05,
+      "loss": 1.3246,
+      "theoretical_loss": 3.5147854662714564,
+      "tokens_seen": 1653276672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010317460317460317,
+      "loss": 1.2343,
+      "theoretical_loss": 3.514761550282894,
+      "tokens_seen": 1653407744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010714285714285714,
+      "loss": 1.2603,
+      "theoretical_loss": 3.514737636720977,
+      "tokens_seen": 1653538816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001111111111111111,
+      "loss": 1.2225,
+      "theoretical_loss": 3.514713725585266,
+      "tokens_seen": 1653669888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011507936507936508,
+      "loss": 1.2936,
+      "theoretical_loss": 3.5146898168753236,
+      "tokens_seen": 1653800960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011904761904761905,
+      "loss": 1.2528,
+      "theoretical_loss": 3.5146659105907103,
+      "tokens_seen": 1653932032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000123015873015873,
+      "loss": 1.2661,
+      "theoretical_loss": 3.5146420067309885,
+      "tokens_seen": 1654063104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012698412698412698,
+      "loss": 1.1561,
+      "theoretical_loss": 3.5146181052957197,
+      "tokens_seen": 1654194176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013095238095238096,
+      "loss": 1.3022,
+      "theoretical_loss": 3.5145942062844666,
+      "tokens_seen": 1654325248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001349206349206349,
+      "loss": 1.2089,
+      "theoretical_loss": 3.514570309696791,
+      "tokens_seen": 1654456320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001388888888888889,
+      "loss": 1.2576,
+      "theoretical_loss": 3.514546415532255,
+      "tokens_seen": 1654587392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014285714285714284,
+      "loss": 1.3197,
+      "theoretical_loss": 3.514522523790421,
+      "tokens_seen": 1654718464
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014682539682539685,
+      "loss": 1.2411,
+      "theoretical_loss": 3.514498634470852,
+      "tokens_seen": 1654849536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001507936507936508,
+      "loss": 1.2911,
+      "theoretical_loss": 3.51447474757311,
+      "tokens_seen": 1654980608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015476190476190478,
+      "loss": 1.2035,
+      "theoretical_loss": 3.5144508630967577,
+      "tokens_seen": 1655111680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015873015873015873,
+      "loss": 1.2914,
+      "theoretical_loss": 3.5144269810413586,
+      "tokens_seen": 1655242752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001626984126984127,
+      "loss": 1.1916,
+      "theoretical_loss": 3.5144031014064745,
+      "tokens_seen": 1655373824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016666666666666666,
+      "loss": 1.2601,
+      "theoretical_loss": 3.5143792241916696,
+      "tokens_seen": 1655504896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017063492063492064,
+      "loss": 1.2655,
+      "theoretical_loss": 3.514355349396506,
+      "tokens_seen": 1655635968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001746031746031746,
+      "loss": 1.1841,
+      "theoretical_loss": 3.514331477020548,
+      "tokens_seen": 1655767040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017857142857142857,
+      "loss": 1.3176,
+      "theoretical_loss": 3.5143076070633583,
+      "tokens_seen": 1655898112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018253968253968252,
+      "loss": 1.2994,
+      "theoretical_loss": 3.514283739524501,
+      "tokens_seen": 1656029184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001865079365079365,
+      "loss": 1.2398,
+      "theoretical_loss": 3.514259874403539,
+      "tokens_seen": 1656160256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 1.261,
+      "theoretical_loss": 3.5142360117000364,
+      "tokens_seen": 1656291328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019444444444444446,
+      "loss": 1.3179,
+      "theoretical_loss": 3.514212151413557,
+      "tokens_seen": 1656422400
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 557587,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3530542850494385,
+      "objective/train/theoretical_loss": 3.514188293543665,
+      "objective/train/tokens_used": 27012576,
+      "theoretical_loss": 3.514188293543665,
+      "tokens_seen": 1656553472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001984126984126984,
+      "loss": 1.28,
+      "theoretical_loss": 3.514188293543665,
+      "tokens_seen": 1656553472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002023809523809524,
+      "loss": 1.2405,
+      "theoretical_loss": 3.5141644380899235,
+      "tokens_seen": 1656684544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020634920634920634,
+      "loss": 1.2796,
+      "theoretical_loss": 3.514140585051898,
+      "tokens_seen": 1656815616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021031746031746032,
+      "loss": 1.2485,
+      "theoretical_loss": 3.5141167344291517,
+      "tokens_seen": 1656946688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021428571428571427,
+      "loss": 1.2376,
+      "theoretical_loss": 3.51409288622125,
+      "tokens_seen": 1657077760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021825396825396825,
+      "loss": 1.265,
+      "theoretical_loss": 3.514069040427757,
+      "tokens_seen": 1657208832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002222222222222222,
+      "loss": 1.2147,
+      "theoretical_loss": 3.5140451970482367,
+      "tokens_seen": 1657339904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022619047619047618,
+      "loss": 1.2563,
+      "theoretical_loss": 3.514021356082255,
+      "tokens_seen": 1657470976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023015873015873016,
+      "loss": 1.2865,
+      "theoretical_loss": 3.5139975175293756,
+      "tokens_seen": 1657602048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023412698412698414,
+      "loss": 1.2764,
+      "theoretical_loss": 3.5139736813891647,
+      "tokens_seen": 1657733120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002380952380952381,
+      "loss": 1.2268,
+      "theoretical_loss": 3.513949847661186,
+      "tokens_seen": 1657864192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024206349206349207,
+      "loss": 1.2752,
+      "theoretical_loss": 3.513926016345006,
+      "tokens_seen": 1657995264
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000246031746031746,
+      "loss": 1.2821,
+      "theoretical_loss": 3.5139021874401886,
+      "tokens_seen": 1658126336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025,
+      "loss": 1.2606,
+      "theoretical_loss": 3.5138783609463005,
+      "tokens_seen": 1658257408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025396825396825396,
+      "loss": 1.2455,
+      "theoretical_loss": 3.513854536862907,
+      "tokens_seen": 1658388480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025793650793650796,
+      "loss": 1.2722,
+      "theoretical_loss": 3.513830715189573,
+      "tokens_seen": 1658519552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002619047619047619,
+      "loss": 1.3252,
+      "theoretical_loss": 3.5138068959258653,
+      "tokens_seen": 1658650624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026587301587301587,
+      "loss": 1.2851,
+      "theoretical_loss": 3.5137830790713487,
+      "tokens_seen": 1658781696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002698412698412698,
+      "loss": 1.2701,
+      "theoretical_loss": 3.51375926462559,
+      "tokens_seen": 1658912768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002738095238095238,
+      "loss": 1.2215,
+      "theoretical_loss": 3.5137354525881555,
+      "tokens_seen": 1659043840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002777777777777778,
+      "loss": 1.168,
+      "theoretical_loss": 3.5137116429586106,
+      "tokens_seen": 1659174912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028174603174603173,
+      "loss": 1.3272,
+      "theoretical_loss": 3.5136878357365218,
+      "tokens_seen": 1659305984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002857142857142857,
+      "loss": 1.2902,
+      "theoretical_loss": 3.5136640309214555,
+      "tokens_seen": 1659437056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002896825396825397,
+      "loss": 1.2602,
+      "theoretical_loss": 3.513640228512979,
+      "tokens_seen": 1659568128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002936507936507937,
+      "loss": 1.2853,
+      "theoretical_loss": 3.513616428510658,
+      "tokens_seen": 1659699200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029761904761904765,
+      "loss": 1.291,
+      "theoretical_loss": 3.5135926309140597,
+      "tokens_seen": 1659830272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003015873015873016,
+      "loss": 1.2445,
+      "theoretical_loss": 3.513568835722751,
+      "tokens_seen": 1659961344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003055555555555556,
+      "loss": 1.2604,
+      "theoretical_loss": 3.5135450429362987,
+      "tokens_seen": 1660092416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030952380952380956,
+      "loss": 1.2177,
+      "theoretical_loss": 3.5135212525542707,
+      "tokens_seen": 1660223488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003134920634920635,
+      "loss": 1.2235,
+      "theoretical_loss": 3.5134974645762327,
+      "tokens_seen": 1660354560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031746031746031746,
+      "loss": 1.3341,
+      "theoretical_loss": 3.513473679001754,
+      "tokens_seen": 1660485632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 1.2587,
+      "theoretical_loss": 3.5134498958303997,
+      "tokens_seen": 1660616704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003253968253968254,
+      "loss": 1.3394,
+      "theoretical_loss": 3.5134261150617396,
+      "tokens_seen": 1660747776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032936507936507937,
+      "loss": 1.2105,
+      "theoretical_loss": 3.5134023366953397,
+      "tokens_seen": 1660878848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003333333333333333,
+      "loss": 1.2035,
+      "theoretical_loss": 3.513378560730769,
+      "tokens_seen": 1661009920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033730158730158733,
+      "loss": 1.2496,
+      "theoretical_loss": 3.5133547871675943,
+      "tokens_seen": 1661140992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003412698412698413,
+      "loss": 1.2273,
+      "theoretical_loss": 3.5133310160053846,
+      "tokens_seen": 1661272064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034523809523809523,
+      "loss": 1.282,
+      "theoretical_loss": 3.5133072472437075,
+      "tokens_seen": 1661403136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003492063492063492,
+      "loss": 1.2472,
+      "theoretical_loss": 3.513283480882131,
+      "tokens_seen": 1661534208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003531746031746032,
+      "loss": 1.2549,
+      "theoretical_loss": 3.513259716920224,
+      "tokens_seen": 1661665280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035714285714285714,
+      "loss": 1.2882,
+      "theoretical_loss": 3.5132359553575547,
+      "tokens_seen": 1661796352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003611111111111111,
+      "loss": 1.2318,
+      "theoretical_loss": 3.5132121961936917,
+      "tokens_seen": 1661927424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036507936507936505,
+      "loss": 1.2945,
+      "theoretical_loss": 3.5131884394282036,
+      "tokens_seen": 1662058496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036904761904761905,
+      "loss": 1.2902,
+      "theoretical_loss": 3.5131646850606586,
+      "tokens_seen": 1662189568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000373015873015873,
+      "loss": 1.2843,
+      "theoretical_loss": 3.513140933090627,
+      "tokens_seen": 1662320640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000376984126984127,
+      "loss": 1.3069,
+      "theoretical_loss": 3.513117183517677,
+      "tokens_seen": 1662451712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038095238095238096,
+      "loss": 1.2235,
+      "theoretical_loss": 3.513093436341377,
+      "tokens_seen": 1662582784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038492063492063497,
+      "loss": 1.308,
+      "theoretical_loss": 3.513069691561298,
+      "tokens_seen": 1662713856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003888888888888889,
+      "loss": 1.2904,
+      "theoretical_loss": 3.513045949177007,
+      "tokens_seen": 1662844928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003928571428571429,
+      "loss": 1.3377,
+      "theoretical_loss": 3.513022209188076,
+      "tokens_seen": 1662976000
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 559635,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3291172981262207,
+      "objective/train/theoretical_loss": 3.5129984715940727,
+      "objective/train/tokens_used": 33566176,
+      "theoretical_loss": 3.5129984715940727,
+      "tokens_seen": 1663107072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003968253968253968,
+      "loss": 1.2956,
+      "theoretical_loss": 3.5129984715940727,
+      "tokens_seen": 1663107072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040079365079365083,
+      "loss": 1.2374,
+      "theoretical_loss": 3.5129747363945674,
+      "tokens_seen": 1663238144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004047619047619048,
+      "loss": 1.2654,
+      "theoretical_loss": 3.51295100358913,
+      "tokens_seen": 1663369216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040873015873015874,
+      "loss": 1.2903,
+      "theoretical_loss": 3.51292727317733,
+      "tokens_seen": 1663500288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004126984126984127,
+      "loss": 1.2669,
+      "theoretical_loss": 3.5129035451587383,
+      "tokens_seen": 1663631360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004166666666666667,
+      "loss": 1.2714,
+      "theoretical_loss": 3.512879819532924,
+      "tokens_seen": 1663762432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042063492063492065,
+      "loss": 1.306,
+      "theoretical_loss": 3.5128560962994575,
+      "tokens_seen": 1663893504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004246031746031746,
+      "loss": 1.3073,
+      "theoretical_loss": 3.5128323754579096,
+      "tokens_seen": 1664024576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042857142857142855,
+      "loss": 1.2328,
+      "theoretical_loss": 3.512808657007851,
+      "tokens_seen": 1664155648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043253968253968256,
+      "loss": 1.3287,
+      "theoretical_loss": 3.512784940948851,
+      "tokens_seen": 1664286720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004365079365079365,
+      "loss": 1.3224,
+      "theoretical_loss": 3.512761227280482,
+      "tokens_seen": 1664417792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044047619047619046,
+      "loss": 1.2213,
+      "theoretical_loss": 3.5127375160023133,
+      "tokens_seen": 1664548864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004444444444444444,
+      "loss": 1.3487,
+      "theoretical_loss": 3.512713807113917,
+      "tokens_seen": 1664679936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004484126984126984,
+      "loss": 1.2523,
+      "theoretical_loss": 3.5126901006148628,
+      "tokens_seen": 1664811008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045238095238095237,
+      "loss": 1.2725,
+      "theoretical_loss": 3.5126663965047227,
+      "tokens_seen": 1664942080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004563492063492063,
+      "loss": 1.2295,
+      "theoretical_loss": 3.5126426947830676,
+      "tokens_seen": 1665073152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046031746031746033,
+      "loss": 1.332,
+      "theoretical_loss": 3.5126189954494693,
+      "tokens_seen": 1665204224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046428571428571433,
+      "loss": 1.2685,
+      "theoretical_loss": 3.5125952985034985,
+      "tokens_seen": 1665335296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004682539682539683,
+      "loss": 1.3093,
+      "theoretical_loss": 3.5125716039447274,
+      "tokens_seen": 1665466368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047222222222222224,
+      "loss": 1.3139,
+      "theoretical_loss": 3.5125479117727276,
+      "tokens_seen": 1665597440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004761904761904762,
+      "loss": 1.3776,
+      "theoretical_loss": 3.5125242219870705,
+      "tokens_seen": 1665728512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004801587301587302,
+      "loss": 1.3334,
+      "theoretical_loss": 3.5125005345873284,
+      "tokens_seen": 1665859584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048412698412698415,
+      "loss": 1.281,
+      "theoretical_loss": 3.512476849573073,
+      "tokens_seen": 1665990656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004880952380952381,
+      "loss": 1.348,
+      "theoretical_loss": 3.5124531669438768,
+      "tokens_seen": 1666121728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000492063492063492,
+      "loss": 1.3137,
+      "theoretical_loss": 3.5124294866993115,
+      "tokens_seen": 1666252800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000496031746031746,
+      "loss": 1.388,
+      "theoretical_loss": 3.51240580883895,
+      "tokens_seen": 1666383872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005,
+      "loss": 1.3686,
+      "theoretical_loss": 3.5123821333623644,
+      "tokens_seen": 1666514944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999598780292088,
+      "loss": 1.3103,
+      "theoretical_loss": 3.512358460269127,
+      "tokens_seen": 1666646016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999197560584176,
+      "loss": 1.4478,
+      "theoretical_loss": 3.5123347895588113,
+      "tokens_seen": 1666777088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998796340876264,
+      "loss": 1.3534,
+      "theoretical_loss": 3.512311121230989,
+      "tokens_seen": 1666908160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998395121168352,
+      "loss": 1.2722,
+      "theoretical_loss": 3.5122874552852337,
+      "tokens_seen": 1667039232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997993901460439,
+      "loss": 1.3701,
+      "theoretical_loss": 3.512263791721119,
+      "tokens_seen": 1667170304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997592681752528,
+      "loss": 1.2875,
+      "theoretical_loss": 3.512240130538216,
+      "tokens_seen": 1667301376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997191462044615,
+      "loss": 1.4349,
+      "theoretical_loss": 3.5122164717361,
+      "tokens_seen": 1667432448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996790242336704,
+      "loss": 1.3922,
+      "theoretical_loss": 3.512192815314343,
+      "tokens_seen": 1667563520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996389022628792,
+      "loss": 1.3067,
+      "theoretical_loss": 3.5121691612725194,
+      "tokens_seen": 1667694592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499598780292088,
+      "loss": 1.2017,
+      "theoretical_loss": 3.512145509610202,
+      "tokens_seen": 1667825664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995586583212968,
+      "loss": 1.2287,
+      "theoretical_loss": 3.512121860326965,
+      "tokens_seen": 1667956736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995185363505056,
+      "loss": 1.3179,
+      "theoretical_loss": 3.5120982134223815,
+      "tokens_seen": 1668087808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994784143797144,
+      "loss": 1.3628,
+      "theoretical_loss": 3.512074568896026,
+      "tokens_seen": 1668218880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994382924089231,
+      "loss": 1.342,
+      "theoretical_loss": 3.512050926747472,
+      "tokens_seen": 1668349952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993981704381319,
+      "loss": 1.2932,
+      "theoretical_loss": 3.512027286976294,
+      "tokens_seen": 1668481024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993580484673407,
+      "loss": 1.3425,
+      "theoretical_loss": 3.5120036495820663,
+      "tokens_seen": 1668612096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993179264965495,
+      "loss": 1.3112,
+      "theoretical_loss": 3.511980014564363,
+      "tokens_seen": 1668743168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992778045257583,
+      "loss": 1.354,
+      "theoretical_loss": 3.5119563819227584,
+      "tokens_seen": 1668874240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992376825549671,
+      "loss": 1.2664,
+      "theoretical_loss": 3.5119327516568273,
+      "tokens_seen": 1669005312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991975605841759,
+      "loss": 1.3195,
+      "theoretical_loss": 3.5119091237661437,
+      "tokens_seen": 1669136384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991574386133847,
+      "loss": 1.2459,
+      "theoretical_loss": 3.511885498250283,
+      "tokens_seen": 1669267456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991173166425935,
+      "loss": 1.4082,
+      "theoretical_loss": 3.51186187510882,
+      "tokens_seen": 1669398528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990771946718023,
+      "loss": 1.3036,
+      "theoretical_loss": 3.51183825434133,
+      "tokens_seen": 1669529600
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 561964,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3683562278747559,
+      "objective/train/theoretical_loss": 3.5118146359473874,
+      "objective/train/tokens_used": 40119776,
+      "theoretical_loss": 3.5118146359473874,
+      "tokens_seen": 1669660672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499037072701011,
+      "loss": 1.375,
+      "theoretical_loss": 3.5118146359473874,
+      "tokens_seen": 1669660672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989969507302198,
+      "loss": 1.2776,
+      "theoretical_loss": 3.5117910199265676,
+      "tokens_seen": 1669791744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989568287594286,
+      "loss": 1.3598,
+      "theoretical_loss": 3.5117674062784454,
+      "tokens_seen": 1669922816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989167067886375,
+      "loss": 1.3086,
+      "theoretical_loss": 3.5117437950025976,
+      "tokens_seen": 1670053888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988765848178463,
+      "loss": 1.3037,
+      "theoretical_loss": 3.5117201860985983,
+      "tokens_seen": 1670184960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988364628470551,
+      "loss": 1.257,
+      "theoretical_loss": 3.511696579566024,
+      "tokens_seen": 1670316032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987963408762639,
+      "loss": 1.2924,
+      "theoretical_loss": 3.5116729754044504,
+      "tokens_seen": 1670447104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987562189054726,
+      "loss": 1.2507,
+      "theoretical_loss": 3.5116493736134524,
+      "tokens_seen": 1670578176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987160969346815,
+      "loss": 1.2877,
+      "theoretical_loss": 3.5116257741926074,
+      "tokens_seen": 1670709248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986759749638902,
+      "loss": 1.3646,
+      "theoretical_loss": 3.5116021771414903,
+      "tokens_seen": 1670840320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986358529930991,
+      "loss": 1.2318,
+      "theoretical_loss": 3.511578582459678,
+      "tokens_seen": 1670971392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985957310223078,
+      "loss": 1.329,
+      "theoretical_loss": 3.511554990146746,
+      "tokens_seen": 1671102464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985556090515166,
+      "loss": 1.2606,
+      "theoretical_loss": 3.5115314002022715,
+      "tokens_seen": 1671233536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985154870807254,
+      "loss": 1.2907,
+      "theoretical_loss": 3.5115078126258306,
+      "tokens_seen": 1671364608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984753651099342,
+      "loss": 1.3361,
+      "theoretical_loss": 3.511484227417,
+      "tokens_seen": 1671495680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498435243139143,
+      "loss": 1.3065,
+      "theoretical_loss": 3.5114606445753567,
+      "tokens_seen": 1671626752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983951211683518,
+      "loss": 1.3094,
+      "theoretical_loss": 3.511437064100477,
+      "tokens_seen": 1671757824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983549991975606,
+      "loss": 1.3199,
+      "theoretical_loss": 3.5114134859919375,
+      "tokens_seen": 1671888896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983148772267693,
+      "loss": 1.3722,
+      "theoretical_loss": 3.5113899102493167,
+      "tokens_seen": 1672019968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982747552559782,
+      "loss": 1.351,
+      "theoretical_loss": 3.5113663368721904,
+      "tokens_seen": 1672151040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982346332851869,
+      "loss": 1.2809,
+      "theoretical_loss": 3.5113427658601366,
+      "tokens_seen": 1672282112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981945113143958,
+      "loss": 1.4316,
+      "theoretical_loss": 3.5113191972127327,
+      "tokens_seen": 1672413184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981543893436046,
+      "loss": 1.3456,
+      "theoretical_loss": 3.5112956309295553,
+      "tokens_seen": 1672544256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981142673728134,
+      "loss": 1.3165,
+      "theoretical_loss": 3.5112720670101827,
+      "tokens_seen": 1672675328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980741454020222,
+      "loss": 1.224,
+      "theoretical_loss": 3.5112485054541924,
+      "tokens_seen": 1672806400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498034023431231,
+      "loss": 1.2185,
+      "theoretical_loss": 3.511224946261162,
+      "tokens_seen": 1672937472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979939014604398,
+      "loss": 1.3662,
+      "theoretical_loss": 3.5112013894306706,
+      "tokens_seen": 1673068544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979537794896486,
+      "loss": 1.4679,
+      "theoretical_loss": 3.511177834962295,
+      "tokens_seen": 1673199616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979136575188573,
+      "loss": 1.3216,
+      "theoretical_loss": 3.5111542828556135,
+      "tokens_seen": 1673330688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978735355480661,
+      "loss": 1.3483,
+      "theoretical_loss": 3.5111307331102046,
+      "tokens_seen": 1673461760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978334135772749,
+      "loss": 1.2716,
+      "theoretical_loss": 3.5111071857256464,
+      "tokens_seen": 1673592832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977932916064837,
+      "loss": 1.3642,
+      "theoretical_loss": 3.511083640701518,
+      "tokens_seen": 1673723904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977531696356925,
+      "loss": 1.3388,
+      "theoretical_loss": 3.5110600980373965,
+      "tokens_seen": 1673854976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977130476649013,
+      "loss": 1.3577,
+      "theoretical_loss": 3.5110365577328624,
+      "tokens_seen": 1673986048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976729256941101,
+      "loss": 1.3594,
+      "theoretical_loss": 3.511013019787493,
+      "tokens_seen": 1674117120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976328037233188,
+      "loss": 1.4174,
+      "theoretical_loss": 3.510989484200868,
+      "tokens_seen": 1674248192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975926817525277,
+      "loss": 1.2646,
+      "theoretical_loss": 3.5109659509725666,
+      "tokens_seen": 1674379264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975525597817364,
+      "loss": 1.3308,
+      "theoretical_loss": 3.5109424201021673,
+      "tokens_seen": 1674510336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975124378109452,
+      "loss": 1.3809,
+      "theoretical_loss": 3.5109188915892497,
+      "tokens_seen": 1674641408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497472315840154,
+      "loss": 1.2727,
+      "theoretical_loss": 3.5108953654333925,
+      "tokens_seen": 1674772480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974321938693629,
+      "loss": 1.3636,
+      "theoretical_loss": 3.510871841634176,
+      "tokens_seen": 1674903552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973920718985717,
+      "loss": 1.328,
+      "theoretical_loss": 3.510848320191179,
+      "tokens_seen": 1675034624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973519499277805,
+      "loss": 1.2689,
+      "theoretical_loss": 3.5108248011039818,
+      "tokens_seen": 1675165696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973118279569893,
+      "loss": 1.3691,
+      "theoretical_loss": 3.510801284372164,
+      "tokens_seen": 1675296768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972717059861981,
+      "loss": 1.3373,
+      "theoretical_loss": 3.510777769995305,
+      "tokens_seen": 1675427840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972315840154069,
+      "loss": 1.3271,
+      "theoretical_loss": 3.5107542579729847,
+      "tokens_seen": 1675558912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971914620446156,
+      "loss": 1.2849,
+      "theoretical_loss": 3.510730748304784,
+      "tokens_seen": 1675689984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971513400738245,
+      "loss": 1.3184,
+      "theoretical_loss": 3.5107072409902824,
+      "tokens_seen": 1675821056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971112181030332,
+      "loss": 1.2925,
+      "theoretical_loss": 3.51068373602906,
+      "tokens_seen": 1675952128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497071096132242,
+      "loss": 1.3755,
+      "theoretical_loss": 3.5106602334206984,
+      "tokens_seen": 1676083200
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 564144,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2208685874938965,
+      "objective/train/theoretical_loss": 3.5106367331647768,
+      "objective/train/tokens_used": 46673376,
+      "theoretical_loss": 3.5106367331647768,
+      "tokens_seen": 1676214272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970309741614508,
+      "loss": 1.3643,
+      "theoretical_loss": 3.5106367331647768,
+      "tokens_seen": 1676214272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969908521906596,
+      "loss": 1.3453,
+      "theoretical_loss": 3.5106132352608768,
+      "tokens_seen": 1676345344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969507302198684,
+      "loss": 1.3673,
+      "theoretical_loss": 3.5105897397085783,
+      "tokens_seen": 1676476416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969106082490772,
+      "loss": 1.3364,
+      "theoretical_loss": 3.5105662465074627,
+      "tokens_seen": 1676607488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496870486278286,
+      "loss": 1.3742,
+      "theoretical_loss": 3.5105427556571107,
+      "tokens_seen": 1676738560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968303643074947,
+      "loss": 1.334,
+      "theoretical_loss": 3.510519267157103,
+      "tokens_seen": 1676869632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967902423367036,
+      "loss": 1.2829,
+      "theoretical_loss": 3.5104957810070214,
+      "tokens_seen": 1677000704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967501203659123,
+      "loss": 1.302,
+      "theoretical_loss": 3.5104722972064475,
+      "tokens_seen": 1677131776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967099983951212,
+      "loss": 1.391,
+      "theoretical_loss": 3.510448815754961,
+      "tokens_seen": 1677262848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049666987642433,
+      "loss": 1.2919,
+      "theoretical_loss": 3.510425336652145,
+      "tokens_seen": 1677393920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966297544535388,
+      "loss": 1.2898,
+      "theoretical_loss": 3.5104018598975806,
+      "tokens_seen": 1677524992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965896324827476,
+      "loss": 1.3491,
+      "theoretical_loss": 3.5103783854908492,
+      "tokens_seen": 1677656064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965495105119564,
+      "loss": 1.3022,
+      "theoretical_loss": 3.5103549134315326,
+      "tokens_seen": 1677787136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965093885411652,
+      "loss": 1.3331,
+      "theoretical_loss": 3.5103314437192132,
+      "tokens_seen": 1677918208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496469266570374,
+      "loss": 1.2374,
+      "theoretical_loss": 3.510307976353473,
+      "tokens_seen": 1678049280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964291445995827,
+      "loss": 1.3765,
+      "theoretical_loss": 3.5102845113338934,
+      "tokens_seen": 1678180352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963890226287915,
+      "loss": 1.3246,
+      "theoretical_loss": 3.510261048660057,
+      "tokens_seen": 1678311424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963489006580003,
+      "loss": 1.2651,
+      "theoretical_loss": 3.510237588331546,
+      "tokens_seen": 1678442496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963087786872091,
+      "loss": 1.2701,
+      "theoretical_loss": 3.510214130347944,
+      "tokens_seen": 1678573568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962686567164179,
+      "loss": 1.2873,
+      "theoretical_loss": 3.5101906747088316,
+      "tokens_seen": 1678704640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962285347456267,
+      "loss": 1.4074,
+      "theoretical_loss": 3.510167221413793,
+      "tokens_seen": 1678835712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961884127748355,
+      "loss": 1.3371,
+      "theoretical_loss": 3.51014377046241,
+      "tokens_seen": 1678966784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961482908040443,
+      "loss": 1.3412,
+      "theoretical_loss": 3.5101203218542665,
+      "tokens_seen": 1679097856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961081688332531,
+      "loss": 1.3593,
+      "theoretical_loss": 3.5100968755889443,
+      "tokens_seen": 1679228928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960680468624618,
+      "loss": 1.3192,
+      "theoretical_loss": 3.5100734316660267,
+      "tokens_seen": 1679360000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960279248916708,
+      "loss": 1.372,
+      "theoretical_loss": 3.510049990085098,
+      "tokens_seen": 1679491072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959878029208794,
+      "loss": 1.2672,
+      "theoretical_loss": 3.51002655084574,
+      "tokens_seen": 1679622144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959476809500883,
+      "loss": 1.3377,
+      "theoretical_loss": 3.510003113947537,
+      "tokens_seen": 1679753216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959075589792971,
+      "loss": 1.3349,
+      "theoretical_loss": 3.5099796793900726,
+      "tokens_seen": 1679884288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958674370085059,
+      "loss": 1.3077,
+      "theoretical_loss": 3.5099562471729295,
+      "tokens_seen": 1680015360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958273150377147,
+      "loss": 1.3645,
+      "theoretical_loss": 3.5099328172956925,
+      "tokens_seen": 1680146432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957871930669235,
+      "loss": 1.2471,
+      "theoretical_loss": 3.5099093897579445,
+      "tokens_seen": 1680277504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957470710961323,
+      "loss": 1.2957,
+      "theoretical_loss": 3.5098859645592704,
+      "tokens_seen": 1680408576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495706949125341,
+      "loss": 1.3056,
+      "theoretical_loss": 3.5098625416992535,
+      "tokens_seen": 1680539648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956668271545499,
+      "loss": 1.3997,
+      "theoretical_loss": 3.509839121177478,
+      "tokens_seen": 1680670720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956267051837586,
+      "loss": 1.2373,
+      "theoretical_loss": 3.509815702993529,
+      "tokens_seen": 1680801792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955865832129674,
+      "loss": 1.3768,
+      "theoretical_loss": 3.5097922871469898,
+      "tokens_seen": 1680932864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955464612421762,
+      "loss": 1.2651,
+      "theoretical_loss": 3.5097688736374453,
+      "tokens_seen": 1681063936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495506339271385,
+      "loss": 1.2465,
+      "theoretical_loss": 3.5097454624644797,
+      "tokens_seen": 1681195008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954662173005938,
+      "loss": 1.2606,
+      "theoretical_loss": 3.5097220536276783,
+      "tokens_seen": 1681326080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954260953298026,
+      "loss": 1.2466,
+      "theoretical_loss": 3.5096986471266254,
+      "tokens_seen": 1681457152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953859733590114,
+      "loss": 1.3305,
+      "theoretical_loss": 3.509675242960906,
+      "tokens_seen": 1681588224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953458513882202,
+      "loss": 1.3339,
+      "theoretical_loss": 3.509651841130106,
+      "tokens_seen": 1681719296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495305729417429,
+      "loss": 1.2869,
+      "theoretical_loss": 3.509628441633809,
+      "tokens_seen": 1681850368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952656074466377,
+      "loss": 1.359,
+      "theoretical_loss": 3.5096050444716003,
+      "tokens_seen": 1681981440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952254854758466,
+      "loss": 1.2784,
+      "theoretical_loss": 3.5095816496430667,
+      "tokens_seen": 1682112512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951853635050554,
+      "loss": 1.3032,
+      "theoretical_loss": 3.509558257147792,
+      "tokens_seen": 1682243584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951452415342642,
+      "loss": 1.2866,
+      "theoretical_loss": 3.509534866985363,
+      "tokens_seen": 1682374656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495105119563473,
+      "loss": 1.3713,
+      "theoretical_loss": 3.509511479155365,
+      "tokens_seen": 1682505728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950649975926818,
+      "loss": 1.3304,
+      "theoretical_loss": 3.5094880936573833,
+      "tokens_seen": 1682636800
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 566429,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.278090476989746,
+      "objective/train/theoretical_loss": 3.5094647104910033,
+      "objective/train/tokens_used": 53226976,
+      "theoretical_loss": 3.5094647104910033,
+      "tokens_seen": 1682767872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950248756218906,
+      "loss": 1.3391,
+      "theoretical_loss": 3.5094647104910033,
+      "tokens_seen": 1682767872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949847536510994,
+      "loss": 1.3943,
+      "theoretical_loss": 3.5094413296558127,
+      "tokens_seen": 1682898944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949446316803081,
+      "loss": 1.2879,
+      "theoretical_loss": 3.5094179511513954,
+      "tokens_seen": 1683030016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494904509709517,
+      "loss": 1.301,
+      "theoretical_loss": 3.509394574977339,
+      "tokens_seen": 1683161088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948643877387257,
+      "loss": 1.3127,
+      "theoretical_loss": 3.50937120113323,
+      "tokens_seen": 1683292160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948242657679345,
+      "loss": 1.2854,
+      "theoretical_loss": 3.509347829618653,
+      "tokens_seen": 1683423232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947841437971433,
+      "loss": 1.3658,
+      "theoretical_loss": 3.509324460433197,
+      "tokens_seen": 1683554304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947440218263521,
+      "loss": 1.3498,
+      "theoretical_loss": 3.509301093576446,
+      "tokens_seen": 1683685376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947038998555609,
+      "loss": 1.3668,
+      "theoretical_loss": 3.509277729047988,
+      "tokens_seen": 1683816448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946637778847697,
+      "loss": 1.3516,
+      "theoretical_loss": 3.5092543668474105,
+      "tokens_seen": 1683947520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946236559139785,
+      "loss": 1.3282,
+      "theoretical_loss": 3.509231006974299,
+      "tokens_seen": 1684078592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945835339431872,
+      "loss": 1.2943,
+      "theoretical_loss": 3.5092076494282414,
+      "tokens_seen": 1684209664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945434119723962,
+      "loss": 1.305,
+      "theoretical_loss": 3.5091842942088247,
+      "tokens_seen": 1684340736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945032900016048,
+      "loss": 1.414,
+      "theoretical_loss": 3.5091609413156357,
+      "tokens_seen": 1684471808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944631680308137,
+      "loss": 1.2808,
+      "theoretical_loss": 3.509137590748262,
+      "tokens_seen": 1684602880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944230460600225,
+      "loss": 1.3258,
+      "theoretical_loss": 3.5091142425062904,
+      "tokens_seen": 1684733952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943829240892313,
+      "loss": 1.2836,
+      "theoretical_loss": 3.5090908965893095,
+      "tokens_seen": 1684865024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943428021184401,
+      "loss": 1.3094,
+      "theoretical_loss": 3.509067552996907,
+      "tokens_seen": 1684996096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943026801476489,
+      "loss": 1.2581,
+      "theoretical_loss": 3.509044211728669,
+      "tokens_seen": 1685127168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942625581768577,
+      "loss": 1.2764,
+      "theoretical_loss": 3.509020872784185,
+      "tokens_seen": 1685258240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942224362060665,
+      "loss": 1.1652,
+      "theoretical_loss": 3.5089975361630428,
+      "tokens_seen": 1685389312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941823142352753,
+      "loss": 1.3245,
+      "theoretical_loss": 3.508974201864829,
+      "tokens_seen": 1685520384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494142192264484,
+      "loss": 1.2989,
+      "theoretical_loss": 3.5089508698891336,
+      "tokens_seen": 1685651456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941020702936929,
+      "loss": 1.2835,
+      "theoretical_loss": 3.508927540235544,
+      "tokens_seen": 1685782528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940619483229016,
+      "loss": 1.3065,
+      "theoretical_loss": 3.5089042129036487,
+      "tokens_seen": 1685913600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940218263521104,
+      "loss": 1.2509,
+      "theoretical_loss": 3.5088808878930364,
+      "tokens_seen": 1686044672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939817043813192,
+      "loss": 1.3255,
+      "theoretical_loss": 3.5088575652032947,
+      "tokens_seen": 1686175744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493941582410528,
+      "loss": 1.2945,
+      "theoretical_loss": 3.5088342448340137,
+      "tokens_seen": 1686306816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939014604397368,
+      "loss": 1.3027,
+      "theoretical_loss": 3.508810926784781,
+      "tokens_seen": 1686437888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938613384689456,
+      "loss": 1.2404,
+      "theoretical_loss": 3.5087876110551868,
+      "tokens_seen": 1686568960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938212164981544,
+      "loss": 1.281,
+      "theoretical_loss": 3.5087642976448183,
+      "tokens_seen": 1686700032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937810945273631,
+      "loss": 1.3281,
+      "theoretical_loss": 3.508740986553266,
+      "tokens_seen": 1686831104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493740972556572,
+      "loss": 1.2565,
+      "theoretical_loss": 3.508717677780119,
+      "tokens_seen": 1686962176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937008505857808,
+      "loss": 1.2558,
+      "theoretical_loss": 3.5086943713249656,
+      "tokens_seen": 1687093248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936607286149896,
+      "loss": 1.3571,
+      "theoretical_loss": 3.5086710671873966,
+      "tokens_seen": 1687224320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936206066441984,
+      "loss": 1.286,
+      "theoretical_loss": 3.5086477653670007,
+      "tokens_seen": 1687355392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935804846734072,
+      "loss": 1.3001,
+      "theoretical_loss": 3.5086244658633676,
+      "tokens_seen": 1687486464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493540362702616,
+      "loss": 1.2365,
+      "theoretical_loss": 3.508601168676088,
+      "tokens_seen": 1687617536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935002407318248,
+      "loss": 1.3805,
+      "theoretical_loss": 3.5085778738047497,
+      "tokens_seen": 1687748608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934601187610335,
+      "loss": 1.3149,
+      "theoretical_loss": 3.5085545812489443,
+      "tokens_seen": 1687879680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934199967902424,
+      "loss": 1.3892,
+      "theoretical_loss": 3.5085312910082616,
+      "tokens_seen": 1688010752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933798748194511,
+      "loss": 1.3257,
+      "theoretical_loss": 3.5085080030822913,
+      "tokens_seen": 1688141824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933397528486599,
+      "loss": 1.3508,
+      "theoretical_loss": 3.5084847174706235,
+      "tokens_seen": 1688272896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932996308778687,
+      "loss": 1.2926,
+      "theoretical_loss": 3.5084614341728497,
+      "tokens_seen": 1688403968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932595089070775,
+      "loss": 1.3173,
+      "theoretical_loss": 3.5084381531885596,
+      "tokens_seen": 1688535040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932193869362863,
+      "loss": 1.3015,
+      "theoretical_loss": 3.508414874517343,
+      "tokens_seen": 1688666112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931792649654951,
+      "loss": 1.3672,
+      "theoretical_loss": 3.5083915981587923,
+      "tokens_seen": 1688797184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931391429947039,
+      "loss": 1.3219,
+      "theoretical_loss": 3.5083683241124968,
+      "tokens_seen": 1688928256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930990210239126,
+      "loss": 1.3287,
+      "theoretical_loss": 3.508345052378048,
+      "tokens_seen": 1689059328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930588990531215,
+      "loss": 1.2355,
+      "theoretical_loss": 3.508321782955037,
+      "tokens_seen": 1689190400
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 568731,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2468483448028564,
+      "objective/train/theoretical_loss": 3.5082985158430544,
+      "objective/train/tokens_used": 59780576,
+      "theoretical_loss": 3.5082985158430544,
+      "tokens_seen": 1689321472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930187770823302,
+      "loss": 1.2696,
+      "theoretical_loss": 3.5082985158430544,
+      "tokens_seen": 1689321472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929786551115392,
+      "loss": 1.2672,
+      "theoretical_loss": 3.5082752510416917,
+      "tokens_seen": 1689452544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929385331407479,
+      "loss": 1.2738,
+      "theoretical_loss": 3.5082519885505405,
+      "tokens_seen": 1689583616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928984111699567,
+      "loss": 1.3175,
+      "theoretical_loss": 3.5082287283691915,
+      "tokens_seen": 1689714688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928582891991655,
+      "loss": 1.2491,
+      "theoretical_loss": 3.508205470497237,
+      "tokens_seen": 1689845760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928181672283743,
+      "loss": 1.2344,
+      "theoretical_loss": 3.5081822149342683,
+      "tokens_seen": 1689976832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927780452575831,
+      "loss": 1.2943,
+      "theoretical_loss": 3.5081589616798765,
+      "tokens_seen": 1690107904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927379232867919,
+      "loss": 1.293,
+      "theoretical_loss": 3.5081357107336544,
+      "tokens_seen": 1690238976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926978013160007,
+      "loss": 1.3144,
+      "theoretical_loss": 3.508112462095193,
+      "tokens_seen": 1690370048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926576793452094,
+      "loss": 1.3388,
+      "theoretical_loss": 3.5080892157640853,
+      "tokens_seen": 1690501120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926175573744183,
+      "loss": 1.3123,
+      "theoretical_loss": 3.508065971739923,
+      "tokens_seen": 1690632192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492577435403627,
+      "loss": 1.2945,
+      "theoretical_loss": 3.508042730022298,
+      "tokens_seen": 1690763264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925373134328358,
+      "loss": 1.2338,
+      "theoretical_loss": 3.5080194906108026,
+      "tokens_seen": 1690894336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924971914620446,
+      "loss": 1.3252,
+      "theoretical_loss": 3.50799625350503,
+      "tokens_seen": 1691025408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924570694912534,
+      "loss": 1.264,
+      "theoretical_loss": 3.507973018704572,
+      "tokens_seen": 1691156480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924169475204622,
+      "loss": 1.3112,
+      "theoretical_loss": 3.507949786209022,
+      "tokens_seen": 1691287552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492376825549671,
+      "loss": 1.3407,
+      "theoretical_loss": 3.507926556017972,
+      "tokens_seen": 1691418624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923367035788798,
+      "loss": 1.2757,
+      "theoretical_loss": 3.507903328131015,
+      "tokens_seen": 1691549696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922965816080887,
+      "loss": 1.2516,
+      "theoretical_loss": 3.507880102547744,
+      "tokens_seen": 1691680768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922564596372974,
+      "loss": 1.298,
+      "theoretical_loss": 3.5078568792677527,
+      "tokens_seen": 1691811840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922163376665062,
+      "loss": 1.3383,
+      "theoretical_loss": 3.5078336582906333,
+      "tokens_seen": 1691942912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492176215695715,
+      "loss": 1.1897,
+      "theoretical_loss": 3.5078104396159797,
+      "tokens_seen": 1692073984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921360937249238,
+      "loss": 1.3168,
+      "theoretical_loss": 3.507787223243385,
+      "tokens_seen": 1692205056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920959717541326,
+      "loss": 1.3229,
+      "theoretical_loss": 3.5077640091724427,
+      "tokens_seen": 1692336128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920558497833414,
+      "loss": 1.239,
+      "theoretical_loss": 3.5077407974027466,
+      "tokens_seen": 1692467200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920157278125502,
+      "loss": 1.3439,
+      "theoretical_loss": 3.5077175879338895,
+      "tokens_seen": 1692598272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919756058417589,
+      "loss": 1.4007,
+      "theoretical_loss": 3.507694380765467,
+      "tokens_seen": 1692729344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919354838709678,
+      "loss": 1.4025,
+      "theoretical_loss": 3.5076711758970713,
+      "tokens_seen": 1692860416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918953619001765,
+      "loss": 1.2642,
+      "theoretical_loss": 3.5076479733282966,
+      "tokens_seen": 1692991488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918552399293853,
+      "loss": 1.3812,
+      "theoretical_loss": 3.5076247730587378,
+      "tokens_seen": 1693122560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918151179585941,
+      "loss": 1.2364,
+      "theoretical_loss": 3.507601575087989,
+      "tokens_seen": 1693253632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917749959878029,
+      "loss": 1.3014,
+      "theoretical_loss": 3.5075783794156434,
+      "tokens_seen": 1693384704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917348740170117,
+      "loss": 1.3655,
+      "theoretical_loss": 3.507555186041297,
+      "tokens_seen": 1693515776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916947520462205,
+      "loss": 1.2644,
+      "theoretical_loss": 3.507531994964543,
+      "tokens_seen": 1693646848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916546300754293,
+      "loss": 1.2977,
+      "theoretical_loss": 3.507508806184976,
+      "tokens_seen": 1693777920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916145081046381,
+      "loss": 1.2603,
+      "theoretical_loss": 3.5074856197021917,
+      "tokens_seen": 1693908992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491574386133847,
+      "loss": 1.2629,
+      "theoretical_loss": 3.5074624355157846,
+      "tokens_seen": 1694040064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915342641630556,
+      "loss": 1.2559,
+      "theoretical_loss": 3.5074392536253494,
+      "tokens_seen": 1694171136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914941421922646,
+      "loss": 1.2267,
+      "theoretical_loss": 3.507416074030481,
+      "tokens_seen": 1694302208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914540202214733,
+      "loss": 1.2857,
+      "theoretical_loss": 3.5073928967307744,
+      "tokens_seen": 1694433280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914138982506821,
+      "loss": 1.2899,
+      "theoretical_loss": 3.5073697217258255,
+      "tokens_seen": 1694564352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913737762798909,
+      "loss": 1.3106,
+      "theoretical_loss": 3.507346549015229,
+      "tokens_seen": 1694695424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913336543090997,
+      "loss": 1.3877,
+      "theoretical_loss": 3.507323378598581,
+      "tokens_seen": 1694826496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912935323383085,
+      "loss": 1.2524,
+      "theoretical_loss": 3.507300210475476,
+      "tokens_seen": 1694957568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912534103675173,
+      "loss": 1.3017,
+      "theoretical_loss": 3.5072770446455106,
+      "tokens_seen": 1695088640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912132883967261,
+      "loss": 1.2845,
+      "theoretical_loss": 3.50725388110828,
+      "tokens_seen": 1695219712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911731664259348,
+      "loss": 1.3238,
+      "theoretical_loss": 3.5072307198633803,
+      "tokens_seen": 1695350784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911330444551437,
+      "loss": 1.3368,
+      "theoretical_loss": 3.507207560910407,
+      "tokens_seen": 1695481856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910929224843524,
+      "loss": 1.3239,
+      "theoretical_loss": 3.507184404248957,
+      "tokens_seen": 1695612928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910528005135612,
+      "loss": 1.3475,
+      "theoretical_loss": 3.507161249878626,
+      "tokens_seen": 1695744000
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 570870,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3209856748580933,
+      "objective/train/theoretical_loss": 3.50713809779901,
+      "objective/train/tokens_used": 66334176,
+      "theoretical_loss": 3.50713809779901,
+      "tokens_seen": 1695875072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049101267854277,
+      "loss": 1.3391,
+      "theoretical_loss": 3.50713809779901,
+      "tokens_seen": 1695875072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909725565719788,
+      "loss": 1.3247,
+      "theoretical_loss": 3.507114948009706,
+      "tokens_seen": 1696006144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909324346011876,
+      "loss": 1.293,
+      "theoretical_loss": 3.507091800510309,
+      "tokens_seen": 1696137216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908923126303964,
+      "loss": 1.2824,
+      "theoretical_loss": 3.5070686553004173,
+      "tokens_seen": 1696268288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908521906596052,
+      "loss": 1.3984,
+      "theoretical_loss": 3.5070455123796265,
+      "tokens_seen": 1696399360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490812068688814,
+      "loss": 1.3013,
+      "theoretical_loss": 3.507022371747534,
+      "tokens_seen": 1696530432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907719467180228,
+      "loss": 1.363,
+      "theoretical_loss": 3.5069992334037368,
+      "tokens_seen": 1696661504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907318247472316,
+      "loss": 1.2967,
+      "theoretical_loss": 3.5069760973478306,
+      "tokens_seen": 1696792576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906917027764404,
+      "loss": 1.2244,
+      "theoretical_loss": 3.5069529635794137,
+      "tokens_seen": 1696923648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906515808056492,
+      "loss": 1.3077,
+      "theoretical_loss": 3.5069298320980824,
+      "tokens_seen": 1697054720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490611458834858,
+      "loss": 1.3061,
+      "theoretical_loss": 3.506906702903435,
+      "tokens_seen": 1697185792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905713368640668,
+      "loss": 1.3462,
+      "theoretical_loss": 3.506883575995068,
+      "tokens_seen": 1697316864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905312148932756,
+      "loss": 1.3275,
+      "theoretical_loss": 3.5068604513725794,
+      "tokens_seen": 1697447936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904910929224843,
+      "loss": 1.2847,
+      "theoretical_loss": 3.5068373290355668,
+      "tokens_seen": 1697579008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904509709516932,
+      "loss": 1.2984,
+      "theoretical_loss": 3.506814208983627,
+      "tokens_seen": 1697710080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904108489809019,
+      "loss": 1.2539,
+      "theoretical_loss": 3.5067910912163587,
+      "tokens_seen": 1697841152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903707270101108,
+      "loss": 1.3076,
+      "theoretical_loss": 3.50676797573336,
+      "tokens_seen": 1697972224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903306050393195,
+      "loss": 1.2532,
+      "theoretical_loss": 3.5067448625342283,
+      "tokens_seen": 1698103296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902904830685283,
+      "loss": 1.3875,
+      "theoretical_loss": 3.5067217516185614,
+      "tokens_seen": 1698234368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902503610977371,
+      "loss": 1.3349,
+      "theoretical_loss": 3.506698642985958,
+      "tokens_seen": 1698365440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902102391269459,
+      "loss": 1.3731,
+      "theoretical_loss": 3.5066755366360165,
+      "tokens_seen": 1698496512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901701171561547,
+      "loss": 1.2714,
+      "theoretical_loss": 3.506652432568335,
+      "tokens_seen": 1698627584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901299951853635,
+      "loss": 1.2716,
+      "theoretical_loss": 3.506629330782512,
+      "tokens_seen": 1698758656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900898732145723,
+      "loss": 1.2846,
+      "theoretical_loss": 3.5066062312781465,
+      "tokens_seen": 1698889728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490049751243781,
+      "loss": 1.2901,
+      "theoretical_loss": 3.5065831340548366,
+      "tokens_seen": 1699020800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049000962927299,
+      "loss": 1.3921,
+      "theoretical_loss": 3.5065600391121814,
+      "tokens_seen": 1699151872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899695073021987,
+      "loss": 1.312,
+      "theoretical_loss": 3.5065369464497795,
+      "tokens_seen": 1699282944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899293853314075,
+      "loss": 1.3109,
+      "theoretical_loss": 3.5065138560672304,
+      "tokens_seen": 1699414016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898892633606163,
+      "loss": 1.348,
+      "theoretical_loss": 3.506490767964133,
+      "tokens_seen": 1699545088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898491413898251,
+      "loss": 1.2699,
+      "theoretical_loss": 3.5064676821400864,
+      "tokens_seen": 1699676160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898090194190339,
+      "loss": 1.3522,
+      "theoretical_loss": 3.5064445985946904,
+      "tokens_seen": 1699807232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897688974482427,
+      "loss": 1.2432,
+      "theoretical_loss": 3.5064215173275435,
+      "tokens_seen": 1699938304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897287754774515,
+      "loss": 1.3016,
+      "theoretical_loss": 3.506398438338246,
+      "tokens_seen": 1700069376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896886535066603,
+      "loss": 1.3224,
+      "theoretical_loss": 3.5063753616263975,
+      "tokens_seen": 1700200448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896485315358691,
+      "loss": 1.2561,
+      "theoretical_loss": 3.5063522871915964,
+      "tokens_seen": 1700331520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896084095650778,
+      "loss": 1.3267,
+      "theoretical_loss": 3.5063292150334444,
+      "tokens_seen": 1700462592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895682875942866,
+      "loss": 1.3576,
+      "theoretical_loss": 3.506306145151541,
+      "tokens_seen": 1700593664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895281656234954,
+      "loss": 1.2614,
+      "theoretical_loss": 3.506283077545485,
+      "tokens_seen": 1700724736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894880436527042,
+      "loss": 1.2738,
+      "theoretical_loss": 3.5062600122148773,
+      "tokens_seen": 1700855808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489447921681913,
+      "loss": 1.2435,
+      "theoretical_loss": 3.5062369491593186,
+      "tokens_seen": 1700986880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894077997111218,
+      "loss": 1.2836,
+      "theoretical_loss": 3.5062138883784084,
+      "tokens_seen": 1701117952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893676777403306,
+      "loss": 1.2858,
+      "theoretical_loss": 3.5061908298717475,
+      "tokens_seen": 1701249024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893275557695395,
+      "loss": 1.296,
+      "theoretical_loss": 3.5061677736389365,
+      "tokens_seen": 1701380096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892874337987481,
+      "loss": 1.2393,
+      "theoretical_loss": 3.506144719679576,
+      "tokens_seen": 1701511168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489247311827957,
+      "loss": 1.2971,
+      "theoretical_loss": 3.5061216679932663,
+      "tokens_seen": 1701642240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892071898571658,
+      "loss": 1.2555,
+      "theoretical_loss": 3.506098618579609,
+      "tokens_seen": 1701773312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891670678863746,
+      "loss": 1.2754,
+      "theoretical_loss": 3.5060755714382044,
+      "tokens_seen": 1701904384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891269459155834,
+      "loss": 1.3006,
+      "theoretical_loss": 3.506052526568653,
+      "tokens_seen": 1702035456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890868239447922,
+      "loss": 1.2484,
+      "theoretical_loss": 3.5060294839705577,
+      "tokens_seen": 1702166528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489046701974001,
+      "loss": 1.307,
+      "theoretical_loss": 3.506006443643518,
+      "tokens_seen": 1702297600
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 572947,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2438222169876099,
+      "objective/train/theoretical_loss": 3.5059834055871364,
+      "objective/train/tokens_used": 72887776,
+      "theoretical_loss": 3.5059834055871364,
+      "tokens_seen": 1702428672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890065800032098,
+      "loss": 1.2549,
+      "theoretical_loss": 3.5059834055871364,
+      "tokens_seen": 1702428672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889664580324186,
+      "loss": 1.3412,
+      "theoretical_loss": 3.5059603698010138,
+      "tokens_seen": 1702559744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889263360616273,
+      "loss": 1.2528,
+      "theoretical_loss": 3.5059373362847515,
+      "tokens_seen": 1702690816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888862140908362,
+      "loss": 1.3339,
+      "theoretical_loss": 3.505914305037951,
+      "tokens_seen": 1702821888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888460921200449,
+      "loss": 1.3278,
+      "theoretical_loss": 3.505891276060215,
+      "tokens_seen": 1702952960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888059701492537,
+      "loss": 1.28,
+      "theoretical_loss": 3.505868249351144,
+      "tokens_seen": 1703084032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887658481784625,
+      "loss": 1.2662,
+      "theoretical_loss": 3.5058452249103413,
+      "tokens_seen": 1703215104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887257262076713,
+      "loss": 1.2981,
+      "theoretical_loss": 3.5058222027374084,
+      "tokens_seen": 1703346176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886856042368801,
+      "loss": 1.2964,
+      "theoretical_loss": 3.505799182831947,
+      "tokens_seen": 1703477248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886454822660889,
+      "loss": 1.2245,
+      "theoretical_loss": 3.5057761651935597,
+      "tokens_seen": 1703608320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886053602952977,
+      "loss": 1.2749,
+      "theoretical_loss": 3.5057531498218486,
+      "tokens_seen": 1703739392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885652383245064,
+      "loss": 1.3278,
+      "theoretical_loss": 3.5057301367164166,
+      "tokens_seen": 1703870464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885251163537154,
+      "loss": 1.1624,
+      "theoretical_loss": 3.5057071258768664,
+      "tokens_seen": 1704001536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884849943829241,
+      "loss": 1.3159,
+      "theoretical_loss": 3.5056841173027995,
+      "tokens_seen": 1704132608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884448724121329,
+      "loss": 1.2829,
+      "theoretical_loss": 3.5056611109938194,
+      "tokens_seen": 1704263680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884047504413417,
+      "loss": 1.2405,
+      "theoretical_loss": 3.5056381069495295,
+      "tokens_seen": 1704394752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883646284705505,
+      "loss": 1.1986,
+      "theoretical_loss": 3.5056151051695315,
+      "tokens_seen": 1704525824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883245064997593,
+      "loss": 1.2289,
+      "theoretical_loss": 3.505592105653429,
+      "tokens_seen": 1704656896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882843845289681,
+      "loss": 1.2947,
+      "theoretical_loss": 3.5055691084008256,
+      "tokens_seen": 1704787968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882442625581769,
+      "loss": 1.2512,
+      "theoretical_loss": 3.5055461134113237,
+      "tokens_seen": 1704919040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048820414058738565,
+      "loss": 1.2968,
+      "theoretical_loss": 3.5055231206845274,
+      "tokens_seen": 1705050112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048816401861659446,
+      "loss": 1.2953,
+      "theoretical_loss": 3.5055001302200397,
+      "tokens_seen": 1705181184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048812389664580326,
+      "loss": 1.2461,
+      "theoretical_loss": 3.505477142017464,
+      "tokens_seen": 1705312256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488083774675012,
+      "loss": 1.3122,
+      "theoretical_loss": 3.505454156076404,
+      "tokens_seen": 1705443328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880436527042209,
+      "loss": 1.3264,
+      "theoretical_loss": 3.505431172396464,
+      "tokens_seen": 1705574400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880035307334296,
+      "loss": 1.308,
+      "theoretical_loss": 3.5054081909772474,
+      "tokens_seen": 1705705472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048796340876263843,
+      "loss": 1.2653,
+      "theoretical_loss": 3.505385211818358,
+      "tokens_seen": 1705836544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048792328679184724,
+      "loss": 1.1756,
+      "theoretical_loss": 3.5053622349194,
+      "tokens_seen": 1705967616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048788316482105605,
+      "loss": 1.265,
+      "theoretical_loss": 3.5053392602799773,
+      "tokens_seen": 1706098688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878430428502648,
+      "loss": 1.2949,
+      "theoretical_loss": 3.505316287899695,
+      "tokens_seen": 1706229760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878029208794736,
+      "loss": 1.2471,
+      "theoretical_loss": 3.5052933177781562,
+      "tokens_seen": 1706360832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877627989086824,
+      "loss": 1.3129,
+      "theoretical_loss": 3.5052703499149658,
+      "tokens_seen": 1706491904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048772267693789116,
+      "loss": 1.3141,
+      "theoretical_loss": 3.5052473843097287,
+      "tokens_seen": 1706622976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876825549671,
+      "loss": 1.2393,
+      "theoretical_loss": 3.5052244209620493,
+      "tokens_seen": 1706754048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876424329963088,
+      "loss": 1.3379,
+      "theoretical_loss": 3.5052014598715324,
+      "tokens_seen": 1706885120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048760231102551764,
+      "loss": 1.2468,
+      "theoretical_loss": 3.5051785010377827,
+      "tokens_seen": 1707016192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875621890547264,
+      "loss": 1.2531,
+      "theoretical_loss": 3.5051555444604054,
+      "tokens_seen": 1707147264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048752206708393514,
+      "loss": 1.2774,
+      "theoretical_loss": 3.505132590139005,
+      "tokens_seen": 1707278336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000487481945113144,
+      "loss": 1.3031,
+      "theoretical_loss": 3.505109638073187,
+      "tokens_seen": 1707409408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048744182314235275,
+      "loss": 1.273,
+      "theoretical_loss": 3.5050866882625566,
+      "tokens_seen": 1707540480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048740170117156156,
+      "loss": 1.3045,
+      "theoretical_loss": 3.5050637407067193,
+      "tokens_seen": 1707671552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048736157920077036,
+      "loss": 1.247,
+      "theoretical_loss": 3.5050407954052805,
+      "tokens_seen": 1707802624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048732145722997917,
+      "loss": 1.2631,
+      "theoretical_loss": 3.5050178523578452,
+      "tokens_seen": 1707933696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872813352591879,
+      "loss": 1.2719,
+      "theoretical_loss": 3.504994911564019,
+      "tokens_seen": 1708064768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048724121328839673,
+      "loss": 1.3176,
+      "theoretical_loss": 3.504971973023409,
+      "tokens_seen": 1708195840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048720109131760554,
+      "loss": 1.1928,
+      "theoretical_loss": 3.504949036735619,
+      "tokens_seen": 1708326912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871609693468143,
+      "loss": 1.2617,
+      "theoretical_loss": 3.5049261027002565,
+      "tokens_seen": 1708457984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048712084737602315,
+      "loss": 1.3029,
+      "theoretical_loss": 3.504903170916927,
+      "tokens_seen": 1708589056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870807254052319,
+      "loss": 1.3029,
+      "theoretical_loss": 3.5048802413852367,
+      "tokens_seen": 1708720128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870406034344407,
+      "loss": 1.2235,
+      "theoretical_loss": 3.5048573141047914,
+      "tokens_seen": 1708851200
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 575138,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3581287860870361,
+      "objective/train/theoretical_loss": 3.5048343890751976,
+      "objective/train/tokens_used": 79441376,
+      "theoretical_loss": 3.5048343890751976,
+      "tokens_seen": 1708982272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870004814636495,
+      "loss": 1.2634,
+      "theoretical_loss": 3.5048343890751976,
+      "tokens_seen": 1708982272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048696035949285827,
+      "loss": 1.3311,
+      "theoretical_loss": 3.5048114662960623,
+      "tokens_seen": 1709113344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869202375220671,
+      "loss": 1.3034,
+      "theoretical_loss": 3.504788545766992,
+      "tokens_seen": 1709244416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868801155512759,
+      "loss": 1.2487,
+      "theoretical_loss": 3.5047656274875925,
+      "tokens_seen": 1709375488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868399935804847,
+      "loss": 1.3384,
+      "theoretical_loss": 3.5047427114574705,
+      "tokens_seen": 1709506560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867998716096935,
+      "loss": 1.2861,
+      "theoretical_loss": 3.5047197976762337,
+      "tokens_seen": 1709637632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867597496389023,
+      "loss": 1.2672,
+      "theoretical_loss": 3.5046968861434884,
+      "tokens_seen": 1709768704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048671962766811105,
+      "loss": 1.2776,
+      "theoretical_loss": 3.5046739768588417,
+      "tokens_seen": 1709899776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048667950569731986,
+      "loss": 1.3054,
+      "theoretical_loss": 3.504651069821901,
+      "tokens_seen": 1710030848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048663938372652866,
+      "loss": 1.3591,
+      "theoretical_loss": 3.504628165032273,
+      "tokens_seen": 1710161920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865992617557374,
+      "loss": 1.3266,
+      "theoretical_loss": 3.504605262489566,
+      "tokens_seen": 1710292992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865591397849463,
+      "loss": 1.2992,
+      "theoretical_loss": 3.5045823621933856,
+      "tokens_seen": 1710424064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486519017814155,
+      "loss": 1.2148,
+      "theoretical_loss": 3.504559464143341,
+      "tokens_seen": 1710555136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048647889584336383,
+      "loss": 1.344,
+      "theoretical_loss": 3.5045365683390397,
+      "tokens_seen": 1710686208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048643877387257264,
+      "loss": 1.3043,
+      "theoretical_loss": 3.5045136747800885,
+      "tokens_seen": 1710817280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048639865190178145,
+      "loss": 1.2376,
+      "theoretical_loss": 3.5044907834660957,
+      "tokens_seen": 1710948352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863585299309902,
+      "loss": 1.3436,
+      "theoretical_loss": 3.5044678943966687,
+      "tokens_seen": 1711079424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486318407960199,
+      "loss": 1.297,
+      "theoretical_loss": 3.504445007571417,
+      "tokens_seen": 1711210496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862782859894078,
+      "loss": 1.2066,
+      "theoretical_loss": 3.5044221229899466,
+      "tokens_seen": 1711341568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862381640186166,
+      "loss": 1.2834,
+      "theoretical_loss": 3.504399240651867,
+      "tokens_seen": 1711472640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861980420478254,
+      "loss": 1.2697,
+      "theoretical_loss": 3.5043763605567864,
+      "tokens_seen": 1711603712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861579200770342,
+      "loss": 1.3236,
+      "theoretical_loss": 3.504353482704313,
+      "tokens_seen": 1711734784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048611779810624304,
+      "loss": 1.3232,
+      "theoretical_loss": 3.5043306070940554,
+      "tokens_seen": 1711865856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860776761354518,
+      "loss": 1.2734,
+      "theoretical_loss": 3.504307733725622,
+      "tokens_seen": 1711996928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048603755416466054,
+      "loss": 1.2674,
+      "theoretical_loss": 3.5042848625986216,
+      "tokens_seen": 1712128000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859974321938694,
+      "loss": 1.2836,
+      "theoretical_loss": 3.504261993712663,
+      "tokens_seen": 1712259072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048595731022307815,
+      "loss": 1.1915,
+      "theoretical_loss": 3.504239127067355,
+      "tokens_seen": 1712390144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048591718825228696,
+      "loss": 1.3479,
+      "theoretical_loss": 3.5042162626623066,
+      "tokens_seen": 1712521216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048587706628149576,
+      "loss": 1.3155,
+      "theoretical_loss": 3.504193400497127,
+      "tokens_seen": 1712652288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048583694431070457,
+      "loss": 1.26,
+      "theoretical_loss": 3.504170540571425,
+      "tokens_seen": 1712783360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857968223399133,
+      "loss": 1.3001,
+      "theoretical_loss": 3.504147682884811,
+      "tokens_seen": 1712914432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048575670036912213,
+      "loss": 1.2064,
+      "theoretical_loss": 3.5041248274368932,
+      "tokens_seen": 1713045504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048571657839833094,
+      "loss": 1.2527,
+      "theoretical_loss": 3.5041019742272814,
+      "tokens_seen": 1713176576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856764564275397,
+      "loss": 1.2908,
+      "theoretical_loss": 3.5040791232555852,
+      "tokens_seen": 1713307648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048563633445674855,
+      "loss": 1.2495,
+      "theoretical_loss": 3.504056274521414,
+      "tokens_seen": 1713438720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855962124859573,
+      "loss": 1.2675,
+      "theoretical_loss": 3.5040334280243783,
+      "tokens_seen": 1713569792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048555609051516616,
+      "loss": 1.2824,
+      "theoretical_loss": 3.504010583764087,
+      "tokens_seen": 1713700864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855159685443749,
+      "loss": 1.3044,
+      "theoretical_loss": 3.5039877417401515,
+      "tokens_seen": 1713831936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048547584657358366,
+      "loss": 1.237,
+      "theoretical_loss": 3.5039649019521804,
+      "tokens_seen": 1713963008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854357246027925,
+      "loss": 1.2497,
+      "theoretical_loss": 3.503942064399784,
+      "tokens_seen": 1714094080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853956026320013,
+      "loss": 1.3093,
+      "theoretical_loss": 3.503919229082573,
+      "tokens_seen": 1714225152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853554806612101,
+      "loss": 1.3441,
+      "theoretical_loss": 3.503896396000158,
+      "tokens_seen": 1714356224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853153586904189,
+      "loss": 1.3401,
+      "theoretical_loss": 3.5038735651521495,
+      "tokens_seen": 1714487296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852752367196277,
+      "loss": 1.3646,
+      "theoretical_loss": 3.5038507365381575,
+      "tokens_seen": 1714618368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048523511474883645,
+      "loss": 1.2524,
+      "theoretical_loss": 3.503827910157793,
+      "tokens_seen": 1714749440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048519499277804525,
+      "loss": 1.302,
+      "theoretical_loss": 3.5038050860106664,
+      "tokens_seen": 1714880512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048515487080725406,
+      "loss": 1.2968,
+      "theoretical_loss": 3.5037822640963885,
+      "tokens_seen": 1715011584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851147488364628,
+      "loss": 1.3509,
+      "theoretical_loss": 3.503759444414571,
+      "tokens_seen": 1715142656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850746268656717,
+      "loss": 1.2825,
+      "theoretical_loss": 3.503736626964824,
+      "tokens_seen": 1715273728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850345048948804,
+      "loss": 1.3955,
+      "theoretical_loss": 3.5037138117467594,
+      "tokens_seen": 1715404800
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 577290,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2057476043701172,
+      "objective/train/theoretical_loss": 3.5036909987599882,
+      "objective/train/tokens_used": 85994976,
+      "theoretical_loss": 3.5036909987599882,
+      "tokens_seen": 1715535872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849943829240893,
+      "loss": 1.2811,
+      "theoretical_loss": 3.5036909987599882,
+      "tokens_seen": 1715535872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048495426095329804,
+      "loss": 1.3022,
+      "theoretical_loss": 3.503668188004121,
+      "tokens_seen": 1715666944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048491413898250684,
+      "loss": 1.3167,
+      "theoretical_loss": 3.5036453794787707,
+      "tokens_seen": 1715798016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048487401701171565,
+      "loss": 1.3069,
+      "theoretical_loss": 3.5036225731835478,
+      "tokens_seen": 1715929088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848338950409244,
+      "loss": 1.3149,
+      "theoretical_loss": 3.503599769118064,
+      "tokens_seen": 1716060160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847937730701332,
+      "loss": 1.3028,
+      "theoretical_loss": 3.503576967281931,
+      "tokens_seen": 1716191232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484753651099342,
+      "loss": 1.2528,
+      "theoretical_loss": 3.5035541676747606,
+      "tokens_seen": 1716322304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847135291285508,
+      "loss": 1.2597,
+      "theoretical_loss": 3.503531370296165,
+      "tokens_seen": 1716453376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846734071577596,
+      "loss": 1.2565,
+      "theoretical_loss": 3.503508575145756,
+      "tokens_seen": 1716584448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048463328518696843,
+      "loss": 1.3146,
+      "theoretical_loss": 3.5034857822231458,
+      "tokens_seen": 1716715520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845931632161772,
+      "loss": 1.2794,
+      "theoretical_loss": 3.5034629915279467,
+      "tokens_seen": 1716846592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048455304124538594,
+      "loss": 1.2692,
+      "theoretical_loss": 3.5034402030597707,
+      "tokens_seen": 1716977664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845129192745948,
+      "loss": 1.2761,
+      "theoretical_loss": 3.5034174168182304,
+      "tokens_seen": 1717108736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048447279730380355,
+      "loss": 1.2828,
+      "theoretical_loss": 3.5033946328029386,
+      "tokens_seen": 1717239808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048443267533301236,
+      "loss": 1.2532,
+      "theoretical_loss": 3.5033718510135072,
+      "tokens_seen": 1717370880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048439255336222116,
+      "loss": 1.3494,
+      "theoretical_loss": 3.5033490714495494,
+      "tokens_seen": 1717501952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048435243139142997,
+      "loss": 1.3019,
+      "theoretical_loss": 3.5033262941106784,
+      "tokens_seen": 1717633024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843123094206388,
+      "loss": 1.256,
+      "theoretical_loss": 3.503303518996506,
+      "tokens_seen": 1717764096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048427218744984753,
+      "loss": 1.2779,
+      "theoretical_loss": 3.503280746106646,
+      "tokens_seen": 1717895168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048423206547905634,
+      "loss": 1.2746,
+      "theoretical_loss": 3.503257975440711,
+      "tokens_seen": 1718026240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048419194350826514,
+      "loss": 1.3365,
+      "theoretical_loss": 3.5032352069983146,
+      "tokens_seen": 1718157312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048415182153747395,
+      "loss": 1.3406,
+      "theoretical_loss": 3.5032124407790697,
+      "tokens_seen": 1718288384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841116995666827,
+      "loss": 1.3641,
+      "theoretical_loss": 3.5031896767825894,
+      "tokens_seen": 1718419456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048407157759589156,
+      "loss": 1.3499,
+      "theoretical_loss": 3.503166915008488,
+      "tokens_seen": 1718550528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840314556251003,
+      "loss": 1.2482,
+      "theoretical_loss": 3.5031441554563787,
+      "tokens_seen": 1718681600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048399133365430906,
+      "loss": 1.3492,
+      "theoretical_loss": 3.503121398125875,
+      "tokens_seen": 1718812672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839512116835179,
+      "loss": 1.2835,
+      "theoretical_loss": 3.5030986430165907,
+      "tokens_seen": 1718943744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839110897127267,
+      "loss": 1.3241,
+      "theoretical_loss": 3.50307589012814,
+      "tokens_seen": 1719074816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838709677419355,
+      "loss": 1.3304,
+      "theoretical_loss": 3.5030531394601363,
+      "tokens_seen": 1719205888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838308457711443,
+      "loss": 1.2795,
+      "theoretical_loss": 3.503030391012194,
+      "tokens_seen": 1719336960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837907238003531,
+      "loss": 1.3479,
+      "theoretical_loss": 3.503007644783928,
+      "tokens_seen": 1719468032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837506018295619,
+      "loss": 1.3592,
+      "theoretical_loss": 3.5029849007749503,
+      "tokens_seen": 1719599104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048371047985877065,
+      "loss": 1.2987,
+      "theoretical_loss": 3.502962158984877,
+      "tokens_seen": 1719730176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048367035788797946,
+      "loss": 1.394,
+      "theoretical_loss": 3.5029394194133223,
+      "tokens_seen": 1719861248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048363023591718827,
+      "loss": 1.271,
+      "theoretical_loss": 3.502916682059901,
+      "tokens_seen": 1719992320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835901139463971,
+      "loss": 1.3081,
+      "theoretical_loss": 3.502893946924227,
+      "tokens_seen": 1720123392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835499919756058,
+      "loss": 1.2758,
+      "theoretical_loss": 3.502871214005915,
+      "tokens_seen": 1720254464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835098700048147,
+      "loss": 1.3405,
+      "theoretical_loss": 3.5028484833045805,
+      "tokens_seen": 1720385536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048346974803402344,
+      "loss": 1.3166,
+      "theoretical_loss": 3.5028257548198383,
+      "tokens_seen": 1720516608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834296260632322,
+      "loss": 1.3006,
+      "theoretical_loss": 3.5028030285513028,
+      "tokens_seen": 1720647680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048338950409244105,
+      "loss": 1.3237,
+      "theoretical_loss": 3.5027803044985895,
+      "tokens_seen": 1720778752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833493821216498,
+      "loss": 1.3289,
+      "theoretical_loss": 3.5027575826613138,
+      "tokens_seen": 1720909824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833092601508586,
+      "loss": 1.2296,
+      "theoretical_loss": 3.50273486303909,
+      "tokens_seen": 1721040896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832691381800674,
+      "loss": 1.3477,
+      "theoretical_loss": 3.502712145631535,
+      "tokens_seen": 1721171968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832290162092762,
+      "loss": 1.2698,
+      "theoretical_loss": 3.502689430438264,
+      "tokens_seen": 1721303040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483188894238485,
+      "loss": 1.2639,
+      "theoretical_loss": 3.5026667174588915,
+      "tokens_seen": 1721434112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048314877226769383,
+      "loss": 1.3207,
+      "theoretical_loss": 3.5026440066930338,
+      "tokens_seen": 1721565184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831086502969026,
+      "loss": 1.283,
+      "theoretical_loss": 3.5026212981403066,
+      "tokens_seen": 1721696256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830685283261114,
+      "loss": 1.3006,
+      "theoretical_loss": 3.5025985918003264,
+      "tokens_seen": 1721827328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830284063553202,
+      "loss": 1.1998,
+      "theoretical_loss": 3.502575887672708,
+      "tokens_seen": 1721958400
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 579595,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.297034740447998,
+      "objective/train/theoretical_loss": 3.502553185757069,
+      "objective/train/tokens_used": 92548576,
+      "theoretical_loss": 3.502553185757069,
+      "tokens_seen": 1722089472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048298828438452895,
+      "loss": 1.2605,
+      "theoretical_loss": 3.502553185757069,
+      "tokens_seen": 1722089472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829481624137378,
+      "loss": 1.2734,
+      "theoretical_loss": 3.5025304860530238,
+      "tokens_seen": 1722220544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048290804044294656,
+      "loss": 1.3063,
+      "theoretical_loss": 3.50250778856019,
+      "tokens_seen": 1722351616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048286791847215537,
+      "loss": 1.3269,
+      "theoretical_loss": 3.5024850932781835,
+      "tokens_seen": 1722482688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828277965013642,
+      "loss": 1.381,
+      "theoretical_loss": 3.5024624002066203,
+      "tokens_seen": 1722613760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048278767453057293,
+      "loss": 1.3247,
+      "theoretical_loss": 3.5024397093451176,
+      "tokens_seen": 1722744832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048274755255978173,
+      "loss": 1.2907,
+      "theoretical_loss": 3.502417020693292,
+      "tokens_seen": 1722875904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048270743058899054,
+      "loss": 1.2766,
+      "theoretical_loss": 3.50239433425076,
+      "tokens_seen": 1723006976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048266730861819935,
+      "loss": 1.2901,
+      "theoretical_loss": 3.502371650017138,
+      "tokens_seen": 1723138048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826271866474081,
+      "loss": 1.244,
+      "theoretical_loss": 3.5023489679920443,
+      "tokens_seen": 1723269120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048258706467661696,
+      "loss": 1.2454,
+      "theoretical_loss": 3.5023262881750945,
+      "tokens_seen": 1723400192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825469427058257,
+      "loss": 1.2511,
+      "theoretical_loss": 3.5023036105659062,
+      "tokens_seen": 1723531264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048250682073503446,
+      "loss": 1.2535,
+      "theoretical_loss": 3.5022809351640967,
+      "tokens_seen": 1723662336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824666987642433,
+      "loss": 1.2385,
+      "theoretical_loss": 3.502258261969283,
+      "tokens_seen": 1723793408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824265767934521,
+      "loss": 1.2945,
+      "theoretical_loss": 3.5022355909810834,
+      "tokens_seen": 1723924480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048238645482266094,
+      "loss": 1.2943,
+      "theoretical_loss": 3.5022129221991145,
+      "tokens_seen": 1724055552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823463328518697,
+      "loss": 1.2837,
+      "theoretical_loss": 3.5021902556229945,
+      "tokens_seen": 1724186624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004823062108810785,
+      "loss": 1.2532,
+      "theoretical_loss": 3.5021675912523405,
+      "tokens_seen": 1724317696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822660889102873,
+      "loss": 1.2824,
+      "theoretical_loss": 3.50214492908677,
+      "tokens_seen": 1724448768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048222596693949605,
+      "loss": 1.2469,
+      "theoretical_loss": 3.502122269125902,
+      "tokens_seen": 1724579840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048218584496870486,
+      "loss": 1.2781,
+      "theoretical_loss": 3.5020996113693537,
+      "tokens_seen": 1724710912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048214572299791367,
+      "loss": 1.3626,
+      "theoretical_loss": 3.5020769558167437,
+      "tokens_seen": 1724841984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821056010271225,
+      "loss": 1.3409,
+      "theoretical_loss": 3.50205430246769,
+      "tokens_seen": 1724973056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820654790563312,
+      "loss": 1.192,
+      "theoretical_loss": 3.50203165132181,
+      "tokens_seen": 1725104128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820253570855401,
+      "loss": 1.429,
+      "theoretical_loss": 3.502009002378723,
+      "tokens_seen": 1725235200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048198523511474884,
+      "loss": 1.3295,
+      "theoretical_loss": 3.5019863556380475,
+      "tokens_seen": 1725366272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004819451131439576,
+      "loss": 1.2996,
+      "theoretical_loss": 3.5019637110994015,
+      "tokens_seen": 1725497344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048190499117316645,
+      "loss": 1.2954,
+      "theoretical_loss": 3.501941068762404,
+      "tokens_seen": 1725628416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818648692023752,
+      "loss": 1.2858,
+      "theoretical_loss": 3.5019184286266736,
+      "tokens_seen": 1725759488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048182474723158406,
+      "loss": 1.266,
+      "theoretical_loss": 3.5018957906918295,
+      "tokens_seen": 1725890560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817846252607928,
+      "loss": 1.284,
+      "theoretical_loss": 3.5018731549574897,
+      "tokens_seen": 1726021632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817445032900016,
+      "loss": 1.2774,
+      "theoretical_loss": 3.501850521423274,
+      "tokens_seen": 1726152704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048170438131921043,
+      "loss": 1.2957,
+      "theoretical_loss": 3.501827890088802,
+      "tokens_seen": 1726283776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048166425934841923,
+      "loss": 1.2929,
+      "theoretical_loss": 3.5018052609536916,
+      "tokens_seen": 1726414848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481624137377628,
+      "loss": 1.3453,
+      "theoretical_loss": 3.5017826340175624,
+      "tokens_seen": 1726545920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815840154068368,
+      "loss": 1.2689,
+      "theoretical_loss": 3.501760009280035,
+      "tokens_seen": 1726676992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815438934360456,
+      "loss": 1.3148,
+      "theoretical_loss": 3.5017373867407278,
+      "tokens_seen": 1726808064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048150377146525435,
+      "loss": 1.2324,
+      "theoretical_loss": 3.5017147663992603,
+      "tokens_seen": 1726939136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004814636494944632,
+      "loss": 1.2249,
+      "theoretical_loss": 3.5016921482552528,
+      "tokens_seen": 1727070208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048142352752367196,
+      "loss": 1.2618,
+      "theoretical_loss": 3.501669532308325,
+      "tokens_seen": 1727201280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048138340555288077,
+      "loss": 1.2508,
+      "theoretical_loss": 3.5016469185580963,
+      "tokens_seen": 1727332352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813432835820896,
+      "loss": 1.2241,
+      "theoretical_loss": 3.501624307004187,
+      "tokens_seen": 1727463424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048130316161129833,
+      "loss": 1.3003,
+      "theoretical_loss": 3.501601697646217,
+      "tokens_seen": 1727594496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048126303964050713,
+      "loss": 1.2411,
+      "theoretical_loss": 3.501579090483807,
+      "tokens_seen": 1727725568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048122291766971594,
+      "loss": 1.2712,
+      "theoretical_loss": 3.5015564855165766,
+      "tokens_seen": 1727856640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048118279569892475,
+      "loss": 1.2593,
+      "theoretical_loss": 3.5015338827441465,
+      "tokens_seen": 1727987712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048114267372813355,
+      "loss": 1.2162,
+      "theoretical_loss": 3.5015112821661365,
+      "tokens_seen": 1728118784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048110255175734236,
+      "loss": 1.209,
+      "theoretical_loss": 3.5014886837821684,
+      "tokens_seen": 1728249856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810624297865511,
+      "loss": 1.2359,
+      "theoretical_loss": 3.5014660875918615,
+      "tokens_seen": 1728380928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810223078157599,
+      "loss": 1.3215,
+      "theoretical_loss": 3.5014434935948375,
+      "tokens_seen": 1728512000
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 581638,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.257768154144287,
+      "objective/train/theoretical_loss": 3.5014209017907167,
+      "objective/train/tokens_used": 99102176,
+      "theoretical_loss": 3.5014209017907167,
+      "tokens_seen": 1728643072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809821858449687,
+      "loss": 1.2666,
+      "theoretical_loss": 3.5014209017907167,
+      "tokens_seen": 1728643072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809420638741775,
+      "loss": 1.2947,
+      "theoretical_loss": 3.50139831217912,
+      "tokens_seen": 1728774144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048090194190338634,
+      "loss": 1.2922,
+      "theoretical_loss": 3.5013757247596686,
+      "tokens_seen": 1728905216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808618199325951,
+      "loss": 1.3036,
+      "theoretical_loss": 3.501353139531984,
+      "tokens_seen": 1729036288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808216979618039,
+      "loss": 1.2818,
+      "theoretical_loss": 3.5013305564956867,
+      "tokens_seen": 1729167360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807815759910127,
+      "loss": 1.1881,
+      "theoretical_loss": 3.5013079756503984,
+      "tokens_seen": 1729298432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048074145402022145,
+      "loss": 1.2901,
+      "theoretical_loss": 3.5012853969957396,
+      "tokens_seen": 1729429504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048070133204943026,
+      "loss": 1.2915,
+      "theoretical_loss": 3.501262820531333,
+      "tokens_seen": 1729560576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048066121007863907,
+      "loss": 1.1903,
+      "theoretical_loss": 3.5012402462568,
+      "tokens_seen": 1729691648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048062108810784787,
+      "loss": 1.2729,
+      "theoretical_loss": 3.5012176741717616,
+      "tokens_seen": 1729822720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805809661370567,
+      "loss": 1.3178,
+      "theoretical_loss": 3.50119510427584,
+      "tokens_seen": 1729953792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805408441662655,
+      "loss": 1.196,
+      "theoretical_loss": 3.501172536568657,
+      "tokens_seen": 1730084864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048050072219547424,
+      "loss": 1.2711,
+      "theoretical_loss": 3.501149971049834,
+      "tokens_seen": 1730215936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048046060022468304,
+      "loss": 1.169,
+      "theoretical_loss": 3.5011274077189944,
+      "tokens_seen": 1730347008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048042047825389185,
+      "loss": 1.2576,
+      "theoretical_loss": 3.5011048465757595,
+      "tokens_seen": 1730478080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803803562831006,
+      "loss": 1.2595,
+      "theoretical_loss": 3.501082287619751,
+      "tokens_seen": 1730609152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048034023431230946,
+      "loss": 1.3634,
+      "theoretical_loss": 3.501059730850592,
+      "tokens_seen": 1730740224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803001123415182,
+      "loss": 1.3193,
+      "theoretical_loss": 3.5010371762679044,
+      "tokens_seen": 1730871296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480259990370727,
+      "loss": 1.3301,
+      "theoretical_loss": 3.501014623871311,
+      "tokens_seen": 1731002368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048021986839993583,
+      "loss": 1.2331,
+      "theoretical_loss": 3.5009920736604347,
+      "tokens_seen": 1731133440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048017974642914463,
+      "loss": 1.21,
+      "theoretical_loss": 3.500969525634898,
+      "tokens_seen": 1731264512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801396244583534,
+      "loss": 1.2178,
+      "theoretical_loss": 3.5009469797943233,
+      "tokens_seen": 1731395584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800995024875622,
+      "loss": 1.216,
+      "theoretical_loss": 3.500924436138334,
+      "tokens_seen": 1731526656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480059380516771,
+      "loss": 1.3047,
+      "theoretical_loss": 3.500901894666552,
+      "tokens_seen": 1731657728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048001925854597975,
+      "loss": 1.2864,
+      "theoretical_loss": 3.5008793553786024,
+      "tokens_seen": 1731788800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799791365751886,
+      "loss": 1.3153,
+      "theoretical_loss": 3.500856818274107,
+      "tokens_seen": 1731919872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047993901460439736,
+      "loss": 1.1886,
+      "theoretical_loss": 3.5008342833526886,
+      "tokens_seen": 1732050944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798988926336062,
+      "loss": 1.2837,
+      "theoretical_loss": 3.5008117506139715,
+      "tokens_seen": 1732182016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000479858770662815,
+      "loss": 1.2867,
+      "theoretical_loss": 3.500789220057579,
+      "tokens_seen": 1732313088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047981864869202373,
+      "loss": 1.3196,
+      "theoretical_loss": 3.5007666916831344,
+      "tokens_seen": 1732444160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797785267212326,
+      "loss": 1.3174,
+      "theoretical_loss": 3.5007441654902616,
+      "tokens_seen": 1732575232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047973840475044134,
+      "loss": 1.2453,
+      "theoretical_loss": 3.5007216414785836,
+      "tokens_seen": 1732706304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047969828277965015,
+      "loss": 1.3487,
+      "theoretical_loss": 3.5006991196477255,
+      "tokens_seen": 1732837376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047965816080885895,
+      "loss": 1.1872,
+      "theoretical_loss": 3.50067659999731,
+      "tokens_seen": 1732968448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047961803883806776,
+      "loss": 1.2922,
+      "theoretical_loss": 3.500654082526962,
+      "tokens_seen": 1733099520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795779168672765,
+      "loss": 1.2234,
+      "theoretical_loss": 3.5006315672363053,
+      "tokens_seen": 1733230592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795377948964853,
+      "loss": 1.2851,
+      "theoretical_loss": 3.5006090541249635,
+      "tokens_seen": 1733361664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794976729256941,
+      "loss": 1.1955,
+      "theoretical_loss": 3.5005865431925614,
+      "tokens_seen": 1733492736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794575509549029,
+      "loss": 1.2356,
+      "theoretical_loss": 3.5005640344387237,
+      "tokens_seen": 1733623808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047941742898411174,
+      "loss": 1.262,
+      "theoretical_loss": 3.5005415278630743,
+      "tokens_seen": 1733754880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793773070133205,
+      "loss": 1.2849,
+      "theoretical_loss": 3.5005190234652384,
+      "tokens_seen": 1733885952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047933718504252935,
+      "loss": 1.1741,
+      "theoretical_loss": 3.5004965212448402,
+      "tokens_seen": 1734017024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792970630717381,
+      "loss": 1.1868,
+      "theoretical_loss": 3.500474021201504,
+      "tokens_seen": 1734148096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047925694110094685,
+      "loss": 1.2592,
+      "theoretical_loss": 3.500451523334856,
+      "tokens_seen": 1734279168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792168191301557,
+      "loss": 1.2543,
+      "theoretical_loss": 3.50042902764452,
+      "tokens_seen": 1734410240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047917669715936447,
+      "loss": 1.2533,
+      "theoretical_loss": 3.5004065341301214,
+      "tokens_seen": 1734541312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047913657518857327,
+      "loss": 1.3025,
+      "theoretical_loss": 3.5003840427912847,
+      "tokens_seen": 1734672384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790964532177821,
+      "loss": 1.2489,
+      "theoretical_loss": 3.500361553627636,
+      "tokens_seen": 1734803456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790563312469909,
+      "loss": 1.3069,
+      "theoretical_loss": 3.5003390666388006,
+      "tokens_seen": 1734934528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047901620927619964,
+      "loss": 1.2439,
+      "theoretical_loss": 3.5003165818244035,
+      "tokens_seen": 1735065600
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 583892,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3189393281936646,
+      "objective/train/theoretical_loss": 3.50029409918407,
+      "objective/train/tokens_used": 105655776,
+      "theoretical_loss": 3.50029409918407,
+      "tokens_seen": 1735196672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047897608730540844,
+      "loss": 1.3733,
+      "theoretical_loss": 3.50029409918407,
+      "tokens_seen": 1735196672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047893596533461725,
+      "loss": 1.2426,
+      "theoretical_loss": 3.5002716187174263,
+      "tokens_seen": 1735327744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478895843363826,
+      "loss": 1.3011,
+      "theoretical_loss": 3.5002491404240983,
+      "tokens_seen": 1735458816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047885572139303486,
+      "loss": 1.3247,
+      "theoretical_loss": 3.5002266643037103,
+      "tokens_seen": 1735589888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788155994222436,
+      "loss": 1.2703,
+      "theoretical_loss": 3.50020419035589,
+      "tokens_seen": 1735720960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787754774514524,
+      "loss": 1.3175,
+      "theoretical_loss": 3.500181718580262,
+      "tokens_seen": 1735852032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787353554806612,
+      "loss": 1.335,
+      "theoretical_loss": 3.500159248976453,
+      "tokens_seen": 1735983104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047869523350987,
+      "loss": 1.2427,
+      "theoretical_loss": 3.5001367815440885,
+      "tokens_seen": 1736114176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047865511153907884,
+      "loss": 1.2664,
+      "theoretical_loss": 3.500114316282796,
+      "tokens_seen": 1736245248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004786149895682876,
+      "loss": 1.2948,
+      "theoretical_loss": 3.5000918531922007,
+      "tokens_seen": 1736376320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785748675974964,
+      "loss": 1.3073,
+      "theoretical_loss": 3.5000693922719295,
+      "tokens_seen": 1736507392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785347456267052,
+      "loss": 1.1911,
+      "theoretical_loss": 3.5000469335216087,
+      "tokens_seen": 1736638464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478494623655914,
+      "loss": 1.261,
+      "theoretical_loss": 3.5000244769408644,
+      "tokens_seen": 1736769536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047845450168512276,
+      "loss": 1.2991,
+      "theoretical_loss": 3.5000020225293245,
+      "tokens_seen": 1736900608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784143797143316,
+      "loss": 1.269,
+      "theoretical_loss": 3.4999795702866154,
+      "tokens_seen": 1737031680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783742577435404,
+      "loss": 1.1967,
+      "theoretical_loss": 3.499957120212364,
+      "tokens_seen": 1737162752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047833413577274913,
+      "loss": 1.284,
+      "theoretical_loss": 3.499934672306196,
+      "tokens_seen": 1737293824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478294013801958,
+      "loss": 1.2974,
+      "theoretical_loss": 3.4999122265677403,
+      "tokens_seen": 1737424896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047825389183116674,
+      "loss": 1.2879,
+      "theoretical_loss": 3.4998897829966236,
+      "tokens_seen": 1737555968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047821376986037555,
+      "loss": 1.2917,
+      "theoretical_loss": 3.499867341592472,
+      "tokens_seen": 1737687040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047817364788958435,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4998449023549143,
+      "tokens_seen": 1737818112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047813352591879316,
+      "loss": 1.2633,
+      "theoretical_loss": 3.499822465283577,
+      "tokens_seen": 1737949184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780934039480019,
+      "loss": 1.228,
+      "theoretical_loss": 3.499800030378088,
+      "tokens_seen": 1738080256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780532819772107,
+      "loss": 1.1979,
+      "theoretical_loss": 3.4997775976380754,
+      "tokens_seen": 1738211328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780131600064195,
+      "loss": 1.2848,
+      "theoretical_loss": 3.4997551670631664,
+      "tokens_seen": 1738342400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047797303803562833,
+      "loss": 1.388,
+      "theoretical_loss": 3.4997327386529884,
+      "tokens_seen": 1738473472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047793291606483714,
+      "loss": 1.2752,
+      "theoretical_loss": 3.49971031240717,
+      "tokens_seen": 1738604544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778927940940459,
+      "loss": 1.2883,
+      "theoretical_loss": 3.4996878883253384,
+      "tokens_seen": 1738735616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047785267212325475,
+      "loss": 1.2687,
+      "theoretical_loss": 3.4996654664071225,
+      "tokens_seen": 1738866688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778125501524635,
+      "loss": 1.2811,
+      "theoretical_loss": 3.4996430466521504,
+      "tokens_seen": 1738997760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047777242818167225,
+      "loss": 1.251,
+      "theoretical_loss": 3.4996206290600496,
+      "tokens_seen": 1739128832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777323062108811,
+      "loss": 1.2503,
+      "theoretical_loss": 3.4995982136304495,
+      "tokens_seen": 1739259904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047769218424008987,
+      "loss": 1.3038,
+      "theoretical_loss": 3.4995758003629778,
+      "tokens_seen": 1739390976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047765206226929867,
+      "loss": 1.3073,
+      "theoretical_loss": 3.4995533892572634,
+      "tokens_seen": 1739522048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776119402985075,
+      "loss": 1.2392,
+      "theoretical_loss": 3.499530980312935,
+      "tokens_seen": 1739653120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775718183277163,
+      "loss": 1.368,
+      "theoretical_loss": 3.4995085735296207,
+      "tokens_seen": 1739784192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047753169635692504,
+      "loss": 1.2865,
+      "theoretical_loss": 3.49948616890695,
+      "tokens_seen": 1739915264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047749157438613384,
+      "loss": 1.2395,
+      "theoretical_loss": 3.499463766444552,
+      "tokens_seen": 1740046336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047745145241534265,
+      "loss": 1.3148,
+      "theoretical_loss": 3.4994413661420545,
+      "tokens_seen": 1740177408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774113304445514,
+      "loss": 1.3288,
+      "theoretical_loss": 3.499418967999088,
+      "tokens_seen": 1740308480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047737120847376026,
+      "loss": 1.304,
+      "theoretical_loss": 3.499396572015281,
+      "tokens_seen": 1740439552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000477331086502969,
+      "loss": 1.2621,
+      "theoretical_loss": 3.4993741781902625,
+      "tokens_seen": 1740570624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004772909645321779,
+      "loss": 1.212,
+      "theoretical_loss": 3.4993517865236625,
+      "tokens_seen": 1740701696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772508425613866,
+      "loss": 1.3253,
+      "theoretical_loss": 3.49932939701511,
+      "tokens_seen": 1740832768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772107205905954,
+      "loss": 1.2269,
+      "theoretical_loss": 3.4993070096642347,
+      "tokens_seen": 1740963840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047717059861980424,
+      "loss": 1.3123,
+      "theoretical_loss": 3.4992846244706666,
+      "tokens_seen": 1741094912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000477130476649013,
+      "loss": 1.3027,
+      "theoretical_loss": 3.499262241434035,
+      "tokens_seen": 1741225984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770903546782218,
+      "loss": 1.3087,
+      "theoretical_loss": 3.4992398605539696,
+      "tokens_seen": 1741357056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770502327074306,
+      "loss": 1.3382,
+      "theoretical_loss": 3.499217481830101,
+      "tokens_seen": 1741488128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770101107366394,
+      "loss": 1.2534,
+      "theoretical_loss": 3.499195105262058,
+      "tokens_seen": 1741619200
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 586110,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2312313318252563,
+      "objective/train/theoretical_loss": 3.4991727308494722,
+      "objective/train/tokens_used": 112209376,
+      "theoretical_loss": 3.4991727308494722,
+      "tokens_seen": 1741750272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047696998876584816,
+      "loss": 1.2143,
+      "theoretical_loss": 3.4991727308494722,
+      "tokens_seen": 1741750272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476929866795057,
+      "loss": 1.2248,
+      "theoretical_loss": 3.4991503585919723,
+      "tokens_seen": 1741881344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768897448242658,
+      "loss": 1.3207,
+      "theoretical_loss": 3.4991279884891897,
+      "tokens_seen": 1742012416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768496228534745,
+      "loss": 1.3226,
+      "theoretical_loss": 3.499105620540755,
+      "tokens_seen": 1742143488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768095008826834,
+      "loss": 1.228,
+      "theoretical_loss": 3.4990832547462976,
+      "tokens_seen": 1742274560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047676937891189214,
+      "loss": 1.301,
+      "theoretical_loss": 3.499060891105448,
+      "tokens_seen": 1742405632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476729256941101,
+      "loss": 1.2944,
+      "theoretical_loss": 3.499038529617838,
+      "tokens_seen": 1742536704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047668913497030975,
+      "loss": 1.2628,
+      "theoretical_loss": 3.4990161702830975,
+      "tokens_seen": 1742667776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047664901299951856,
+      "loss": 1.3363,
+      "theoretical_loss": 3.4989938131008573,
+      "tokens_seen": 1742798848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047660889102872736,
+      "loss": 1.3066,
+      "theoretical_loss": 3.498971458070749,
+      "tokens_seen": 1742929920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765687690579361,
+      "loss": 1.2497,
+      "theoretical_loss": 3.498949105192403,
+      "tokens_seen": 1743060992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765286470871449,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4989267544654505,
+      "tokens_seen": 1743192064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047648852511635373,
+      "loss": 1.3246,
+      "theoretical_loss": 3.498904405889523,
+      "tokens_seen": 1743323136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047644840314556254,
+      "loss": 1.2804,
+      "theoretical_loss": 3.4988820594642513,
+      "tokens_seen": 1743454208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764082811747713,
+      "loss": 1.1948,
+      "theoretical_loss": 3.4988597151892673,
+      "tokens_seen": 1743585280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047636815920398015,
+      "loss": 1.3166,
+      "theoretical_loss": 3.4988373730642017,
+      "tokens_seen": 1743716352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763280372331889,
+      "loss": 1.3079,
+      "theoretical_loss": 3.498815033088687,
+      "tokens_seen": 1743847424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047628791526239765,
+      "loss": 1.3029,
+      "theoretical_loss": 3.4987926952623543,
+      "tokens_seen": 1743978496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762477932916065,
+      "loss": 1.2495,
+      "theoretical_loss": 3.498770359584835,
+      "tokens_seen": 1744109568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047620767132081527,
+      "loss": 1.278,
+      "theoretical_loss": 3.498748026055762,
+      "tokens_seen": 1744240640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047616754935002407,
+      "loss": 1.2313,
+      "theoretical_loss": 3.498725694674766,
+      "tokens_seen": 1744371712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761274273792329,
+      "loss": 1.3124,
+      "theoretical_loss": 3.49870336544148,
+      "tokens_seen": 1744502784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760873054084417,
+      "loss": 1.276,
+      "theoretical_loss": 3.498681038355536,
+      "tokens_seen": 1744633856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760471834376505,
+      "loss": 1.3021,
+      "theoretical_loss": 3.498658713416565,
+      "tokens_seen": 1744764928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047600706146685924,
+      "loss": 1.3157,
+      "theoretical_loss": 3.498636390624201,
+      "tokens_seen": 1744896000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047596693949606805,
+      "loss": 1.23,
+      "theoretical_loss": 3.498614069978075,
+      "tokens_seen": 1745027072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047592681752527686,
+      "loss": 1.2961,
+      "theoretical_loss": 3.4985917514778206,
+      "tokens_seen": 1745158144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047588669555448566,
+      "loss": 1.2339,
+      "theoretical_loss": 3.498569435123069,
+      "tokens_seen": 1745289216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004758465735836944,
+      "loss": 1.2563,
+      "theoretical_loss": 3.498547120913454,
+      "tokens_seen": 1745420288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004758064516129033,
+      "loss": 1.2859,
+      "theoretical_loss": 3.4985248088486083,
+      "tokens_seen": 1745551360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475766329642112,
+      "loss": 1.3138,
+      "theoretical_loss": 3.498502498928164,
+      "tokens_seen": 1745682432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757262076713208,
+      "loss": 1.2782,
+      "theoretical_loss": 3.4984801911517542,
+      "tokens_seen": 1745813504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047568608570052964,
+      "loss": 1.2498,
+      "theoretical_loss": 3.498457885519012,
+      "tokens_seen": 1745944576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756459637297384,
+      "loss": 1.2827,
+      "theoretical_loss": 3.498435582029571,
+      "tokens_seen": 1746075648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756058417589472,
+      "loss": 1.218,
+      "theoretical_loss": 3.498413280683064,
+      "tokens_seen": 1746206720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475565719788156,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4983909814791243,
+      "tokens_seen": 1746337792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755255978173648,
+      "loss": 1.3425,
+      "theoretical_loss": 3.498368684417385,
+      "tokens_seen": 1746468864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754854758465736,
+      "loss": 1.2504,
+      "theoretical_loss": 3.49834638949748,
+      "tokens_seen": 1746599936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754453538757824,
+      "loss": 1.3159,
+      "theoretical_loss": 3.4983240967190423,
+      "tokens_seen": 1746731008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754052319049912,
+      "loss": 1.1728,
+      "theoretical_loss": 3.498301806081706,
+      "tokens_seen": 1746862080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753651099342,
+      "loss": 1.2861,
+      "theoretical_loss": 3.4982795175851047,
+      "tokens_seen": 1746993152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753249879634088,
+      "loss": 1.2309,
+      "theoretical_loss": 3.498257231228872,
+      "tokens_seen": 1747124224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047528486599261754,
+      "loss": 1.268,
+      "theoretical_loss": 3.4982349470126426,
+      "tokens_seen": 1747255296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752447440218264,
+      "loss": 1.2904,
+      "theoretical_loss": 3.49821266493605,
+      "tokens_seen": 1747386368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047520462205103515,
+      "loss": 1.3587,
+      "theoretical_loss": 3.4981903849987277,
+      "tokens_seen": 1747517440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047516450008024396,
+      "loss": 1.2384,
+      "theoretical_loss": 3.4981681072003106,
+      "tokens_seen": 1747648512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047512437810945276,
+      "loss": 1.2626,
+      "theoretical_loss": 3.498145831540433,
+      "tokens_seen": 1747779584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750842561386615,
+      "loss": 1.2855,
+      "theoretical_loss": 3.4981235580187287,
+      "tokens_seen": 1747910656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750441341678703,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4981012866348324,
+      "tokens_seen": 1748041728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047500401219707913,
+      "loss": 1.3097,
+      "theoretical_loss": 3.498079017388379,
+      "tokens_seen": 1748172800
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 588356,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1916943788528442,
+      "objective/train/theoretical_loss": 3.4980567502790025,
+      "objective/train/tokens_used": 118762976,
+      "theoretical_loss": 3.4980567502790025,
+      "tokens_seen": 1748303872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047496389022628794,
+      "loss": 1.301,
+      "theoretical_loss": 3.4980567502790025,
+      "tokens_seen": 1748303872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004749237682554967,
+      "loss": 1.2844,
+      "theoretical_loss": 3.498034485306338,
+      "tokens_seen": 1748434944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047488364628470555,
+      "loss": 1.3659,
+      "theoretical_loss": 3.4980122224700203,
+      "tokens_seen": 1748566016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748435243139143,
+      "loss": 1.2649,
+      "theoretical_loss": 3.497989961769685,
+      "tokens_seen": 1748697088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748034023431231,
+      "loss": 1.3465,
+      "theoretical_loss": 3.4979677032049654,
+      "tokens_seen": 1748828160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747632803723319,
+      "loss": 1.2947,
+      "theoretical_loss": 3.497945446775498,
+      "tokens_seen": 1748959232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047472315840154066,
+      "loss": 1.2035,
+      "theoretical_loss": 3.4979231924809175,
+      "tokens_seen": 1749090304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746830364307495,
+      "loss": 1.2577,
+      "theoretical_loss": 3.497900940320859,
+      "tokens_seen": 1749221376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746429144599583,
+      "loss": 1.2572,
+      "theoretical_loss": 3.497878690294958,
+      "tokens_seen": 1749352448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746027924891671,
+      "loss": 1.2589,
+      "theoretical_loss": 3.49785644240285,
+      "tokens_seen": 1749483520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745626705183759,
+      "loss": 1.3133,
+      "theoretical_loss": 3.4978341966441704,
+      "tokens_seen": 1749614592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047452254854758464,
+      "loss": 1.231,
+      "theoretical_loss": 3.497811953018555,
+      "tokens_seen": 1749745664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047448242657679345,
+      "loss": 1.3289,
+      "theoretical_loss": 3.4977897115256393,
+      "tokens_seen": 1749876736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047444230460600225,
+      "loss": 1.3021,
+      "theoretical_loss": 3.4977674721650596,
+      "tokens_seen": 1750007808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047440218263521106,
+      "loss": 1.2745,
+      "theoretical_loss": 3.497745234936451,
+      "tokens_seen": 1750138880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743620606644198,
+      "loss": 1.3082,
+      "theoretical_loss": 3.49772299983945,
+      "tokens_seen": 1750269952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743219386936287,
+      "loss": 1.2416,
+      "theoretical_loss": 3.497700766873692,
+      "tokens_seen": 1750401024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742818167228374,
+      "loss": 1.3152,
+      "theoretical_loss": 3.4976785360388143,
+      "tokens_seen": 1750532096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742416947520462,
+      "loss": 1.3097,
+      "theoretical_loss": 3.497656307334452,
+      "tokens_seen": 1750663168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047420157278125504,
+      "loss": 1.2693,
+      "theoretical_loss": 3.4976340807602426,
+      "tokens_seen": 1750794240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741614508104638,
+      "loss": 1.2809,
+      "theoretical_loss": 3.497611856315821,
+      "tokens_seen": 1750925312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047412132883967265,
+      "loss": 1.2956,
+      "theoretical_loss": 3.4975896340008252,
+      "tokens_seen": 1751056384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740812068688814,
+      "loss": 1.241,
+      "theoretical_loss": 3.497567413814891,
+      "tokens_seen": 1751187456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740410848980902,
+      "loss": 1.3205,
+      "theoretical_loss": 3.497545195757655,
+      "tokens_seen": 1751318528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474000962927299,
+      "loss": 1.2917,
+      "theoretical_loss": 3.4975229798287546,
+      "tokens_seen": 1751449600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047396084095650777,
+      "loss": 1.2823,
+      "theoretical_loss": 3.4975007660278266,
+      "tokens_seen": 1751580672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004739207189857166,
+      "loss": 1.2909,
+      "theoretical_loss": 3.497478554354507,
+      "tokens_seen": 1751711744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738805970149254,
+      "loss": 1.3113,
+      "theoretical_loss": 3.4974563448084335,
+      "tokens_seen": 1751842816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738404750441342,
+      "loss": 1.3046,
+      "theoretical_loss": 3.4974341373892432,
+      "tokens_seen": 1751973888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047380035307334294,
+      "loss": 1.3574,
+      "theoretical_loss": 3.4974119320965738,
+      "tokens_seen": 1752104960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737602311025518,
+      "loss": 1.3465,
+      "theoretical_loss": 3.497389728930062,
+      "tokens_seen": 1752236032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047372010913176055,
+      "loss": 1.36,
+      "theoretical_loss": 3.4973675278893452,
+      "tokens_seen": 1752367104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047367998716096936,
+      "loss": 1.2987,
+      "theoretical_loss": 3.497345328974061,
+      "tokens_seen": 1752498176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047363986519017816,
+      "loss": 1.3045,
+      "theoretical_loss": 3.4973231321838476,
+      "tokens_seen": 1752629248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004735997432193869,
+      "loss": 1.3048,
+      "theoretical_loss": 3.4973009375183413,
+      "tokens_seen": 1752760320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004735596212485958,
+      "loss": 1.3195,
+      "theoretical_loss": 3.497278744977181,
+      "tokens_seen": 1752891392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047351949927780453,
+      "loss": 1.2953,
+      "theoretical_loss": 3.4972565545600043,
+      "tokens_seen": 1753022464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047347937730701334,
+      "loss": 1.3772,
+      "theoretical_loss": 3.497234366266449,
+      "tokens_seen": 1753153536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047343925533622214,
+      "loss": 1.2615,
+      "theoretical_loss": 3.4972121800961533,
+      "tokens_seen": 1753284608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047339913336543095,
+      "loss": 1.2822,
+      "theoretical_loss": 3.4971899960487547,
+      "tokens_seen": 1753415680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733590113946397,
+      "loss": 1.3105,
+      "theoretical_loss": 3.497167814123892,
+      "tokens_seen": 1753546752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733188894238485,
+      "loss": 1.3213,
+      "theoretical_loss": 3.4971456343212037,
+      "tokens_seen": 1753677824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732787674530573,
+      "loss": 1.2597,
+      "theoretical_loss": 3.4971234566403275,
+      "tokens_seen": 1753808896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047323864548226606,
+      "loss": 1.2744,
+      "theoretical_loss": 3.4971012810809023,
+      "tokens_seen": 1753939968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731985235114749,
+      "loss": 1.3734,
+      "theoretical_loss": 3.497079107642567,
+      "tokens_seen": 1754071040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731584015406837,
+      "loss": 1.3371,
+      "theoretical_loss": 3.4970569363249595,
+      "tokens_seen": 1754202112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731182795698925,
+      "loss": 1.2685,
+      "theoretical_loss": 3.497034767127719,
+      "tokens_seen": 1754333184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730781575991013,
+      "loss": 1.2895,
+      "theoretical_loss": 3.4970126000504838,
+      "tokens_seen": 1754464256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047303803562831004,
+      "loss": 1.278,
+      "theoretical_loss": 3.496990435092893,
+      "tokens_seen": 1754595328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047299791365751885,
+      "loss": 1.2793,
+      "theoretical_loss": 3.496968272254586,
+      "tokens_seen": 1754726400
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 590929,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2585644721984863,
+      "objective/train/theoretical_loss": 3.496946111535202,
+      "objective/train/tokens_used": 125316576,
+      "theoretical_loss": 3.496946111535202,
+      "tokens_seen": 1754857472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047295779168672765,
+      "loss": 1.3049,
+      "theoretical_loss": 3.496946111535202,
+      "tokens_seen": 1754857472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047291766971593646,
+      "loss": 1.2694,
+      "theoretical_loss": 3.496923952934379,
+      "tokens_seen": 1754988544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047287754774514527,
+      "loss": 1.2811,
+      "theoretical_loss": 3.496901796451758,
+      "tokens_seen": 1755119616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728374257743541,
+      "loss": 1.2734,
+      "theoretical_loss": 3.496879642086977,
+      "tokens_seen": 1755250688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727973038035628,
+      "loss": 1.2728,
+      "theoretical_loss": 3.496857489839676,
+      "tokens_seen": 1755381760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047275718183277163,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4968353397094942,
+      "tokens_seen": 1755512832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047271705986198044,
+      "loss": 1.3628,
+      "theoretical_loss": 3.4968131916960714,
+      "tokens_seen": 1755643904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004726769378911892,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4967910457990476,
+      "tokens_seen": 1755774976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047263681592039805,
+      "loss": 1.3023,
+      "theoretical_loss": 3.4967689020180623,
+      "tokens_seen": 1755906048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725966939496068,
+      "loss": 1.2567,
+      "theoretical_loss": 3.496746760352755,
+      "tokens_seen": 1756037120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725565719788156,
+      "loss": 1.2775,
+      "theoretical_loss": 3.4967246208027665,
+      "tokens_seen": 1756168192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725164500080244,
+      "loss": 1.3319,
+      "theoretical_loss": 3.496702483367736,
+      "tokens_seen": 1756299264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047247632803723317,
+      "loss": 1.2647,
+      "theoretical_loss": 3.4966803480473043,
+      "tokens_seen": 1756430336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472436206066442,
+      "loss": 1.3328,
+      "theoretical_loss": 3.4966582148411116,
+      "tokens_seen": 1756561408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723960840956508,
+      "loss": 1.3062,
+      "theoretical_loss": 3.4966360837487978,
+      "tokens_seen": 1756692480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723559621248596,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4966139547700035,
+      "tokens_seen": 1756823552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723158401540684,
+      "loss": 1.2898,
+      "theoretical_loss": 3.4965918279043695,
+      "tokens_seen": 1756954624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004722757181832772,
+      "loss": 1.2152,
+      "theoretical_loss": 3.496569703151536,
+      "tokens_seen": 1757085696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047223559621248595,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4965475805111437,
+      "tokens_seen": 1757216768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721954742416948,
+      "loss": 1.2305,
+      "theoretical_loss": 3.4965254599828333,
+      "tokens_seen": 1757347840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047215535227090356,
+      "loss": 1.3164,
+      "theoretical_loss": 3.496503341566246,
+      "tokens_seen": 1757478912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721152303001123,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4964812252610225,
+      "tokens_seen": 1757609984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004720751083293212,
+      "loss": 1.3564,
+      "theoretical_loss": 3.496459111066804,
+      "tokens_seen": 1757741056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047203498635852993,
+      "loss": 1.2281,
+      "theoretical_loss": 3.4964369989832313,
+      "tokens_seen": 1757872128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047199486438773873,
+      "loss": 1.3635,
+      "theoretical_loss": 3.496414889009946,
+      "tokens_seen": 1758003200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047195474241694754,
+      "loss": 1.3007,
+      "theoretical_loss": 3.4963927811465885,
+      "tokens_seen": 1758134272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047191462044615635,
+      "loss": 1.2745,
+      "theoretical_loss": 3.4963706753928014,
+      "tokens_seen": 1758265344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718744984753651,
+      "loss": 1.3214,
+      "theoretical_loss": 3.496348571748226,
+      "tokens_seen": 1758396416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718343765045739,
+      "loss": 1.3308,
+      "theoretical_loss": 3.4963264702125025,
+      "tokens_seen": 1758527488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717942545337827,
+      "loss": 1.2834,
+      "theoretical_loss": 3.496304370785274,
+      "tokens_seen": 1758658560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047175413256299146,
+      "loss": 1.3041,
+      "theoretical_loss": 3.4962822734661816,
+      "tokens_seen": 1758789632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717140105922003,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4962601782548672,
+      "tokens_seen": 1758920704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716738886214091,
+      "loss": 1.3085,
+      "theoretical_loss": 3.496238085150973,
+      "tokens_seen": 1759051776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047163376665061794,
+      "loss": 1.3276,
+      "theoretical_loss": 3.4962159941541406,
+      "tokens_seen": 1759182848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715936446798267,
+      "loss": 1.3799,
+      "theoretical_loss": 3.496193905264012,
+      "tokens_seen": 1759313920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047155352270903544,
+      "loss": 1.2359,
+      "theoretical_loss": 3.4961718184802297,
+      "tokens_seen": 1759444992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715134007382443,
+      "loss": 1.2677,
+      "theoretical_loss": 3.4961497338024357,
+      "tokens_seen": 1759576064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047147327876745305,
+      "loss": 1.2361,
+      "theoretical_loss": 3.496127651230273,
+      "tokens_seen": 1759707136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047143315679666186,
+      "loss": 1.2811,
+      "theoretical_loss": 3.4961055707633824,
+      "tokens_seen": 1759838208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047139303482587067,
+      "loss": 1.3735,
+      "theoretical_loss": 3.496083492401408,
+      "tokens_seen": 1759969280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713529128550795,
+      "loss": 1.2416,
+      "theoretical_loss": 3.4960614161439922,
+      "tokens_seen": 1760100352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713127908842882,
+      "loss": 1.3116,
+      "theoretical_loss": 3.4960393419907776,
+      "tokens_seen": 1760231424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047127266891349703,
+      "loss": 1.2431,
+      "theoretical_loss": 3.4960172699414063,
+      "tokens_seen": 1760362496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047123254694270584,
+      "loss": 1.2848,
+      "theoretical_loss": 3.495995199995521,
+      "tokens_seen": 1760493568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711924249719146,
+      "loss": 1.2812,
+      "theoretical_loss": 3.4959731321527663,
+      "tokens_seen": 1760624640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047115230300112345,
+      "loss": 1.3576,
+      "theoretical_loss": 3.4959510664127835,
+      "tokens_seen": 1760755712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711121810303322,
+      "loss": 1.3428,
+      "theoretical_loss": 3.4959290027752172,
+      "tokens_seen": 1760886784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047107205905954106,
+      "loss": 1.3087,
+      "theoretical_loss": 3.495906941239709,
+      "tokens_seen": 1761017856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710319370887498,
+      "loss": 1.2535,
+      "theoretical_loss": 3.4958848818059036,
+      "tokens_seen": 1761148928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047099181511795857,
+      "loss": 1.2973,
+      "theoretical_loss": 3.495862824473444,
+      "tokens_seen": 1761280000
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 593108,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.237658977508545,
+      "objective/train/theoretical_loss": 3.4958407692419726,
+      "objective/train/tokens_used": 131870176,
+      "theoretical_loss": 3.4958407692419726,
+      "tokens_seen": 1761411072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047095169314716743,
+      "loss": 1.2443,
+      "theoretical_loss": 3.4958407692419726,
+      "tokens_seen": 1761411072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004709115711763762,
+      "loss": 1.3336,
+      "theoretical_loss": 3.4958187161111347,
+      "tokens_seen": 1761542144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470871449205585,
+      "loss": 1.3029,
+      "theoretical_loss": 3.4957966650805727,
+      "tokens_seen": 1761673216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708313272347938,
+      "loss": 1.3244,
+      "theoretical_loss": 3.495774616149931,
+      "tokens_seen": 1761804288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707912052640026,
+      "loss": 1.3143,
+      "theoretical_loss": 3.495752569318853,
+      "tokens_seen": 1761935360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047075108329321135,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4957305245869827,
+      "tokens_seen": 1762066432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707109613224202,
+      "loss": 1.33,
+      "theoretical_loss": 3.495708481953964,
+      "tokens_seen": 1762197504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047067083935162896,
+      "loss": 1.3479,
+      "theoretical_loss": 3.4956864414194415,
+      "tokens_seen": 1762328576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706307173808377,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4956644029830586,
+      "tokens_seen": 1762459648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705905954100466,
+      "loss": 1.2868,
+      "theoretical_loss": 3.49564236664446,
+      "tokens_seen": 1762590720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047055047343925533,
+      "loss": 1.2813,
+      "theoretical_loss": 3.4956203324032904,
+      "tokens_seen": 1762721792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047051035146846413,
+      "loss": 1.3111,
+      "theoretical_loss": 3.4955983002591937,
+      "tokens_seen": 1762852864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047047022949767294,
+      "loss": 1.3041,
+      "theoretical_loss": 3.4955762702118145,
+      "tokens_seen": 1762983936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047043010752688175,
+      "loss": 1.3313,
+      "theoretical_loss": 3.495554242260797,
+      "tokens_seen": 1763115008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047038998555609055,
+      "loss": 1.2942,
+      "theoretical_loss": 3.4955322164057865,
+      "tokens_seen": 1763246080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703498635852993,
+      "loss": 1.3294,
+      "theoretical_loss": 3.495510192646428,
+      "tokens_seen": 1763377152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703097416145081,
+      "loss": 1.3191,
+      "theoretical_loss": 3.495488170982366,
+      "tokens_seen": 1763508224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702696196437169,
+      "loss": 1.343,
+      "theoretical_loss": 3.495466151413245,
+      "tokens_seen": 1763639296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702294976729257,
+      "loss": 1.2553,
+      "theoretical_loss": 3.495444133938711,
+      "tokens_seen": 1763770368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701893757021345,
+      "loss": 1.2582,
+      "theoretical_loss": 3.495422118558408,
+      "tokens_seen": 1763901440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047014925373134334,
+      "loss": 1.2684,
+      "theoretical_loss": 3.495400105271982,
+      "tokens_seen": 1764032512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701091317605521,
+      "loss": 1.3155,
+      "theoretical_loss": 3.495378094079078,
+      "tokens_seen": 1764163584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047006900978976084,
+      "loss": 1.3194,
+      "theoretical_loss": 3.4953560849793415,
+      "tokens_seen": 1764294656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700288878189697,
+      "loss": 1.2951,
+      "theoretical_loss": 3.4953340779724176,
+      "tokens_seen": 1764425728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046998876584817845,
+      "loss": 1.3292,
+      "theoretical_loss": 3.4953120730579528,
+      "tokens_seen": 1764556800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046994864387738726,
+      "loss": 1.2956,
+      "theoretical_loss": 3.495290070235592,
+      "tokens_seen": 1764687872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046990852190659607,
+      "loss": 1.2978,
+      "theoretical_loss": 3.4952680695049807,
+      "tokens_seen": 1764818944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046986839993580487,
+      "loss": 1.3459,
+      "theoretical_loss": 3.495246070865765,
+      "tokens_seen": 1764950016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698282779650136,
+      "loss": 1.3746,
+      "theoretical_loss": 3.495224074317591,
+      "tokens_seen": 1765081088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046978815599422243,
+      "loss": 1.3791,
+      "theoretical_loss": 3.4952020798601047,
+      "tokens_seen": 1765212160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046974803402343124,
+      "loss": 1.2413,
+      "theoretical_loss": 3.495180087492952,
+      "tokens_seen": 1765343232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046970791205264004,
+      "loss": 1.3123,
+      "theoretical_loss": 3.4951580972157785,
+      "tokens_seen": 1765474304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046966779008184885,
+      "loss": 1.2382,
+      "theoretical_loss": 3.495136109028232,
+      "tokens_seen": 1765605376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696276681110576,
+      "loss": 1.3029,
+      "theoretical_loss": 3.495114122929957,
+      "tokens_seen": 1765736448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046958754614026646,
+      "loss": 1.2682,
+      "theoretical_loss": 3.495092138920601,
+      "tokens_seen": 1765867520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695474241694752,
+      "loss": 1.2845,
+      "theoretical_loss": 3.4950701569998106,
+      "tokens_seen": 1765998592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046950730219868397,
+      "loss": 1.3738,
+      "theoretical_loss": 3.4950481771672313,
+      "tokens_seen": 1766129664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046946718022789283,
+      "loss": 1.3035,
+      "theoretical_loss": 3.4950261994225107,
+      "tokens_seen": 1766260736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694270582571016,
+      "loss": 1.3015,
+      "theoretical_loss": 3.495004223765296,
+      "tokens_seen": 1766391808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693869362863104,
+      "loss": 1.2962,
+      "theoretical_loss": 3.494982250195233,
+      "tokens_seen": 1766522880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693468143155192,
+      "loss": 1.2952,
+      "theoretical_loss": 3.494960278711969,
+      "tokens_seen": 1766653952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469306692344728,
+      "loss": 1.3189,
+      "theoretical_loss": 3.4949383093151507,
+      "tokens_seen": 1766785024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046926657037393675,
+      "loss": 1.3828,
+      "theoretical_loss": 3.4949163420044265,
+      "tokens_seen": 1766916096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046922644840314556,
+      "loss": 1.2911,
+      "theoretical_loss": 3.494894376779442,
+      "tokens_seen": 1767047168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046918632643235436,
+      "loss": 1.3139,
+      "theoretical_loss": 3.494872413639845,
+      "tokens_seen": 1767178240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691462044615631,
+      "loss": 1.261,
+      "theoretical_loss": 3.4948504525852835,
+      "tokens_seen": 1767309312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469106082490772,
+      "loss": 1.2401,
+      "theoretical_loss": 3.494828493615404,
+      "tokens_seen": 1767440384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046906596051998073,
+      "loss": 1.2784,
+      "theoretical_loss": 3.494806536729855,
+      "tokens_seen": 1767571456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690258385491896,
+      "loss": 1.2621,
+      "theoretical_loss": 3.494784581928283,
+      "tokens_seen": 1767702528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046898571657839834,
+      "loss": 1.2918,
+      "theoretical_loss": 3.494762629210337,
+      "tokens_seen": 1767833600
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 595402,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3165714740753174,
+      "objective/train/theoretical_loss": 3.4947406785756634,
+      "objective/train/tokens_used": 138423776,
+      "theoretical_loss": 3.4947406785756634,
+      "tokens_seen": 1767964672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046894559460760715,
+      "loss": 1.2929,
+      "theoretical_loss": 3.4947406785756634,
+      "tokens_seen": 1767964672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046890547263681595,
+      "loss": 1.3235,
+      "theoretical_loss": 3.494718730023911,
+      "tokens_seen": 1768095744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688653506660247,
+      "loss": 1.3263,
+      "theoretical_loss": 3.494696783554727,
+      "tokens_seen": 1768226816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688252286952335,
+      "loss": 1.3029,
+      "theoretical_loss": 3.494674839167761,
+      "tokens_seen": 1768357888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687851067244423,
+      "loss": 1.3348,
+      "theoretical_loss": 3.494652896862659,
+      "tokens_seen": 1768488960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687449847536511,
+      "loss": 1.2327,
+      "theoretical_loss": 3.494630956639071,
+      "tokens_seen": 1768620032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687048627828599,
+      "loss": 1.4026,
+      "theoretical_loss": 3.4946090184966447,
+      "tokens_seen": 1768751104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046866474081206874,
+      "loss": 1.3345,
+      "theoretical_loss": 3.494587082435028,
+      "tokens_seen": 1768882176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686246188412775,
+      "loss": 1.251,
+      "theoretical_loss": 3.4945651484538702,
+      "tokens_seen": 1769013248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046858449687048624,
+      "loss": 1.3408,
+      "theoretical_loss": 3.4945432165528194,
+      "tokens_seen": 1769144320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685443748996951,
+      "loss": 1.2394,
+      "theoretical_loss": 3.4945212867315236,
+      "tokens_seen": 1769275392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046850425292890385,
+      "loss": 1.3295,
+      "theoretical_loss": 3.494499358989633,
+      "tokens_seen": 1769406464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684641309581127,
+      "loss": 1.2799,
+      "theoretical_loss": 3.4944774333267956,
+      "tokens_seen": 1769537536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046842400898732147,
+      "loss": 1.2254,
+      "theoretical_loss": 3.49445550974266,
+      "tokens_seen": 1769668608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046838388701653027,
+      "loss": 1.2646,
+      "theoretical_loss": 3.494433588236875,
+      "tokens_seen": 1769799680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683437650457391,
+      "loss": 1.3428,
+      "theoretical_loss": 3.494411668809091,
+      "tokens_seen": 1769930752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046830364307494783,
+      "loss": 1.3727,
+      "theoretical_loss": 3.4943897514589564,
+      "tokens_seen": 1770061824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046826352110415664,
+      "loss": 1.3272,
+      "theoretical_loss": 3.49436783618612,
+      "tokens_seen": 1770192896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046822339913336544,
+      "loss": 1.3156,
+      "theoretical_loss": 3.4943459229902314,
+      "tokens_seen": 1770323968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046818327716257425,
+      "loss": 1.2652,
+      "theoretical_loss": 3.4943240118709404,
+      "tokens_seen": 1770455040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468143155191783,
+      "loss": 1.3449,
+      "theoretical_loss": 3.4943021028278958,
+      "tokens_seen": 1770586112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046810303322099186,
+      "loss": 1.38,
+      "theoretical_loss": 3.4942801958607483,
+      "tokens_seen": 1770717184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004680629112502006,
+      "loss": 1.3518,
+      "theoretical_loss": 3.494258290969147,
+      "tokens_seen": 1770848256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046802278927940937,
+      "loss": 1.2885,
+      "theoretical_loss": 3.4942363881527405,
+      "tokens_seen": 1770979328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679826673086182,
+      "loss": 1.2459,
+      "theoretical_loss": 3.4942144874111802,
+      "tokens_seen": 1771110400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467942545337827,
+      "loss": 1.2604,
+      "theoretical_loss": 3.4941925887441156,
+      "tokens_seen": 1771241472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679024233670358,
+      "loss": 1.2582,
+      "theoretical_loss": 3.4941706921511964,
+      "tokens_seen": 1771372544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678623013962446,
+      "loss": 1.3287,
+      "theoretical_loss": 3.494148797632073,
+      "tokens_seen": 1771503616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678221794254534,
+      "loss": 1.3273,
+      "theoretical_loss": 3.4941269051863957,
+      "tokens_seen": 1771634688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677820574546622,
+      "loss": 1.3518,
+      "theoretical_loss": 3.4941050148138135,
+      "tokens_seen": 1771765760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046774193548387096,
+      "loss": 1.2383,
+      "theoretical_loss": 3.4940831265139787,
+      "tokens_seen": 1771896832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046770181351307976,
+      "loss": 1.3104,
+      "theoretical_loss": 3.4940612402865407,
+      "tokens_seen": 1772027904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046766169154228857,
+      "loss": 1.2973,
+      "theoretical_loss": 3.4940393561311502,
+      "tokens_seen": 1772158976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676215695714974,
+      "loss": 1.3034,
+      "theoretical_loss": 3.494017474047457,
+      "tokens_seen": 1772290048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046758144760070613,
+      "loss": 1.3789,
+      "theoretical_loss": 3.4939955940351135,
+      "tokens_seen": 1772421120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467541325629915,
+      "loss": 1.2632,
+      "theoretical_loss": 3.4939737160937687,
+      "tokens_seen": 1772552192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046750120365912374,
+      "loss": 1.3573,
+      "theoretical_loss": 3.4939518402230747,
+      "tokens_seen": 1772683264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046746108168833255,
+      "loss": 1.2939,
+      "theoretical_loss": 3.493929966422682,
+      "tokens_seen": 1772814336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046742095971754135,
+      "loss": 1.2631,
+      "theoretical_loss": 3.493908094692241,
+      "tokens_seen": 1772945408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673808377467501,
+      "loss": 1.2778,
+      "theoretical_loss": 3.493886225031404,
+      "tokens_seen": 1773076480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673407157759589,
+      "loss": 1.4262,
+      "theoretical_loss": 3.493864357439821,
+      "tokens_seen": 1773207552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673005938051677,
+      "loss": 1.3158,
+      "theoretical_loss": 3.4938424919171447,
+      "tokens_seen": 1773338624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672604718343765,
+      "loss": 1.2937,
+      "theoretical_loss": 3.493820628463025,
+      "tokens_seen": 1773469696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046722034986358533,
+      "loss": 1.3395,
+      "theoretical_loss": 3.4937987670771142,
+      "tokens_seen": 1773600768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046718022789279414,
+      "loss": 1.3646,
+      "theoretical_loss": 3.493776907759064,
+      "tokens_seen": 1773731840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004671401059220029,
+      "loss": 1.2991,
+      "theoretical_loss": 3.493755050508525,
+      "tokens_seen": 1773862912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004670999839512117,
+      "loss": 1.3089,
+      "theoretical_loss": 3.4937331953251496,
+      "tokens_seen": 1773993984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004670598619804205,
+      "loss": 1.2472,
+      "theoretical_loss": 3.4937113422085897,
+      "tokens_seen": 1774125056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046701974000962925,
+      "loss": 1.3086,
+      "theoretical_loss": 3.493689491158497,
+      "tokens_seen": 1774256128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669796180388381,
+      "loss": 1.1765,
+      "theoretical_loss": 3.4936676421745236,
+      "tokens_seen": 1774387200
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 597604,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.218658685684204,
+      "objective/train/theoretical_loss": 3.493645795256321,
+      "objective/train/tokens_used": 144977376,
+      "theoretical_loss": 3.493645795256321,
+      "tokens_seen": 1774518272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046693949606804687,
+      "loss": 1.2543,
+      "theoretical_loss": 3.493645795256321,
+      "tokens_seen": 1774518272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046689937409725567,
+      "loss": 1.2918,
+      "theoretical_loss": 3.493623950403542,
+      "tokens_seen": 1774649344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668592521264645,
+      "loss": 1.223,
+      "theoretical_loss": 3.4936021076158386,
+      "tokens_seen": 1774780416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046681913015567323,
+      "loss": 1.2933,
+      "theoretical_loss": 3.493580266892863,
+      "tokens_seen": 1774911488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046677900818488204,
+      "loss": 1.2607,
+      "theoretical_loss": 3.4935584282342673,
+      "tokens_seen": 1775042560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046673888621409084,
+      "loss": 1.2328,
+      "theoretical_loss": 3.4935365916397045,
+      "tokens_seen": 1775173632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046669876424329965,
+      "loss": 1.3178,
+      "theoretical_loss": 3.4935147571088265,
+      "tokens_seen": 1775304704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666586422725084,
+      "loss": 1.2836,
+      "theoretical_loss": 3.493492924641287,
+      "tokens_seen": 1775435776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046661852030171726,
+      "loss": 1.1723,
+      "theoretical_loss": 3.493471094236737,
+      "tokens_seen": 1775566848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466578398330926,
+      "loss": 1.2864,
+      "theoretical_loss": 3.4934492658948315,
+      "tokens_seen": 1775697920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665382763601348,
+      "loss": 1.337,
+      "theoretical_loss": 3.493427439615221,
+      "tokens_seen": 1775828992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664981543893436,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4934056153975606,
+      "tokens_seen": 1775960064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664580324185524,
+      "loss": 1.2555,
+      "theoretical_loss": 3.4933837932415024,
+      "tokens_seen": 1776091136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046641791044776124,
+      "loss": 1.2744,
+      "theoretical_loss": 3.493361973146699,
+      "tokens_seen": 1776222208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046637778847697,
+      "loss": 1.249,
+      "theoretical_loss": 3.4933401551128043,
+      "tokens_seen": 1776353280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663376665061788,
+      "loss": 1.3393,
+      "theoretical_loss": 3.4933183391394715,
+      "tokens_seen": 1776484352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662975445353876,
+      "loss": 1.2572,
+      "theoretical_loss": 3.493296525226354,
+      "tokens_seen": 1776615424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046625742256459636,
+      "loss": 1.2132,
+      "theoretical_loss": 3.4932747133731055,
+      "tokens_seen": 1776746496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046621730059380516,
+      "loss": 1.356,
+      "theoretical_loss": 3.4932529035793793,
+      "tokens_seen": 1776877568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046617717862301397,
+      "loss": 1.2766,
+      "theoretical_loss": 3.4932310958448287,
+      "tokens_seen": 1777008640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661370566522228,
+      "loss": 1.2332,
+      "theoretical_loss": 3.4932092901691076,
+      "tokens_seen": 1777139712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046609693468143153,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4931874865518697,
+      "tokens_seen": 1777270784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660568127106404,
+      "loss": 1.3148,
+      "theoretical_loss": 3.4931656849927695,
+      "tokens_seen": 1777401856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046601669073984914,
+      "loss": 1.1651,
+      "theoretical_loss": 3.4931438854914605,
+      "tokens_seen": 1777532928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465976568769058,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4931220880475973,
+      "tokens_seen": 1777664000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046593644679826675,
+      "loss": 1.268,
+      "theoretical_loss": 3.493100292660833,
+      "tokens_seen": 1777795072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658963248274755,
+      "loss": 1.3284,
+      "theoretical_loss": 3.4930784993308226,
+      "tokens_seen": 1777926144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046585620285668436,
+      "loss": 1.3535,
+      "theoretical_loss": 3.49305670805722,
+      "tokens_seen": 1778057216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658160808858931,
+      "loss": 1.1865,
+      "theoretical_loss": 3.49303491883968,
+      "tokens_seen": 1778188288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004657759589151019,
+      "loss": 1.2648,
+      "theoretical_loss": 3.4930131316778565,
+      "tokens_seen": 1778319360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046573583694431073,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4929913465714053,
+      "tokens_seen": 1778450432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046569571497351954,
+      "loss": 1.3182,
+      "theoretical_loss": 3.4929695635199796,
+      "tokens_seen": 1778581504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656555930027283,
+      "loss": 1.2161,
+      "theoretical_loss": 3.4929477825232347,
+      "tokens_seen": 1778712576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656154710319371,
+      "loss": 1.2854,
+      "theoretical_loss": 3.492926003580825,
+      "tokens_seen": 1778843648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655753490611459,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4929042266924064,
+      "tokens_seen": 1778974720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046553522709035465,
+      "loss": 1.3418,
+      "theoretical_loss": 3.492882451857633,
+      "tokens_seen": 1779105792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654951051195635,
+      "loss": 1.2947,
+      "theoretical_loss": 3.49286067907616,
+      "tokens_seen": 1779236864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046545498314877227,
+      "loss": 1.25,
+      "theoretical_loss": 3.4928389083476423,
+      "tokens_seen": 1779367936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046541486117798107,
+      "loss": 1.3604,
+      "theoretical_loss": 3.492817139671736,
+      "tokens_seen": 1779499008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653747392071899,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4927953730480956,
+      "tokens_seen": 1779630080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046533461723639863,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4927736084763774,
+      "tokens_seen": 1779761152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652944952656075,
+      "loss": 1.2523,
+      "theoretical_loss": 3.492751845956236,
+      "tokens_seen": 1779892224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046525437329481624,
+      "loss": 1.2623,
+      "theoretical_loss": 3.4927300854873264,
+      "tokens_seen": 1780023296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046521425132402505,
+      "loss": 1.1974,
+      "theoretical_loss": 3.492708327069306,
+      "tokens_seen": 1780154368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046517412935323386,
+      "loss": 1.2556,
+      "theoretical_loss": 3.492686570701829,
+      "tokens_seen": 1780285440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046513400738244266,
+      "loss": 1.3253,
+      "theoretical_loss": 3.492664816384552,
+      "tokens_seen": 1780416512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650938854116514,
+      "loss": 1.2586,
+      "theoretical_loss": 3.4926430641171304,
+      "tokens_seen": 1780547584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650537634408602,
+      "loss": 1.3246,
+      "theoretical_loss": 3.4926213138992206,
+      "tokens_seen": 1780678656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465013641470069,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4925995657304787,
+      "tokens_seen": 1780809728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004649735194992778,
+      "loss": 1.2549,
+      "theoretical_loss": 3.4925778196105606,
+      "tokens_seen": 1780940800
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 599799,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2370692491531372,
+      "objective/train/theoretical_loss": 3.492556075539122,
+      "objective/train/tokens_used": 151530976,
+      "theoretical_loss": 3.492556075539122,
+      "tokens_seen": 1781071872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046493339752848664,
+      "loss": 1.2818,
+      "theoretical_loss": 3.492556075539122,
+      "tokens_seen": 1781071872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648932755576954,
+      "loss": 1.3001,
+      "theoretical_loss": 3.4925343335158203,
+      "tokens_seen": 1781202944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648531535869042,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4925125935403107,
+      "tokens_seen": 1781334016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464813031616113,
+      "loss": 1.2961,
+      "theoretical_loss": 3.4924908556122505,
+      "tokens_seen": 1781465088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046477290964532176,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4924691197312967,
+      "tokens_seen": 1781596160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046473278767453056,
+      "loss": 1.2548,
+      "theoretical_loss": 3.492447385897105,
+      "tokens_seen": 1781727232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046469266570373937,
+      "loss": 1.3495,
+      "theoretical_loss": 3.492425654109332,
+      "tokens_seen": 1781858304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004646525437329482,
+      "loss": 1.2781,
+      "theoretical_loss": 3.4924039243676352,
+      "tokens_seen": 1781989376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464612421762157,
+      "loss": 1.3324,
+      "theoretical_loss": 3.492382196671671,
+      "tokens_seen": 1782120448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004645722997913658,
+      "loss": 1.303,
+      "theoretical_loss": 3.4923604710210965,
+      "tokens_seen": 1782251520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046453217782057454,
+      "loss": 1.3166,
+      "theoretical_loss": 3.492338747415569,
+      "tokens_seen": 1782382592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046449205584978335,
+      "loss": 1.3307,
+      "theoretical_loss": 3.492317025854746,
+      "tokens_seen": 1782513664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046445193387899215,
+      "loss": 1.3757,
+      "theoretical_loss": 3.492295306338283,
+      "tokens_seen": 1782644736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644118119082009,
+      "loss": 1.3502,
+      "theoretical_loss": 3.4922735888658387,
+      "tokens_seen": 1782775808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046437168993740976,
+      "loss": 1.2562,
+      "theoretical_loss": 3.4922518734370707,
+      "tokens_seen": 1782906880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004643315679666185,
+      "loss": 1.4091,
+      "theoretical_loss": 3.492230160051636,
+      "tokens_seen": 1783037952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642914459958273,
+      "loss": 1.316,
+      "theoretical_loss": 3.4922084487091922,
+      "tokens_seen": 1783169024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046425132402503613,
+      "loss": 1.287,
+      "theoretical_loss": 3.4921867394093966,
+      "tokens_seen": 1783300096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046421120205424494,
+      "loss": 1.2959,
+      "theoretical_loss": 3.492165032151907,
+      "tokens_seen": 1783431168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641710800834537,
+      "loss": 1.2754,
+      "theoretical_loss": 3.4921433269363815,
+      "tokens_seen": 1783562240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641309581126625,
+      "loss": 1.3524,
+      "theoretical_loss": 3.492121623762478,
+      "tokens_seen": 1783693312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640908361418713,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4920999226298544,
+      "tokens_seen": 1783824384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046405071417108005,
+      "loss": 1.366,
+      "theoretical_loss": 3.4920782235381687,
+      "tokens_seen": 1783955456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640105922002889,
+      "loss": 1.3517,
+      "theoretical_loss": 3.4920565264870786,
+      "tokens_seen": 1784086528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046397047022949767,
+      "loss": 1.3174,
+      "theoretical_loss": 3.492034831476243,
+      "tokens_seen": 1784217600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639303482587065,
+      "loss": 1.304,
+      "theoretical_loss": 3.4920131385053192,
+      "tokens_seen": 1784348672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004638902262879153,
+      "loss": 1.3009,
+      "theoretical_loss": 3.491991447573967,
+      "tokens_seen": 1784479744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046385010431712403,
+      "loss": 1.3861,
+      "theoretical_loss": 3.4919697586818437,
+      "tokens_seen": 1784610816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004638099823463329,
+      "loss": 1.3494,
+      "theoretical_loss": 3.4919480718286082,
+      "tokens_seen": 1784741888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046376986037554164,
+      "loss": 1.3566,
+      "theoretical_loss": 3.491926387013919,
+      "tokens_seen": 1784872960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046372973840475045,
+      "loss": 1.2978,
+      "theoretical_loss": 3.4919047042374354,
+      "tokens_seen": 1785004032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046368961643395926,
+      "loss": 1.3379,
+      "theoretical_loss": 3.491883023498816,
+      "tokens_seen": 1785135104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046364949446316806,
+      "loss": 1.3265,
+      "theoretical_loss": 3.4918613447977185,
+      "tokens_seen": 1785266176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636093724923768,
+      "loss": 1.3721,
+      "theoretical_loss": 3.491839668133803,
+      "tokens_seen": 1785397248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635692505215856,
+      "loss": 1.2868,
+      "theoretical_loss": 3.491817993506728,
+      "tokens_seen": 1785528320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635291285507944,
+      "loss": 1.2728,
+      "theoretical_loss": 3.4917963209161536,
+      "tokens_seen": 1785659392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634890065800032,
+      "loss": 1.3587,
+      "theoretical_loss": 3.4917746503617373,
+      "tokens_seen": 1785790464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046344888460921204,
+      "loss": 1.3097,
+      "theoretical_loss": 3.49175298184314,
+      "tokens_seen": 1785921536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634087626384208,
+      "loss": 1.3031,
+      "theoretical_loss": 3.4917313153600205,
+      "tokens_seen": 1786052608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046336864066762965,
+      "loss": 1.3133,
+      "theoretical_loss": 3.4917096509120378,
+      "tokens_seen": 1786183680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004633285186968384,
+      "loss": 1.3537,
+      "theoretical_loss": 3.4916879884988514,
+      "tokens_seen": 1786314752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046328839672604716,
+      "loss": 1.3442,
+      "theoretical_loss": 3.4916663281201217,
+      "tokens_seen": 1786445824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000463248274755256,
+      "loss": 1.277,
+      "theoretical_loss": 3.491644669775508,
+      "tokens_seen": 1786576896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046320815278446477,
+      "loss": 1.368,
+      "theoretical_loss": 3.49162301346467,
+      "tokens_seen": 1786707968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631680308136736,
+      "loss": 1.3287,
+      "theoretical_loss": 3.491601359187267,
+      "tokens_seen": 1786839040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631279088428824,
+      "loss": 1.3579,
+      "theoretical_loss": 3.49157970694296,
+      "tokens_seen": 1786970112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630877868720912,
+      "loss": 1.3116,
+      "theoretical_loss": 3.4915580567314084,
+      "tokens_seen": 1787101184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046304766490129994,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4915364085522724,
+      "tokens_seen": 1787232256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046300754293050875,
+      "loss": 1.3085,
+      "theoretical_loss": 3.491514762405212,
+      "tokens_seen": 1787363328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046296742095971755,
+      "loss": 1.3115,
+      "theoretical_loss": 3.4914931182898874,
+      "tokens_seen": 1787494400
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 601994,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3719425201416016,
+      "objective/train/theoretical_loss": 3.4914714762059598,
+      "objective/train/tokens_used": 158084576,
+      "theoretical_loss": 3.4914714762059598,
+      "tokens_seen": 1787625472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629272989889263,
+      "loss": 1.326,
+      "theoretical_loss": 3.4914714762059598,
+      "tokens_seen": 1787625472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046288717701813516,
+      "loss": 1.3017,
+      "theoretical_loss": 3.4914498361530883,
+      "tokens_seen": 1787756544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628470550473439,
+      "loss": 1.3247,
+      "theoretical_loss": 3.491428198130934,
+      "tokens_seen": 1787887616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628069330765527,
+      "loss": 1.3317,
+      "theoretical_loss": 3.491406562139158,
+      "tokens_seen": 1788018688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046276681110576153,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4913849281774203,
+      "tokens_seen": 1788149760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046272668913497034,
+      "loss": 1.23,
+      "theoretical_loss": 3.4913632962453818,
+      "tokens_seen": 1788280832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046268656716417914,
+      "loss": 1.3395,
+      "theoretical_loss": 3.4913416663427035,
+      "tokens_seen": 1788411904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626464451933879,
+      "loss": 1.2527,
+      "theoretical_loss": 3.491320038469046,
+      "tokens_seen": 1788542976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626063232225967,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4912984126240705,
+      "tokens_seen": 1788674048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625662012518055,
+      "loss": 1.3244,
+      "theoretical_loss": 3.491276788807438,
+      "tokens_seen": 1788805120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625260792810143,
+      "loss": 1.3333,
+      "theoretical_loss": 3.49125516701881,
+      "tokens_seen": 1788936192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046248595731022306,
+      "loss": 1.3305,
+      "theoretical_loss": 3.4912335472578473,
+      "tokens_seen": 1789067264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624458353394319,
+      "loss": 1.285,
+      "theoretical_loss": 3.4912119295242117,
+      "tokens_seen": 1789198336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624057133686407,
+      "loss": 1.2974,
+      "theoretical_loss": 3.491190313817564,
+      "tokens_seen": 1789329408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046236559139784943,
+      "loss": 1.3599,
+      "theoretical_loss": 3.4911687001375658,
+      "tokens_seen": 1789460480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623254694270583,
+      "loss": 1.2553,
+      "theoretical_loss": 3.4911470884838787,
+      "tokens_seen": 1789591552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046228534745626704,
+      "loss": 1.315,
+      "theoretical_loss": 3.4911254788561648,
+      "tokens_seen": 1789722624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046224522548547585,
+      "loss": 1.3595,
+      "theoretical_loss": 3.4911038712540847,
+      "tokens_seen": 1789853696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046220510351468465,
+      "loss": 1.2498,
+      "theoretical_loss": 3.491082265677302,
+      "tokens_seen": 1789984768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046216498154389346,
+      "loss": 1.2269,
+      "theoretical_loss": 3.491060662125477,
+      "tokens_seen": 1790115840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046212485957310227,
+      "loss": 1.3534,
+      "theoretical_loss": 3.491039060598272,
+      "tokens_seen": 1790246912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000462084737602311,
+      "loss": 1.2841,
+      "theoretical_loss": 3.4910174610953497,
+      "tokens_seen": 1790377984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620446156315198,
+      "loss": 1.3467,
+      "theoretical_loss": 3.4909958636163716,
+      "tokens_seen": 1790509056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046200449366072863,
+      "loss": 1.259,
+      "theoretical_loss": 3.490974268161,
+      "tokens_seen": 1790640128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046196437168993744,
+      "loss": 1.3633,
+      "theoretical_loss": 3.4909526747288977,
+      "tokens_seen": 1790771200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619242497191462,
+      "loss": 1.328,
+      "theoretical_loss": 3.490931083319726,
+      "tokens_seen": 1790902272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046188412774835505,
+      "loss": 1.2296,
+      "theoretical_loss": 3.490909493933149,
+      "tokens_seen": 1791033344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618440057775638,
+      "loss": 1.2442,
+      "theoretical_loss": 3.4908879065688274,
+      "tokens_seen": 1791164416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046180388380677256,
+      "loss": 1.2926,
+      "theoretical_loss": 3.4908663212264246,
+      "tokens_seen": 1791295488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617637618359814,
+      "loss": 1.3451,
+      "theoretical_loss": 3.4908447379056042,
+      "tokens_seen": 1791426560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046172363986519017,
+      "loss": 1.1907,
+      "theoretical_loss": 3.4908231566060275,
+      "tokens_seen": 1791557632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461683517894399,
+      "loss": 1.2634,
+      "theoretical_loss": 3.490801577327358,
+      "tokens_seen": 1791688704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616433959236078,
+      "loss": 1.3206,
+      "theoretical_loss": 3.490780000069259,
+      "tokens_seen": 1791819776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616032739528166,
+      "loss": 1.2801,
+      "theoretical_loss": 3.4907584248313928,
+      "tokens_seen": 1791950848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046156315198202534,
+      "loss": 1.3338,
+      "theoretical_loss": 3.4907368516134234,
+      "tokens_seen": 1792081920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046152303001123415,
+      "loss": 1.3126,
+      "theoretical_loss": 3.4907152804150128,
+      "tokens_seen": 1792212992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046148290804044295,
+      "loss": 1.3012,
+      "theoretical_loss": 3.490693711235825,
+      "tokens_seen": 1792344064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046144278606965176,
+      "loss": 1.2859,
+      "theoretical_loss": 3.4906721440755235,
+      "tokens_seen": 1792475136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046140266409886056,
+      "loss": 1.2535,
+      "theoretical_loss": 3.4906505789337716,
+      "tokens_seen": 1792606208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613625421280693,
+      "loss": 1.3317,
+      "theoretical_loss": 3.4906290158102324,
+      "tokens_seen": 1792737280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613224201572782,
+      "loss": 1.2882,
+      "theoretical_loss": 3.49060745470457,
+      "tokens_seen": 1792868352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046128229818648693,
+      "loss": 1.214,
+      "theoretical_loss": 3.490585895616447,
+      "tokens_seen": 1792999424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612421762156957,
+      "loss": 1.3223,
+      "theoretical_loss": 3.490564338545529,
+      "tokens_seen": 1793130496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046120205424490454,
+      "loss": 1.2588,
+      "theoretical_loss": 3.4905427834914784,
+      "tokens_seen": 1793261568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611619322741133,
+      "loss": 1.3041,
+      "theoretical_loss": 3.4905212304539592,
+      "tokens_seen": 1793392640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611218103033221,
+      "loss": 1.3429,
+      "theoretical_loss": 3.490499679432636,
+      "tokens_seen": 1793523712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610816883325309,
+      "loss": 1.3733,
+      "theoretical_loss": 3.490478130427173,
+      "tokens_seen": 1793654784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610415663617397,
+      "loss": 1.3249,
+      "theoretical_loss": 3.4904565834372336,
+      "tokens_seen": 1793785856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046100144439094846,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4904350384624823,
+      "tokens_seen": 1793916928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609613224201573,
+      "loss": 1.2672,
+      "theoretical_loss": 3.490413495502583,
+      "tokens_seen": 1794048000
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 604097,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2390307188034058,
+      "objective/train/theoretical_loss": 3.4903919545572015,
+      "objective/train/tokens_used": 164638176,
+      "theoretical_loss": 3.4903919545572015,
+      "tokens_seen": 1794179072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609212004493661,
+      "loss": 1.254,
+      "theoretical_loss": 3.4903919545572015,
+      "tokens_seen": 1794179072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046088107847857483,
+      "loss": 1.3122,
+      "theoretical_loss": 3.4903704156260007,
+      "tokens_seen": 1794310144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608409565077837,
+      "loss": 1.2656,
+      "theoretical_loss": 3.4903488787086463,
+      "tokens_seen": 1794441216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046080083453699244,
+      "loss": 1.2684,
+      "theoretical_loss": 3.4903273438048017,
+      "tokens_seen": 1794572288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607607125662013,
+      "loss": 1.2818,
+      "theoretical_loss": 3.4903058109141325,
+      "tokens_seen": 1794703360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046072059059541005,
+      "loss": 1.3074,
+      "theoretical_loss": 3.4902842800363034,
+      "tokens_seen": 1794834432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046068046862461886,
+      "loss": 1.3034,
+      "theoretical_loss": 3.4902627511709796,
+      "tokens_seen": 1794965504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046064034665382767,
+      "loss": 1.2929,
+      "theoretical_loss": 3.490241224317825,
+      "tokens_seen": 1795096576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606002246830364,
+      "loss": 1.36,
+      "theoretical_loss": 3.490219699476506,
+      "tokens_seen": 1795227648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004605601027122452,
+      "loss": 1.3416,
+      "theoretical_loss": 3.490198176646686,
+      "tokens_seen": 1795358720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046051998074145403,
+      "loss": 1.3,
+      "theoretical_loss": 3.490176655828032,
+      "tokens_seen": 1795489792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046047985877066284,
+      "loss": 1.2404,
+      "theoretical_loss": 3.490155137020208,
+      "tokens_seen": 1795620864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604397367998716,
+      "loss": 1.2798,
+      "theoretical_loss": 3.49013362022288,
+      "tokens_seen": 1795751936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046039961482908045,
+      "loss": 1.2621,
+      "theoretical_loss": 3.4901121054357134,
+      "tokens_seen": 1795883008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603594928582892,
+      "loss": 1.2505,
+      "theoretical_loss": 3.4900905926583734,
+      "tokens_seen": 1796014080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046031937088749795,
+      "loss": 1.2629,
+      "theoretical_loss": 3.4900690818905256,
+      "tokens_seen": 1796145152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602792489167068,
+      "loss": 1.2698,
+      "theoretical_loss": 3.490047573131836,
+      "tokens_seen": 1796276224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046023912694591557,
+      "loss": 1.3067,
+      "theoretical_loss": 3.49002606638197,
+      "tokens_seen": 1796407296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046019900497512443,
+      "loss": 1.2795,
+      "theoretical_loss": 3.4900045616405935,
+      "tokens_seen": 1796538368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601588830043332,
+      "loss": 1.3077,
+      "theoretical_loss": 3.489983058907373,
+      "tokens_seen": 1796669440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460118761033542,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4899615581819736,
+      "tokens_seen": 1796800512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004600786390627508,
+      "loss": 1.2588,
+      "theoretical_loss": 3.489940059464062,
+      "tokens_seen": 1796931584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046003851709195954,
+      "loss": 1.27,
+      "theoretical_loss": 3.4899185627533043,
+      "tokens_seen": 1797062656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045999839512116835,
+      "loss": 1.2834,
+      "theoretical_loss": 3.489897068049366,
+      "tokens_seen": 1797193728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045995827315037716,
+      "loss": 1.3287,
+      "theoretical_loss": 3.489875575351915,
+      "tokens_seen": 1797324800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045991815117958596,
+      "loss": 1.291,
+      "theoretical_loss": 3.4898540846606156,
+      "tokens_seen": 1797455872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598780292087947,
+      "loss": 1.2261,
+      "theoretical_loss": 3.489832595975136,
+      "tokens_seen": 1797586944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598379072380036,
+      "loss": 1.291,
+      "theoretical_loss": 3.4898111092951423,
+      "tokens_seen": 1797718016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045979778526721233,
+      "loss": 1.279,
+      "theoretical_loss": 3.4897896246203004,
+      "tokens_seen": 1797849088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597576632964211,
+      "loss": 1.3524,
+      "theoretical_loss": 3.489768141950278,
+      "tokens_seen": 1797980160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045971754132562994,
+      "loss": 1.3051,
+      "theoretical_loss": 3.489746661284741,
+      "tokens_seen": 1798111232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596774193548387,
+      "loss": 1.2527,
+      "theoretical_loss": 3.4897251826233573,
+      "tokens_seen": 1798242304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596372973840475,
+      "loss": 1.2591,
+      "theoretical_loss": 3.489703705965793,
+      "tokens_seen": 1798373376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595971754132563,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4896822313117153,
+      "tokens_seen": 1798504448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595570534424651,
+      "loss": 1.3019,
+      "theoretical_loss": 3.4896607586607917,
+      "tokens_seen": 1798635520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595169314716739,
+      "loss": 1.2711,
+      "theoretical_loss": 3.4896392880126887,
+      "tokens_seen": 1798766592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594768095008827,
+      "loss": 1.191,
+      "theoretical_loss": 3.4896178193670746,
+      "tokens_seen": 1798897664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594366875300915,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4895963527236153,
+      "tokens_seen": 1799028736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593965655593003,
+      "loss": 1.1669,
+      "theoretical_loss": 3.4895748880819797,
+      "tokens_seen": 1799159808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593564435885091,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4895534254418346,
+      "tokens_seen": 1799290880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045931632161771784,
+      "loss": 1.2563,
+      "theoretical_loss": 3.4895319648028478,
+      "tokens_seen": 1799421952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592761996469267,
+      "loss": 1.2479,
+      "theoretical_loss": 3.4895105061646863,
+      "tokens_seen": 1799553024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045923607767613545,
+      "loss": 1.2751,
+      "theoretical_loss": 3.4894890495270188,
+      "tokens_seen": 1799684096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045919595570534426,
+      "loss": 1.291,
+      "theoretical_loss": 3.489467594889513,
+      "tokens_seen": 1799815168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045915583373455307,
+      "loss": 1.3649,
+      "theoretical_loss": 3.4894461422518357,
+      "tokens_seen": 1799946240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004591157117637618,
+      "loss": 1.2707,
+      "theoretical_loss": 3.489424691613656,
+      "tokens_seen": 1800077312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004590755897929706,
+      "loss": 1.297,
+      "theoretical_loss": 3.489403242974642,
+      "tokens_seen": 1800208384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045903546782217943,
+      "loss": 1.2177,
+      "theoretical_loss": 3.4893817963344613,
+      "tokens_seen": 1800339456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045899534585138824,
+      "loss": 1.3523,
+      "theoretical_loss": 3.489360351692782,
+      "tokens_seen": 1800470528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045895522388059704,
+      "loss": 1.2314,
+      "theoretical_loss": 3.4893389090492732,
+      "tokens_seen": 1800601600
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 606247,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2932188510894775,
+      "objective/train/theoretical_loss": 3.489317468403603,
+      "objective/train/tokens_used": 171191776,
+      "theoretical_loss": 3.489317468403603,
+      "tokens_seen": 1800732672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045891510190980585,
+      "loss": 1.2895,
+      "theoretical_loss": 3.489317468403603,
+      "tokens_seen": 1800732672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588749799390146,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4892960297554385,
+      "tokens_seen": 1800863744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588348579682234,
+      "loss": 1.2941,
+      "theoretical_loss": 3.4892745931044504,
+      "tokens_seen": 1800994816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587947359974322,
+      "loss": 1.2166,
+      "theoretical_loss": 3.489253158450306,
+      "tokens_seen": 1801125888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045875461402664097,
+      "loss": 1.3488,
+      "theoretical_loss": 3.4892317257926746,
+      "tokens_seen": 1801256960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045871449205584983,
+      "loss": 1.2213,
+      "theoretical_loss": 3.4892102951312243,
+      "tokens_seen": 1801388032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004586743700850586,
+      "loss": 1.279,
+      "theoretical_loss": 3.489188866465625,
+      "tokens_seen": 1801519104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004586342481142674,
+      "loss": 1.3441,
+      "theoretical_loss": 3.489167439795544,
+      "tokens_seen": 1801650176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585941261434762,
+      "loss": 1.293,
+      "theoretical_loss": 3.489146015120652,
+      "tokens_seen": 1801781248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045855400417268494,
+      "loss": 1.2555,
+      "theoretical_loss": 3.489124592440618,
+      "tokens_seen": 1801912320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045851388220189375,
+      "loss": 1.3334,
+      "theoretical_loss": 3.4891031717551093,
+      "tokens_seen": 1802043392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045847376023110256,
+      "loss": 1.2193,
+      "theoretical_loss": 3.4890817530637968,
+      "tokens_seen": 1802174464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045843363826031136,
+      "loss": 1.2685,
+      "theoretical_loss": 3.4890603363663497,
+      "tokens_seen": 1802305536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583935162895201,
+      "loss": 1.2327,
+      "theoretical_loss": 3.489038921662437,
+      "tokens_seen": 1802436608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458353394318729,
+      "loss": 1.3008,
+      "theoretical_loss": 3.4890175089517284,
+      "tokens_seen": 1802567680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045831327234793773,
+      "loss": 1.2331,
+      "theoretical_loss": 3.4889960982338932,
+      "tokens_seen": 1802698752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045827315037714653,
+      "loss": 1.3458,
+      "theoretical_loss": 3.4889746895086016,
+      "tokens_seen": 1802829824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045823302840635534,
+      "loss": 1.2179,
+      "theoretical_loss": 3.4889532827755225,
+      "tokens_seen": 1802960896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581929064355641,
+      "loss": 1.37,
+      "theoretical_loss": 3.488931878034327,
+      "tokens_seen": 1803091968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045815278446477295,
+      "loss": 1.2758,
+      "theoretical_loss": 3.4889104752846833,
+      "tokens_seen": 1803223040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581126624939817,
+      "loss": 1.3107,
+      "theoretical_loss": 3.488889074526263,
+      "tokens_seen": 1803354112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580725405231905,
+      "loss": 1.3375,
+      "theoretical_loss": 3.4888676757587347,
+      "tokens_seen": 1803485184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580324185523993,
+      "loss": 1.3064,
+      "theoretical_loss": 3.4888462789817694,
+      "tokens_seen": 1803616256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579922965816081,
+      "loss": 1.3609,
+      "theoretical_loss": 3.4888248841950373,
+      "tokens_seen": 1803747328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579521746108169,
+      "loss": 1.229,
+      "theoretical_loss": 3.4888034913982082,
+      "tokens_seen": 1803878400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579120526400257,
+      "loss": 1.3499,
+      "theoretical_loss": 3.4887821005909525,
+      "tokens_seen": 1804009472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578719306692345,
+      "loss": 1.317,
+      "theoretical_loss": 3.488760711772941,
+      "tokens_seen": 1804140544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045783180869844324,
+      "loss": 1.2982,
+      "theoretical_loss": 3.488739324943844,
+      "tokens_seen": 1804271616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577916867276521,
+      "loss": 1.2487,
+      "theoretical_loss": 3.4887179401033324,
+      "tokens_seen": 1804402688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045775156475686085,
+      "loss": 1.2885,
+      "theoretical_loss": 3.4886965572510764,
+      "tokens_seen": 1804533760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577114427860697,
+      "loss": 1.3745,
+      "theoretical_loss": 3.4886751763867467,
+      "tokens_seen": 1804664832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045767132081527847,
+      "loss": 1.2863,
+      "theoretical_loss": 3.4886537975100147,
+      "tokens_seen": 1804795904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576311988444872,
+      "loss": 1.2086,
+      "theoretical_loss": 3.488632420620551,
+      "tokens_seen": 1804926976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575910768736961,
+      "loss": 1.2725,
+      "theoretical_loss": 3.4886110457180264,
+      "tokens_seen": 1805058048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045755095490290483,
+      "loss": 1.3338,
+      "theoretical_loss": 3.488589672802112,
+      "tokens_seen": 1805189120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045751083293211364,
+      "loss": 1.2907,
+      "theoretical_loss": 3.4885683018724793,
+      "tokens_seen": 1805320192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045747071096132244,
+      "loss": 1.2733,
+      "theoretical_loss": 3.488546932928799,
+      "tokens_seen": 1805451264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045743058899053125,
+      "loss": 1.2341,
+      "theoretical_loss": 3.488525565970743,
+      "tokens_seen": 1805582336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045739046701974,
+      "loss": 1.3359,
+      "theoretical_loss": 3.4885042009979816,
+      "tokens_seen": 1805713408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573503450489488,
+      "loss": 1.2627,
+      "theoretical_loss": 3.4884828380101878,
+      "tokens_seen": 1805844480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573102230781576,
+      "loss": 1.2255,
+      "theoretical_loss": 3.488461477007032,
+      "tokens_seen": 1805975552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045727010110736637,
+      "loss": 1.3375,
+      "theoretical_loss": 3.488440117988186,
+      "tokens_seen": 1806106624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045722997913657523,
+      "loss": 1.2423,
+      "theoretical_loss": 3.488418760953322,
+      "tokens_seen": 1806237696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457189857165784,
+      "loss": 1.2288,
+      "theoretical_loss": 3.488397405902111,
+      "tokens_seen": 1806368768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571497351949928,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4883760528342247,
+      "tokens_seen": 1806499840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571096132242016,
+      "loss": 1.3532,
+      "theoretical_loss": 3.488354701749336,
+      "tokens_seen": 1806630912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045706949125341034,
+      "loss": 1.2219,
+      "theoretical_loss": 3.4883333526471167,
+      "tokens_seen": 1806761984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004570293692826192,
+      "loss": 1.1898,
+      "theoretical_loss": 3.4883120055272387,
+      "tokens_seen": 1806893056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045698924731182796,
+      "loss": 1.3131,
+      "theoretical_loss": 3.4882906603893735,
+      "tokens_seen": 1807024128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045694912534103676,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4882693172331942,
+      "tokens_seen": 1807155200
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 608509,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3582847118377686,
+      "objective/train/theoretical_loss": 3.4882479760583727,
+      "objective/train/tokens_used": 177745376,
+      "theoretical_loss": 3.4882479760583727,
+      "tokens_seen": 1807286272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045690900337024557,
+      "loss": 1.3645,
+      "theoretical_loss": 3.4882479760583727,
+      "tokens_seen": 1807286272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004568688813994544,
+      "loss": 1.2699,
+      "theoretical_loss": 3.4882266368645816,
+      "tokens_seen": 1807417344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045682875942866313,
+      "loss": 1.3004,
+      "theoretical_loss": 3.4882052996514927,
+      "tokens_seen": 1807548416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045678863745787193,
+      "loss": 1.2716,
+      "theoretical_loss": 3.4881839644187798,
+      "tokens_seen": 1807679488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045674851548708074,
+      "loss": 1.2668,
+      "theoretical_loss": 3.488162631166114,
+      "tokens_seen": 1807810560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567083935162895,
+      "loss": 1.3376,
+      "theoretical_loss": 3.4881412998931696,
+      "tokens_seen": 1807941632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045666827154549835,
+      "loss": 1.2833,
+      "theoretical_loss": 3.488119970599618,
+      "tokens_seen": 1808072704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566281495747071,
+      "loss": 1.313,
+      "theoretical_loss": 3.488098643285133,
+      "tokens_seen": 1808203776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565880276039159,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4880773179493874,
+      "tokens_seen": 1808334848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565479056331247,
+      "loss": 1.2987,
+      "theoretical_loss": 3.4880559945920533,
+      "tokens_seen": 1808465920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045650778366233347,
+      "loss": 1.2912,
+      "theoretical_loss": 3.488034673212805,
+      "tokens_seen": 1808596992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564676616915423,
+      "loss": 1.2755,
+      "theoretical_loss": 3.488013353811315,
+      "tokens_seen": 1808728064
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564275397207511,
+      "loss": 1.2864,
+      "theoretical_loss": 3.4879920363872565,
+      "tokens_seen": 1808859136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563874177499599,
+      "loss": 1.3471,
+      "theoretical_loss": 3.487970720940303,
+      "tokens_seen": 1808990208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563472957791687,
+      "loss": 1.2818,
+      "theoretical_loss": 3.4879494074701283,
+      "tokens_seen": 1809121280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563071738083775,
+      "loss": 1.3168,
+      "theoretical_loss": 3.4879280959764047,
+      "tokens_seen": 1809252352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045626705183758625,
+      "loss": 1.2704,
+      "theoretical_loss": 3.487906786458807,
+      "tokens_seen": 1809383424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562269298667951,
+      "loss": 1.2262,
+      "theoretical_loss": 3.4878854789170086,
+      "tokens_seen": 1809514496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045618680789600387,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4878641733506823,
+      "tokens_seen": 1809645568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561466859252126,
+      "loss": 1.2841,
+      "theoretical_loss": 3.487842869759503,
+      "tokens_seen": 1809776640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561065639544215,
+      "loss": 1.2222,
+      "theoretical_loss": 3.4878215681431444,
+      "tokens_seen": 1809907712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045606644198363023,
+      "loss": 1.3179,
+      "theoretical_loss": 3.48780026850128,
+      "tokens_seen": 1810038784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045602632001283904,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4877789708335833,
+      "tokens_seen": 1810169856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045598619804204784,
+      "loss": 1.2234,
+      "theoretical_loss": 3.48775767513973,
+      "tokens_seen": 1810300928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045594607607125665,
+      "loss": 1.2921,
+      "theoretical_loss": 3.487736381419393,
+      "tokens_seen": 1810432000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004559059541004654,
+      "loss": 1.3025,
+      "theoretical_loss": 3.487715089672246,
+      "tokens_seen": 1810563072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558658321296742,
+      "loss": 1.2253,
+      "theoretical_loss": 3.487693799897966,
+      "tokens_seen": 1810694144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455825710158883,
+      "loss": 1.302,
+      "theoretical_loss": 3.487672512096224,
+      "tokens_seen": 1810825216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045578558818809177,
+      "loss": 1.3187,
+      "theoretical_loss": 3.4876512262666965,
+      "tokens_seen": 1810956288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557454662173006,
+      "loss": 1.3333,
+      "theoretical_loss": 3.4876299424090584,
+      "tokens_seen": 1811087360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557053442465094,
+      "loss": 1.2708,
+      "theoretical_loss": 3.487608660522983,
+      "tokens_seen": 1811218432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045566522227571824,
+      "loss": 1.2979,
+      "theoretical_loss": 3.4875873806081454,
+      "tokens_seen": 1811349504
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455625100304927,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4875661026642213,
+      "tokens_seen": 1811480576
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045558497833413574,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4875448266908844,
+      "tokens_seen": 1811611648
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555448563633446,
+      "loss": 1.3523,
+      "theoretical_loss": 3.4875235526878106,
+      "tokens_seen": 1811742720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045550473439255336,
+      "loss": 1.2689,
+      "theoretical_loss": 3.487502280654674,
+      "tokens_seen": 1811873792
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045546461242176216,
+      "loss": 1.2689,
+      "theoretical_loss": 3.48748101059115,
+      "tokens_seen": 1812004864
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045542449045097097,
+      "loss": 1.2987,
+      "theoretical_loss": 3.487459742496914,
+      "tokens_seen": 1812135936
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553843684801798,
+      "loss": 1.3738,
+      "theoretical_loss": 3.4874384763716417,
+      "tokens_seen": 1812267008
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045534424650938853,
+      "loss": 1.231,
+      "theoretical_loss": 3.4874172122150076,
+      "tokens_seen": 1812398080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045530412453859733,
+      "loss": 1.2141,
+      "theoretical_loss": 3.487395950026688,
+      "tokens_seen": 1812529152
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045526400256780614,
+      "loss": 1.3411,
+      "theoretical_loss": 3.487374689806357,
+      "tokens_seen": 1812660224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552238805970149,
+      "loss": 1.3161,
+      "theoretical_loss": 3.487353431553691,
+      "tokens_seen": 1812791296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045518375862622375,
+      "loss": 1.3245,
+      "theoretical_loss": 3.4873321752683655,
+      "tokens_seen": 1812922368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551436366554325,
+      "loss": 1.2667,
+      "theoretical_loss": 3.4873109209500566,
+      "tokens_seen": 1813053440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045510351468464137,
+      "loss": 1.1924,
+      "theoretical_loss": 3.4872896685984403,
+      "tokens_seen": 1813184512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550633927138501,
+      "loss": 1.318,
+      "theoretical_loss": 3.4872684182131914,
+      "tokens_seen": 1813315584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045502327074305887,
+      "loss": 1.2505,
+      "theoretical_loss": 3.487247169793987,
+      "tokens_seen": 1813446656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045498314877226773,
+      "loss": 1.311,
+      "theoretical_loss": 3.487225923340502,
+      "tokens_seen": 1813577728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004549430268014765,
+      "loss": 1.2143,
+      "theoretical_loss": 3.487204678852413,
+      "tokens_seen": 1813708800
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 610938,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2813613414764404,
+      "objective/train/theoretical_loss": 3.4871834363293965,
+      "objective/train/tokens_used": 184298976,
+      "theoretical_loss": 3.4871834363293965,
+      "tokens_seen": 1813839872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004549029048306853,
+      "loss": 1.2636,
+      "theoretical_loss": 3.4871834363293965,
+      "tokens_seen": 1813839872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548627828598941,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4871621957711287,
+      "tokens_seen": 1813970944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548226608891029,
+      "loss": 1.2879,
+      "theoretical_loss": 3.487140957177285,
+      "tokens_seen": 1814102016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045478253891831165,
+      "loss": 1.3549,
+      "theoretical_loss": 3.4871197205475433,
+      "tokens_seen": 1814233088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547424169475205,
+      "loss": 1.2883,
+      "theoretical_loss": 3.487098485881579,
+      "tokens_seen": 1814364160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045470229497672927,
+      "loss": 1.2756,
+      "theoretical_loss": 3.487077253179069,
+      "tokens_seen": 1814495232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454662173005938,
+      "loss": 1.1934,
+      "theoretical_loss": 3.48705602243969,
+      "tokens_seen": 1814626304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546220510351469,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4870347936631183,
+      "tokens_seen": 1814757376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045458192906435563,
+      "loss": 1.3492,
+      "theoretical_loss": 3.4870135668490314,
+      "tokens_seen": 1814888448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045454180709356444,
+      "loss": 1.2206,
+      "theoretical_loss": 3.486992341997106,
+      "tokens_seen": 1815019520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045450168512277324,
+      "loss": 1.2694,
+      "theoretical_loss": 3.486971119107018,
+      "tokens_seen": 1815150592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045446156315198205,
+      "loss": 1.2221,
+      "theoretical_loss": 3.486949898178446,
+      "tokens_seen": 1815281664
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045442144118119086,
+      "loss": 1.3075,
+      "theoretical_loss": 3.486928679211066,
+      "tokens_seen": 1815412736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543813192103996,
+      "loss": 1.2936,
+      "theoretical_loss": 3.486907462204556,
+      "tokens_seen": 1815543808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543411972396084,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4868862471585924,
+      "tokens_seen": 1815674880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543010752688172,
+      "loss": 1.277,
+      "theoretical_loss": 3.486865034072853,
+      "tokens_seen": 1815805952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454260953298026,
+      "loss": 1.2783,
+      "theoretical_loss": 3.4868438229470144,
+      "tokens_seen": 1815937024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004542208313272348,
+      "loss": 1.2191,
+      "theoretical_loss": 3.4868226137807556,
+      "tokens_seen": 1816068096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045418070935644364,
+      "loss": 1.2746,
+      "theoretical_loss": 3.486801406573753,
+      "tokens_seen": 1816199168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004541405873856524,
+      "loss": 1.2482,
+      "theoretical_loss": 3.4867802013256846,
+      "tokens_seen": 1816330240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045410046541486114,
+      "loss": 1.2542,
+      "theoretical_loss": 3.486758998036228,
+      "tokens_seen": 1816461312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045406034344407,
+      "loss": 1.2258,
+      "theoretical_loss": 3.4867377967050617,
+      "tokens_seen": 1816592384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045402022147327876,
+      "loss": 1.2448,
+      "theoretical_loss": 3.486716597331862,
+      "tokens_seen": 1816723456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045398009950248756,
+      "loss": 1.2684,
+      "theoretical_loss": 3.486695399916308,
+      "tokens_seen": 1816854528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045393997753169637,
+      "loss": 1.2708,
+      "theoretical_loss": 3.486674204458078,
+      "tokens_seen": 1816985600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004538998555609052,
+      "loss": 1.2971,
+      "theoretical_loss": 3.486653010956849,
+      "tokens_seen": 1817116672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000453859733590114,
+      "loss": 1.304,
+      "theoretical_loss": 3.4866318194123,
+      "tokens_seen": 1817247744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045381961161932273,
+      "loss": 1.23,
+      "theoretical_loss": 3.486610629824109,
+      "tokens_seen": 1817378816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045377948964853154,
+      "loss": 1.2523,
+      "theoretical_loss": 3.486589442191954,
+      "tokens_seen": 1817509888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045373936767774035,
+      "loss": 1.2389,
+      "theoretical_loss": 3.486568256515514,
+      "tokens_seen": 1817640960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045369924570694915,
+      "loss": 1.2608,
+      "theoretical_loss": 3.486547072794467,
+      "tokens_seen": 1817772032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004536591237361579,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4865258910284913,
+      "tokens_seen": 1817903104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045361900176536676,
+      "loss": 1.2945,
+      "theoretical_loss": 3.4865047112172665,
+      "tokens_seen": 1818034176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004535788797945755,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4864835333604702,
+      "tokens_seen": 1818165248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045353875782378427,
+      "loss": 1.2974,
+      "theoretical_loss": 3.486462357457782,
+      "tokens_seen": 1818296320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045349863585299313,
+      "loss": 1.3206,
+      "theoretical_loss": 3.4864411835088807,
+      "tokens_seen": 1818427392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004534585138822019,
+      "loss": 1.2743,
+      "theoretical_loss": 3.4864200115134443,
+      "tokens_seen": 1818558464
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004534183919114107,
+      "loss": 1.294,
+      "theoretical_loss": 3.486398841471153,
+      "tokens_seen": 1818689536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004533782699406195,
+      "loss": 1.3143,
+      "theoretical_loss": 3.486377673381685,
+      "tokens_seen": 1818820608
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004533381479698283,
+      "loss": 1.1605,
+      "theoretical_loss": 3.4863565072447202,
+      "tokens_seen": 1818951680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045329802599903705,
+      "loss": 1.2951,
+      "theoretical_loss": 3.4863353430599373,
+      "tokens_seen": 1819082752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004532579040282459,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4863141808270153,
+      "tokens_seen": 1819213824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045321778205745467,
+      "loss": 1.3084,
+      "theoretical_loss": 3.4862930205456344,
+      "tokens_seen": 1819344896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045317766008666347,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4862718622154736,
+      "tokens_seen": 1819475968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004531375381158723,
+      "loss": 1.2897,
+      "theoretical_loss": 3.486250705836212,
+      "tokens_seen": 1819607040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045309741614508103,
+      "loss": 1.256,
+      "theoretical_loss": 3.48622955140753,
+      "tokens_seen": 1819738112
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004530572941742899,
+      "loss": 1.2197,
+      "theoretical_loss": 3.486208398929108,
+      "tokens_seen": 1819869184
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045301717220349864,
+      "loss": 1.2749,
+      "theoretical_loss": 3.4861872484006233,
+      "tokens_seen": 1820000256
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045297705023270745,
+      "loss": 1.254,
+      "theoretical_loss": 3.486166099821758,
+      "tokens_seen": 1820131328
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045293692826191626,
+      "loss": 1.2785,
+      "theoretical_loss": 3.4861449531921904,
+      "tokens_seen": 1820262400
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 613170,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3238725662231445,
+      "objective/train/theoretical_loss": 3.486123808511602,
+      "objective/train/tokens_used": 190852576,
+      "theoretical_loss": 3.486123808511602,
+      "tokens_seen": 1820393472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000452896806291125,
+      "loss": 1.3345,
+      "theoretical_loss": 3.486123808511602,
+      "tokens_seen": 1820393472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004528566843203338,
+      "loss": 1.2566,
+      "theoretical_loss": 3.486102665779672,
+      "tokens_seen": 1820524544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004528165623495426,
+      "loss": 1.33,
+      "theoretical_loss": 3.4860815249960804,
+      "tokens_seen": 1820655616
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004527764403787514,
+      "loss": 1.3365,
+      "theoretical_loss": 3.486060386160508,
+      "tokens_seen": 1820786688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004527363184079602,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4860392492726344,
+      "tokens_seen": 1820917760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045269619643716904,
+      "loss": 1.2941,
+      "theoretical_loss": 3.4860181143321407,
+      "tokens_seen": 1821048832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004526560744663778,
+      "loss": 1.2326,
+      "theoretical_loss": 3.485996981338707,
+      "tokens_seen": 1821179904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045261595249558654,
+      "loss": 1.2552,
+      "theoretical_loss": 3.4859758502920135,
+      "tokens_seen": 1821310976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004525758305247954,
+      "loss": 1.2446,
+      "theoretical_loss": 3.4859547211917414,
+      "tokens_seen": 1821442048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045253570855400416,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4859335940375704,
+      "tokens_seen": 1821573120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000452495586583213,
+      "loss": 1.2667,
+      "theoretical_loss": 3.4859124688291825,
+      "tokens_seen": 1821704192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045245546461242177,
+      "loss": 1.2715,
+      "theoretical_loss": 3.485891345566258,
+      "tokens_seen": 1821835264
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004524153426416306,
+      "loss": 1.1905,
+      "theoretical_loss": 3.485870224248477,
+      "tokens_seen": 1821966336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004523752206708394,
+      "loss": 1.3431,
+      "theoretical_loss": 3.4858491048755216,
+      "tokens_seen": 1822097408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045233509870004813,
+      "loss": 1.2775,
+      "theoretical_loss": 3.4858279874470726,
+      "tokens_seen": 1822228480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045229497672925694,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4858068719628106,
+      "tokens_seen": 1822359552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045225485475846575,
+      "loss": 1.2859,
+      "theoretical_loss": 3.4857857584224172,
+      "tokens_seen": 1822490624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045221473278767455,
+      "loss": 1.2377,
+      "theoretical_loss": 3.4857646468255736,
+      "tokens_seen": 1822621696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004521746108168833,
+      "loss": 1.3397,
+      "theoretical_loss": 3.4857435371719614,
+      "tokens_seen": 1822752768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045213448884609216,
+      "loss": 1.2526,
+      "theoretical_loss": 3.485722429461261,
+      "tokens_seen": 1822883840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004520943668753009,
+      "loss": 1.2066,
+      "theoretical_loss": 3.485701323693155,
+      "tokens_seen": 1823014912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045205424490450967,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4856802198673247,
+      "tokens_seen": 1823145984
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045201412293371853,
+      "loss": 1.2771,
+      "theoretical_loss": 3.4856591179834515,
+      "tokens_seen": 1823277056
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004519740009629273,
+      "loss": 1.2188,
+      "theoretical_loss": 3.485638018041217,
+      "tokens_seen": 1823408128
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045193387899213614,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4856169200403038,
+      "tokens_seen": 1823539200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004518937570213449,
+      "loss": 1.3114,
+      "theoretical_loss": 3.4855958239803932,
+      "tokens_seen": 1823670272
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004518536350505537,
+      "loss": 1.281,
+      "theoretical_loss": 3.485574729861167,
+      "tokens_seen": 1823801344
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004518135130797625,
+      "loss": 1.3477,
+      "theoretical_loss": 3.4855536376823073,
+      "tokens_seen": 1823932416
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045177339110897126,
+      "loss": 1.3095,
+      "theoretical_loss": 3.4855325474434964,
+      "tokens_seen": 1824063488
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045173326913818006,
+      "loss": 1.3233,
+      "theoretical_loss": 3.4855114591444165,
+      "tokens_seen": 1824194560
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045169314716738887,
+      "loss": 1.3272,
+      "theoretical_loss": 3.4854903727847493,
+      "tokens_seen": 1824325632
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004516530251965977,
+      "loss": 1.2794,
+      "theoretical_loss": 3.4854692883641776,
+      "tokens_seen": 1824456704
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045161290322580643,
+      "loss": 1.3434,
+      "theoretical_loss": 3.485448205882384,
+      "tokens_seen": 1824587776
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004515727812550153,
+      "loss": 1.2806,
+      "theoretical_loss": 3.48542712533905,
+      "tokens_seen": 1824718848
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045153265928422404,
+      "loss": 1.3252,
+      "theoretical_loss": 3.4854060467338597,
+      "tokens_seen": 1824849920
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045149253731343285,
+      "loss": 1.2636,
+      "theoretical_loss": 3.485384970066494,
+      "tokens_seen": 1824980992
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045145241534264165,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4853638953366373,
+      "tokens_seen": 1825112064
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004514122933718504,
+      "loss": 1.3383,
+      "theoretical_loss": 3.485342822543971,
+      "tokens_seen": 1825243136
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004513721714010592,
+      "loss": 1.3662,
+      "theoretical_loss": 3.485321751688178,
+      "tokens_seen": 1825374208
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000451332049430268,
+      "loss": 1.2782,
+      "theoretical_loss": 3.485300682768942,
+      "tokens_seen": 1825505280
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004512919274594768,
+      "loss": 1.2146,
+      "theoretical_loss": 3.4852796157859456,
+      "tokens_seen": 1825636352
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045125180548868563,
+      "loss": 1.2328,
+      "theoretical_loss": 3.485258550738872,
+      "tokens_seen": 1825767424
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045121168351789444,
+      "loss": 1.3077,
+      "theoretical_loss": 3.485237487627404,
+      "tokens_seen": 1825898496
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004511715615471032,
+      "loss": 1.293,
+      "theoretical_loss": 3.4852164264512253,
+      "tokens_seen": 1826029568
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000451131439576312,
+      "loss": 1.2677,
+      "theoretical_loss": 3.485195367210019,
+      "tokens_seen": 1826160640
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004510913176055208,
+      "loss": 1.2885,
+      "theoretical_loss": 3.485174309903468,
+      "tokens_seen": 1826291712
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045105119563472956,
+      "loss": 1.3336,
+      "theoretical_loss": 3.4851532545312565,
+      "tokens_seen": 1826422784
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004510110736639384,
+      "loss": 1.3465,
+      "theoretical_loss": 3.4851322010930676,
+      "tokens_seen": 1826553856
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045097095169314717,
+      "loss": 1.3219,
+      "theoretical_loss": 3.485111149588585,
+      "tokens_seen": 1826684928
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000450930829722356,
+      "loss": 1.3617,
+      "theoretical_loss": 3.485090100017492,
+      "tokens_seen": 1826816000
+    },
+    {
+      "epoch": 0.11,
+      "objective/train/docs_used": 615374,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3149850368499756,
+      "objective/train/theoretical_loss": 3.485069052379473,
+      "objective/train/tokens_used": 197406176,
+      "theoretical_loss": 3.485069052379473,
+      "tokens_seen": 1826947072
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004508907077515648,
+      "loss": 1.2777,
+      "theoretical_loss": 3.485069052379473,
+      "tokens_seen": 1826947072
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045085058578077353,
+      "loss": 1.1969,
+      "theoretical_loss": 3.4850480066742113,
+      "tokens_seen": 1827078144
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045081046380998234,
+      "loss": 1.269,
+      "theoretical_loss": 3.485026962901391,
+      "tokens_seen": 1827209216
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045077034183919115,
+      "loss": 1.3328,
+      "theoretical_loss": 3.4850059210606954,
+      "tokens_seen": 1827340288
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045073021986839995,
+      "loss": 1.2957,
+      "theoretical_loss": 3.48498488115181,
+      "tokens_seen": 1827471360
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045069009789760876,
+      "loss": 1.2128,
+      "theoretical_loss": 3.484963843174418,
+      "tokens_seen": 1827602432
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045064997592681756,
+      "loss": 1.2084,
+      "theoretical_loss": 3.4849428071282027,
+      "tokens_seen": 1827733504
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004506098539560263,
+      "loss": 1.297,
+      "theoretical_loss": 3.48492177301285,
+      "tokens_seen": 1827864576
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004505697319852351,
+      "loss": 1.3208,
+      "theoretical_loss": 3.484900740828044,
+      "tokens_seen": 1827995648
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045052961001444393,
+      "loss": 1.2377,
+      "theoretical_loss": 3.484879710573468,
+      "tokens_seen": 1828126720
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004504894880436527,
+      "loss": 1.2995,
+      "theoretical_loss": 3.484858682248807,
+      "tokens_seen": 1828257792
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045044936607286154,
+      "loss": 1.2886,
+      "theoretical_loss": 3.4848376558537453,
+      "tokens_seen": 1828388864
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004504092441020703,
+      "loss": 1.3586,
+      "theoretical_loss": 3.4848166313879685,
+      "tokens_seen": 1828519936
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004503691221312791,
+      "loss": 1.247,
+      "theoretical_loss": 3.4847956088511602,
+      "tokens_seen": 1828651008
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004503290001604879,
+      "loss": 1.3103,
+      "theoretical_loss": 3.484774588243006,
+      "tokens_seen": 1828782080
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045028887818969666,
+      "loss": 1.3041,
+      "theoretical_loss": 3.484753569563191,
+      "tokens_seen": 1828913152
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045024875621890546,
+      "loss": 1.2282,
+      "theoretical_loss": 3.4847325528113986,
+      "tokens_seen": 1829044224
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045020863424811427,
+      "loss": 1.2436,
+      "theoretical_loss": 3.484711537987315,
+      "tokens_seen": 1829175296
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004501685122773231,
+      "loss": 1.2743,
+      "theoretical_loss": 3.484690525090625,
+      "tokens_seen": 1829306368
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045012839030653183,
+      "loss": 1.3272,
+      "theoretical_loss": 3.4846695141210136,
+      "tokens_seen": 1829437440
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004500882683357407,
+      "loss": 1.2696,
+      "theoretical_loss": 3.484648505078166,
+      "tokens_seen": 1829568512
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045004814636494944,
+      "loss": 1.3314,
+      "theoretical_loss": 3.484627497961768,
+      "tokens_seen": 1829699584
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004500080243941583,
+      "loss": 1.2602,
+      "theoretical_loss": 3.484606492771505,
+      "tokens_seen": 1829830656
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044996790242336705,
+      "loss": 1.3297,
+      "theoretical_loss": 3.484585489507061,
+      "tokens_seen": 1829961728
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004499277804525758,
+      "loss": 1.3446,
+      "theoretical_loss": 3.4845644881681235,
+      "tokens_seen": 1830092800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044988765848178467,
+      "loss": 1.2974,
+      "theoretical_loss": 3.484543488754377,
+      "tokens_seen": 1830223872
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004498475365109934,
+      "loss": 1.3032,
+      "theoretical_loss": 3.484522491265507,
+      "tokens_seen": 1830354944
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004498074145402022,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4845014957012,
+      "tokens_seen": 1830486016
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044976729256941103,
+      "loss": 1.3171,
+      "theoretical_loss": 3.4844805020611407,
+      "tokens_seen": 1830617088
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044972717059861984,
+      "loss": 1.3472,
+      "theoretical_loss": 3.484459510345016,
+      "tokens_seen": 1830748160
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004496870486278286,
+      "loss": 1.2964,
+      "theoretical_loss": 3.484438520552512,
+      "tokens_seen": 1830879232
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004496469266570374,
+      "loss": 1.2537,
+      "theoretical_loss": 3.484417532683314,
+      "tokens_seen": 1831010304
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004496068046862462,
+      "loss": 1.3605,
+      "theoretical_loss": 3.4843965467371083,
+      "tokens_seen": 1831141376
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044956668271545495,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4843755627135815,
+      "tokens_seen": 1831272448
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004495265607446638,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4843545806124188,
+      "tokens_seen": 1831403520
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044948643877387257,
+      "loss": 1.3381,
+      "theoretical_loss": 3.484333600433308,
+      "tokens_seen": 1831534592
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044944631680308143,
+      "loss": 1.2754,
+      "theoretical_loss": 3.484312622175934,
+      "tokens_seen": 1831665664
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004494061948322902,
+      "loss": 1.2817,
+      "theoretical_loss": 3.484291645839984,
+      "tokens_seen": 1831796736
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044936607286149893,
+      "loss": 1.3396,
+      "theoretical_loss": 3.4842706714251452,
+      "tokens_seen": 1831927808
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004493259508907078,
+      "loss": 1.1982,
+      "theoretical_loss": 3.4842496989311034,
+      "tokens_seen": 1832058880
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044928582891991654,
+      "loss": 1.3134,
+      "theoretical_loss": 3.484228728357545,
+      "tokens_seen": 1832189952
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044924570694912535,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4842077597041574,
+      "tokens_seen": 1832321024
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044920558497833416,
+      "loss": 1.3392,
+      "theoretical_loss": 3.4841867929706276,
+      "tokens_seen": 1832452096
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044916546300754296,
+      "loss": 1.3293,
+      "theoretical_loss": 3.484165828156642,
+      "tokens_seen": 1832583168
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004491253410367517,
+      "loss": 1.3137,
+      "theoretical_loss": 3.4841448652618876,
+      "tokens_seen": 1832714240
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004490852190659605,
+      "loss": 1.1722,
+      "theoretical_loss": 3.484123904286052,
+      "tokens_seen": 1832845312
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044904509709516933,
+      "loss": 1.2462,
+      "theoretical_loss": 3.484102945228822,
+      "tokens_seen": 1832976384
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004490049751243781,
+      "loss": 1.3163,
+      "theoretical_loss": 3.4840819880898843,
+      "tokens_seen": 1833107456
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044896485315358694,
+      "loss": 1.3,
+      "theoretical_loss": 3.484061032868927,
+      "tokens_seen": 1833238528
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004489247311827957,
+      "loss": 1.3591,
+      "theoretical_loss": 3.484040079565637,
+      "tokens_seen": 1833369600
+    },
+    {
+      "epoch": 0.11,
+      "objective/train/docs_used": 617542,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2098336219787598,
+      "objective/train/theoretical_loss": 3.4840191281797024,
+      "objective/train/tokens_used": 203959776,
+      "theoretical_loss": 3.4840191281797024,
+      "tokens_seen": 1833500672
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004488846092120045,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4840191281797024,
+      "tokens_seen": 1833500672
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004488444872412133,
+      "loss": 1.3588,
+      "theoretical_loss": 3.4839981787108094,
+      "tokens_seen": 1833631744
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044880436527042206,
+      "loss": 1.3145,
+      "theoretical_loss": 3.483977231158647,
+      "tokens_seen": 1833762816
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004487642432996309,
+      "loss": 1.2945,
+      "theoretical_loss": 3.483956285522902,
+      "tokens_seen": 1833893888
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044872412132883967,
+      "loss": 1.2748,
+      "theoretical_loss": 3.483935341803262,
+      "tokens_seen": 1834024960
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004486839993580485,
+      "loss": 1.2255,
+      "theoretical_loss": 3.4839143999994153,
+      "tokens_seen": 1834156032
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004486438773872573,
+      "loss": 1.2919,
+      "theoretical_loss": 3.48389346011105,
+      "tokens_seen": 1834287104
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004486037554164661,
+      "loss": 1.2402,
+      "theoretical_loss": 3.483872522137853,
+      "tokens_seen": 1834418176
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044856363344567484,
+      "loss": 1.2806,
+      "theoretical_loss": 3.483851586079513,
+      "tokens_seen": 1834549248
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004485235114748837,
+      "loss": 1.3304,
+      "theoretical_loss": 3.483830651935718,
+      "tokens_seen": 1834680320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044848338950409245,
+      "loss": 1.238,
+      "theoretical_loss": 3.4838097197061564,
+      "tokens_seen": 1834811392
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004484432675333012,
+      "loss": 1.307,
+      "theoretical_loss": 3.483788789390516,
+      "tokens_seen": 1834942464
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044840314556251007,
+      "loss": 1.2528,
+      "theoretical_loss": 3.4837678609884857,
+      "tokens_seen": 1835073536
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004483630235917188,
+      "loss": 1.3208,
+      "theoretical_loss": 3.483746934499753,
+      "tokens_seen": 1835204608
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004483229016209276,
+      "loss": 1.2637,
+      "theoretical_loss": 3.4837260099240073,
+      "tokens_seen": 1835335680
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044828277965013643,
+      "loss": 1.316,
+      "theoretical_loss": 3.4837050872609368,
+      "tokens_seen": 1835466752
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044824265767934524,
+      "loss": 1.3211,
+      "theoretical_loss": 3.48368416651023,
+      "tokens_seen": 1835597824
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000448202535708554,
+      "loss": 1.2594,
+      "theoretical_loss": 3.4836632476715748,
+      "tokens_seen": 1835728896
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004481624137377628,
+      "loss": 1.3187,
+      "theoretical_loss": 3.483642330744661,
+      "tokens_seen": 1835859968
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004481222917669716,
+      "loss": 1.3645,
+      "theoretical_loss": 3.4836214157291776,
+      "tokens_seen": 1835991040
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004480821697961804,
+      "loss": 1.2308,
+      "theoretical_loss": 3.4836005026248125,
+      "tokens_seen": 1836122112
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004480420478253892,
+      "loss": 1.2455,
+      "theoretical_loss": 3.4835795914312557,
+      "tokens_seen": 1836253184
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044800192585459797,
+      "loss": 1.2916,
+      "theoretical_loss": 3.483558682148195,
+      "tokens_seen": 1836384256
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044796180388380683,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4835377747753205,
+      "tokens_seen": 1836515328
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004479216819130156,
+      "loss": 1.3062,
+      "theoretical_loss": 3.4835168693123215,
+      "tokens_seen": 1836646400
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044788155994222433,
+      "loss": 1.2106,
+      "theoretical_loss": 3.4834959657588866,
+      "tokens_seen": 1836777472
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004478414379714332,
+      "loss": 1.3415,
+      "theoretical_loss": 3.483475064114705,
+      "tokens_seen": 1836908544
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044780131600064194,
+      "loss": 1.3818,
+      "theoretical_loss": 3.483454164379467,
+      "tokens_seen": 1837039616
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044776119402985075,
+      "loss": 1.3122,
+      "theoretical_loss": 3.4834332665528613,
+      "tokens_seen": 1837170688
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044772107205905956,
+      "loss": 1.2908,
+      "theoretical_loss": 3.4834123706345776,
+      "tokens_seen": 1837301760
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044768095008826836,
+      "loss": 1.3105,
+      "theoretical_loss": 3.483391476624306,
+      "tokens_seen": 1837432832
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004476408281174771,
+      "loss": 1.2142,
+      "theoretical_loss": 3.4833705845217358,
+      "tokens_seen": 1837563904
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004476007061466859,
+      "loss": 1.3287,
+      "theoretical_loss": 3.4833496943265567,
+      "tokens_seen": 1837694976
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044756058417589473,
+      "loss": 1.2577,
+      "theoretical_loss": 3.4833288060384584,
+      "tokens_seen": 1837826048
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004475204622051035,
+      "loss": 1.3351,
+      "theoretical_loss": 3.483307919657131,
+      "tokens_seen": 1837957120
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044748034023431234,
+      "loss": 1.2205,
+      "theoretical_loss": 3.483287035182265,
+      "tokens_seen": 1838088192
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004474402182635211,
+      "loss": 1.2517,
+      "theoretical_loss": 3.483266152613549,
+      "tokens_seen": 1838219264
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044740009629272995,
+      "loss": 1.3149,
+      "theoretical_loss": 3.4832452719506746,
+      "tokens_seen": 1838350336
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004473599743219387,
+      "loss": 1.2449,
+      "theoretical_loss": 3.483224393193332,
+      "tokens_seen": 1838481408
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044731985235114746,
+      "loss": 1.2488,
+      "theoretical_loss": 3.4832035163412103,
+      "tokens_seen": 1838612480
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004472797303803563,
+      "loss": 1.2209,
+      "theoretical_loss": 3.4831826413940004,
+      "tokens_seen": 1838743552
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044723960840956507,
+      "loss": 1.2083,
+      "theoretical_loss": 3.483161768351393,
+      "tokens_seen": 1838874624
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471994864387739,
+      "loss": 1.3279,
+      "theoretical_loss": 3.4831408972130786,
+      "tokens_seen": 1839005696
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471593644679827,
+      "loss": 1.2327,
+      "theoretical_loss": 3.4831200279787478,
+      "tokens_seen": 1839136768
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471192424971915,
+      "loss": 1.3244,
+      "theoretical_loss": 3.4830991606480906,
+      "tokens_seen": 1839267840
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044707912052640024,
+      "loss": 1.3092,
+      "theoretical_loss": 3.483078295220798,
+      "tokens_seen": 1839398912
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044703899855560905,
+      "loss": 1.3029,
+      "theoretical_loss": 3.483057431696561,
+      "tokens_seen": 1839529984
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044699887658481785,
+      "loss": 1.2686,
+      "theoretical_loss": 3.48303657007507,
+      "tokens_seen": 1839661056
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004469587546140266,
+      "loss": 1.3008,
+      "theoretical_loss": 3.4830157103560166,
+      "tokens_seen": 1839792128
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044691863264323547,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4829948525390915,
+      "tokens_seen": 1839923200
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 619775,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.235106110572815,
+      "objective/train/theoretical_loss": 3.4829739966239854,
+      "objective/train/tokens_used": 210513376,
+      "theoretical_loss": 3.4829739966239854,
+      "tokens_seen": 1840054272
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004468785106724442,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4829739966239854,
+      "tokens_seen": 1840054272
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004468383887016531,
+      "loss": 1.224,
+      "theoretical_loss": 3.4829531426103904,
+      "tokens_seen": 1840185344
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044679826673086183,
+      "loss": 1.3012,
+      "theoretical_loss": 3.4829322904979962,
+      "tokens_seen": 1840316416
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044675814476007064,
+      "loss": 1.2193,
+      "theoretical_loss": 3.482911440286496,
+      "tokens_seen": 1840447488
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044671802278927944,
+      "loss": 1.2802,
+      "theoretical_loss": 3.482890591975579,
+      "tokens_seen": 1840578560
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004466779008184882,
+      "loss": 1.2823,
+      "theoretical_loss": 3.4828697455649382,
+      "tokens_seen": 1840709632
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000446637778847697,
+      "loss": 1.3292,
+      "theoretical_loss": 3.4828489010542647,
+      "tokens_seen": 1840840704
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004465976568769058,
+      "loss": 1.2502,
+      "theoretical_loss": 3.48282805844325,
+      "tokens_seen": 1840971776
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004465575349061146,
+      "loss": 1.2222,
+      "theoretical_loss": 3.4828072177315854,
+      "tokens_seen": 1841102848
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044651741293532337,
+      "loss": 1.2558,
+      "theoretical_loss": 3.4827863789189633,
+      "tokens_seen": 1841233920
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044647729096453223,
+      "loss": 1.3554,
+      "theoretical_loss": 3.482765542005075,
+      "tokens_seen": 1841364992
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000446437168993741,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4827447069896125,
+      "tokens_seen": 1841496064
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044639704702294973,
+      "loss": 1.3982,
+      "theoretical_loss": 3.482723873872268,
+      "tokens_seen": 1841627136
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004463569250521586,
+      "loss": 1.3583,
+      "theoretical_loss": 3.482703042652733,
+      "tokens_seen": 1841758208
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044631680308136734,
+      "loss": 1.2326,
+      "theoretical_loss": 3.4826822133306994,
+      "tokens_seen": 1841889280
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044627668111057615,
+      "loss": 1.2641,
+      "theoretical_loss": 3.4826613859058604,
+      "tokens_seen": 1842020352
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044623655913978496,
+      "loss": 1.3246,
+      "theoretical_loss": 3.4826405603779067,
+      "tokens_seen": 1842151424
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044619643716899376,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4826197367465324,
+      "tokens_seen": 1842282496
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044615631519820257,
+      "loss": 1.1839,
+      "theoretical_loss": 3.4825989150114287,
+      "tokens_seen": 1842413568
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004461161932274113,
+      "loss": 1.3222,
+      "theoretical_loss": 3.482578095172288,
+      "tokens_seen": 1842544640
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044607607125662013,
+      "loss": 1.3401,
+      "theoretical_loss": 3.4825572772288034,
+      "tokens_seen": 1842675712
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044603594928582893,
+      "loss": 1.3144,
+      "theoretical_loss": 3.4825364611806666,
+      "tokens_seen": 1842806784
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044599582731503774,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4825156470275713,
+      "tokens_seen": 1842937856
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004459557053442465,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4824948347692093,
+      "tokens_seen": 1843068928
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044591558337345535,
+      "loss": 1.1952,
+      "theoretical_loss": 3.482474024405274,
+      "tokens_seen": 1843200000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004458754614026641,
+      "loss": 1.2698,
+      "theoretical_loss": 3.4824532159354584,
+      "tokens_seen": 1843331072
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044583533943187286,
+      "loss": 1.2619,
+      "theoretical_loss": 3.482432409359455,
+      "tokens_seen": 1843462144
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004457952174610817,
+      "loss": 1.2666,
+      "theoretical_loss": 3.4824116046769564,
+      "tokens_seen": 1843593216
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044575509549029047,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4823908018876564,
+      "tokens_seen": 1843724288
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004457149735194993,
+      "loss": 1.2067,
+      "theoretical_loss": 3.4823700009912484,
+      "tokens_seen": 1843855360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004456748515487081,
+      "loss": 1.2353,
+      "theoretical_loss": 3.4823492019874247,
+      "tokens_seen": 1843986432
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004456347295779169,
+      "loss": 1.2104,
+      "theoretical_loss": 3.4823284048758794,
+      "tokens_seen": 1844117504
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004455946076071257,
+      "loss": 1.2548,
+      "theoretical_loss": 3.482307609656305,
+      "tokens_seen": 1844248576
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044555448563633445,
+      "loss": 1.1884,
+      "theoretical_loss": 3.4822868163283958,
+      "tokens_seen": 1844379648
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044551436366554325,
+      "loss": 1.2389,
+      "theoretical_loss": 3.482266024891845,
+      "tokens_seen": 1844510720
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044547424169475206,
+      "loss": 1.288,
+      "theoretical_loss": 3.482245235346346,
+      "tokens_seen": 1844641792
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044543411972396087,
+      "loss": 1.2473,
+      "theoretical_loss": 3.4822244476915927,
+      "tokens_seen": 1844772864
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004453939977531696,
+      "loss": 1.2458,
+      "theoretical_loss": 3.4822036619272785,
+      "tokens_seen": 1844903936
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004453538757823785,
+      "loss": 1.2705,
+      "theoretical_loss": 3.4821828780530977,
+      "tokens_seen": 1845035008
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044531375381158723,
+      "loss": 1.2943,
+      "theoretical_loss": 3.482162096068744,
+      "tokens_seen": 1845166080
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044527363184079604,
+      "loss": 1.2117,
+      "theoretical_loss": 3.482141315973911,
+      "tokens_seen": 1845297152
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044523350987000484,
+      "loss": 1.26,
+      "theoretical_loss": 3.482120537768293,
+      "tokens_seen": 1845428224
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004451933878992136,
+      "loss": 1.255,
+      "theoretical_loss": 3.482099761451584,
+      "tokens_seen": 1845559296
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004451532659284224,
+      "loss": 1.2167,
+      "theoretical_loss": 3.482078987023478,
+      "tokens_seen": 1845690368
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004451131439576312,
+      "loss": 1.2875,
+      "theoretical_loss": 3.4820582144836694,
+      "tokens_seen": 1845821440
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044507302198684,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4820374438318527,
+      "tokens_seen": 1845952512
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044503290001604877,
+      "loss": 1.239,
+      "theoretical_loss": 3.482016675067722,
+      "tokens_seen": 1846083584
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004449927780452576,
+      "loss": 1.2506,
+      "theoretical_loss": 3.4819959081909717,
+      "tokens_seen": 1846214656
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004449526560744664,
+      "loss": 1.2492,
+      "theoretical_loss": 3.481975143201296,
+      "tokens_seen": 1846345728
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004449125341036752,
+      "loss": 1.2355,
+      "theoretical_loss": 3.48195438009839,
+      "tokens_seen": 1846476800
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 621685,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3300907611846924,
+      "objective/train/theoretical_loss": 3.4819336188819485,
+      "objective/train/tokens_used": 217066976,
+      "theoretical_loss": 3.4819336188819485,
+      "tokens_seen": 1846607872
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000444872412132884,
+      "loss": 1.2572,
+      "theoretical_loss": 3.4819336188819485,
+      "tokens_seen": 1846607872
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044483229016209274,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4819128595516657,
+      "tokens_seen": 1846738944
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004447921681913016,
+      "loss": 1.256,
+      "theoretical_loss": 3.4818921021072367,
+      "tokens_seen": 1846870016
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044475204622051036,
+      "loss": 1.2582,
+      "theoretical_loss": 3.481871346548356,
+      "tokens_seen": 1847001088
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044471192424971916,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4818505928747197,
+      "tokens_seen": 1847132160
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044467180227892797,
+      "loss": 1.3851,
+      "theoretical_loss": 3.4818298410860207,
+      "tokens_seen": 1847263232
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004446316803081367,
+      "loss": 1.2724,
+      "theoretical_loss": 3.4818090911819564,
+      "tokens_seen": 1847394304
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044459155833734553,
+      "loss": 1.2435,
+      "theoretical_loss": 3.4817883431622203,
+      "tokens_seen": 1847525376
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044455143636655433,
+      "loss": 1.3089,
+      "theoretical_loss": 3.481767597026508,
+      "tokens_seen": 1847656448
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044451131439576314,
+      "loss": 1.376,
+      "theoretical_loss": 3.481746852774515,
+      "tokens_seen": 1847787520
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004444711924249719,
+      "loss": 1.2706,
+      "theoretical_loss": 3.4817261104059365,
+      "tokens_seen": 1847918592
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044443107045418075,
+      "loss": 1.3276,
+      "theoretical_loss": 3.4817053699204683,
+      "tokens_seen": 1848049664
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004443909484833895,
+      "loss": 1.3104,
+      "theoretical_loss": 3.4816846313178056,
+      "tokens_seen": 1848180736
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044435082651259826,
+      "loss": 1.3387,
+      "theoretical_loss": 3.4816638945976437,
+      "tokens_seen": 1848311808
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004443107045418071,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4816431597596784,
+      "tokens_seen": 1848442880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044427058257101587,
+      "loss": 1.2741,
+      "theoretical_loss": 3.481622426803606,
+      "tokens_seen": 1848573952
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044423046060022473,
+      "loss": 1.2035,
+      "theoretical_loss": 3.481601695729122,
+      "tokens_seen": 1848705024
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004441903386294335,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4815809665359216,
+      "tokens_seen": 1848836096
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004441502166586423,
+      "loss": 1.2226,
+      "theoretical_loss": 3.4815602392237013,
+      "tokens_seen": 1848967168
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004441100946878511,
+      "loss": 1.367,
+      "theoretical_loss": 3.481539513792157,
+      "tokens_seen": 1849098240
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044406997271705985,
+      "loss": 1.3494,
+      "theoretical_loss": 3.481518790240985,
+      "tokens_seen": 1849229312
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044402985074626865,
+      "loss": 1.1773,
+      "theoretical_loss": 3.4814980685698806,
+      "tokens_seen": 1849360384
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044398972877547746,
+      "loss": 1.2517,
+      "theoretical_loss": 3.481477348778541,
+      "tokens_seen": 1849491456
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044394960680468627,
+      "loss": 1.2918,
+      "theoretical_loss": 3.4814566308666626,
+      "tokens_seen": 1849622528
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000443909484833895,
+      "loss": 1.2428,
+      "theoretical_loss": 3.481435914833941,
+      "tokens_seen": 1849753600
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004438693628631039,
+      "loss": 1.3075,
+      "theoretical_loss": 3.4814152006800723,
+      "tokens_seen": 1849884672
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044382924089231263,
+      "loss": 1.3383,
+      "theoretical_loss": 3.481394488404754,
+      "tokens_seen": 1850015744
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044378911892152144,
+      "loss": 1.2695,
+      "theoretical_loss": 3.481373778007682,
+      "tokens_seen": 1850146816
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044374899695073024,
+      "loss": 1.3137,
+      "theoretical_loss": 3.481353069488553,
+      "tokens_seen": 1850277888
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000443708874979939,
+      "loss": 1.31,
+      "theoretical_loss": 3.4813323628470645,
+      "tokens_seen": 1850408960
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044366875300914786,
+      "loss": 1.2774,
+      "theoretical_loss": 3.481311658082912,
+      "tokens_seen": 1850540032
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004436286310383566,
+      "loss": 1.3572,
+      "theoretical_loss": 3.481290955195793,
+      "tokens_seen": 1850671104
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004435885090675654,
+      "loss": 1.2993,
+      "theoretical_loss": 3.4812702541854046,
+      "tokens_seen": 1850802176
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004435483870967742,
+      "loss": 1.2015,
+      "theoretical_loss": 3.4812495550514435,
+      "tokens_seen": 1850933248
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000443508265125983,
+      "loss": 1.2773,
+      "theoretical_loss": 3.481228857793607,
+      "tokens_seen": 1851064320
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004434681431551918,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4812081624115914,
+      "tokens_seen": 1851195392
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004434280211844006,
+      "loss": 1.2884,
+      "theoretical_loss": 3.4811874689050955,
+      "tokens_seen": 1851326464
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004433878992136094,
+      "loss": 1.1599,
+      "theoretical_loss": 3.4811667772738146,
+      "tokens_seen": 1851457536
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044334777724281814,
+      "loss": 1.1934,
+      "theoretical_loss": 3.4811460875174474,
+      "tokens_seen": 1851588608
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000443307655272027,
+      "loss": 1.272,
+      "theoretical_loss": 3.4811253996356912,
+      "tokens_seen": 1851719680
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044326753330123576,
+      "loss": 1.3506,
+      "theoretical_loss": 3.481104713628243,
+      "tokens_seen": 1851850752
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044322741133044456,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4810840294948004,
+      "tokens_seen": 1851981824
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044318728935965337,
+      "loss": 1.2254,
+      "theoretical_loss": 3.481063347235061,
+      "tokens_seen": 1852112896
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004431471673888621,
+      "loss": 1.3335,
+      "theoretical_loss": 3.4810426668487233,
+      "tokens_seen": 1852243968
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004431070454180709,
+      "loss": 1.2498,
+      "theoretical_loss": 3.481021988335484,
+      "tokens_seen": 1852375040
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044306692344727973,
+      "loss": 1.2194,
+      "theoretical_loss": 3.4810013116950413,
+      "tokens_seen": 1852506112
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044302680147648854,
+      "loss": 1.1858,
+      "theoretical_loss": 3.480980636927093,
+      "tokens_seen": 1852637184
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044298667950569735,
+      "loss": 1.3486,
+      "theoretical_loss": 3.4809599640313373,
+      "tokens_seen": 1852768256
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044294655753490615,
+      "loss": 1.329,
+      "theoretical_loss": 3.4809392930074723,
+      "tokens_seen": 1852899328
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004429064355641149,
+      "loss": 1.2693,
+      "theoretical_loss": 3.480918623855196,
+      "tokens_seen": 1853030400
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 624097,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2253131866455078,
+      "objective/train/theoretical_loss": 3.480897956574206,
+      "objective/train/tokens_used": 223620576,
+      "theoretical_loss": 3.480897956574206,
+      "tokens_seen": 1853161472
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004428663135933237,
+      "loss": 1.231,
+      "theoretical_loss": 3.480897956574206,
+      "tokens_seen": 1853161472
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004428261916225325,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4808772911642016,
+      "tokens_seen": 1853292544
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044278606965174127,
+      "loss": 1.3195,
+      "theoretical_loss": 3.4808566276248802,
+      "tokens_seen": 1853423616
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044274594768095013,
+      "loss": 1.2701,
+      "theoretical_loss": 3.480835965955941,
+      "tokens_seen": 1853554688
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004427058257101589,
+      "loss": 1.2504,
+      "theoretical_loss": 3.4808153061570817,
+      "tokens_seen": 1853685760
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004426657037393677,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4807946482280014,
+      "tokens_seen": 1853816832
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004426255817685765,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4807739921683982,
+      "tokens_seen": 1853947904
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044258545979778525,
+      "loss": 1.2573,
+      "theoretical_loss": 3.480753337977971,
+      "tokens_seen": 1854078976
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044254533782699405,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4807326856564185,
+      "tokens_seen": 1854210048
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044250521585620286,
+      "loss": 1.2954,
+      "theoretical_loss": 3.4807120352034397,
+      "tokens_seen": 1854341120
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044246509388541167,
+      "loss": 1.2419,
+      "theoretical_loss": 3.4806913866187332,
+      "tokens_seen": 1854472192
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004424249719146204,
+      "loss": 1.3087,
+      "theoretical_loss": 3.4806707399019983,
+      "tokens_seen": 1854603264
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004423848499438293,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4806500950529333,
+      "tokens_seen": 1854734336
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044234472797303803,
+      "loss": 1.2373,
+      "theoretical_loss": 3.480629452071238,
+      "tokens_seen": 1854865408
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044230460600224684,
+      "loss": 1.3015,
+      "theoretical_loss": 3.4806088109566113,
+      "tokens_seen": 1854996480
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044226448403145564,
+      "loss": 1.2687,
+      "theoretical_loss": 3.480588171708753,
+      "tokens_seen": 1855127552
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004422243620606644,
+      "loss": 1.255,
+      "theoretical_loss": 3.480567534327361,
+      "tokens_seen": 1855258624
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044218424008987326,
+      "loss": 1.2299,
+      "theoretical_loss": 3.480546898812136,
+      "tokens_seen": 1855389696
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000442144118119082,
+      "loss": 1.2869,
+      "theoretical_loss": 3.4805262651627773,
+      "tokens_seen": 1855520768
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004421039961482908,
+      "loss": 1.2973,
+      "theoretical_loss": 3.4805056333789834,
+      "tokens_seen": 1855651840
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004420638741774996,
+      "loss": 1.1871,
+      "theoretical_loss": 3.480485003460455,
+      "tokens_seen": 1855782912
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004420237522067084,
+      "loss": 1.2952,
+      "theoretical_loss": 3.480464375406891,
+      "tokens_seen": 1855913984
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004419836302359172,
+      "loss": 1.2848,
+      "theoretical_loss": 3.480443749217991,
+      "tokens_seen": 1856045056
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000441943508265126,
+      "loss": 1.2294,
+      "theoretical_loss": 3.4804231248934556,
+      "tokens_seen": 1856176128
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004419033862943348,
+      "loss": 1.1674,
+      "theoretical_loss": 3.480402502432984,
+      "tokens_seen": 1856307200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044186326432354354,
+      "loss": 1.2644,
+      "theoretical_loss": 3.4803818818362764,
+      "tokens_seen": 1856438272
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004418231423527524,
+      "loss": 1.1655,
+      "theoretical_loss": 3.480361263103033,
+      "tokens_seen": 1856569344
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044178302038196116,
+      "loss": 1.319,
+      "theoretical_loss": 3.4803406462329534,
+      "tokens_seen": 1856700416
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044174289841117,
+      "loss": 1.2702,
+      "theoretical_loss": 3.4803200312257374,
+      "tokens_seen": 1856831488
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044170277644037877,
+      "loss": 1.2696,
+      "theoretical_loss": 3.4802994180810867,
+      "tokens_seen": 1856962560
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004416626544695875,
+      "loss": 1.3457,
+      "theoretical_loss": 3.4802788067987,
+      "tokens_seen": 1857093632
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004416225324987964,
+      "loss": 1.2348,
+      "theoretical_loss": 3.4802581973782782,
+      "tokens_seen": 1857224704
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044158241052800513,
+      "loss": 1.2854,
+      "theoretical_loss": 3.4802375898195215,
+      "tokens_seen": 1857355776
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044154228855721394,
+      "loss": 1.2706,
+      "theoretical_loss": 3.480216984122131,
+      "tokens_seen": 1857486848
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044150216658642275,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4801963802858067,
+      "tokens_seen": 1857617920
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044146204461563155,
+      "loss": 1.2984,
+      "theoretical_loss": 3.480175778310249,
+      "tokens_seen": 1857748992
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004414219226448403,
+      "loss": 1.246,
+      "theoretical_loss": 3.48015517819516,
+      "tokens_seen": 1857880064
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004413818006740491,
+      "loss": 1.2661,
+      "theoretical_loss": 3.480134579940238,
+      "tokens_seen": 1858011136
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004413416787032579,
+      "loss": 1.2907,
+      "theoretical_loss": 3.480113983545186,
+      "tokens_seen": 1858142208
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044130155673246667,
+      "loss": 1.2199,
+      "theoretical_loss": 3.480093389009704,
+      "tokens_seen": 1858273280
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044126143476167553,
+      "loss": 1.2761,
+      "theoretical_loss": 3.480072796333493,
+      "tokens_seen": 1858404352
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004412213127908843,
+      "loss": 1.322,
+      "theoretical_loss": 3.4800522055162544,
+      "tokens_seen": 1858535424
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004411811908200931,
+      "loss": 1.3355,
+      "theoretical_loss": 3.480031616557689,
+      "tokens_seen": 1858666496
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004411410688493019,
+      "loss": 1.2733,
+      "theoretical_loss": 3.480011029457497,
+      "tokens_seen": 1858797568
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044110094687851065,
+      "loss": 1.2578,
+      "theoretical_loss": 3.4799904442153817,
+      "tokens_seen": 1858928640
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004410608249077195,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4799698608310425,
+      "tokens_seen": 1859059712
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044102070293692826,
+      "loss": 1.2713,
+      "theoretical_loss": 3.479949279304182,
+      "tokens_seen": 1859190784
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044098058096613706,
+      "loss": 1.2708,
+      "theoretical_loss": 3.4799286996345016,
+      "tokens_seen": 1859321856
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044094045899534587,
+      "loss": 1.308,
+      "theoretical_loss": 3.479908121821702,
+      "tokens_seen": 1859452928
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004409003370245547,
+      "loss": 1.2499,
+      "theoretical_loss": 3.479887545865485,
+      "tokens_seen": 1859584000
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 626401,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.219209909439087,
+      "objective/train/theoretical_loss": 3.479866971765553,
+      "objective/train/tokens_used": 230174176,
+      "theoretical_loss": 3.479866971765553,
+      "tokens_seen": 1859715072
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044086021505376343,
+      "loss": 1.2475,
+      "theoretical_loss": 3.479866971765553,
+      "tokens_seen": 1859715072
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044082009308297224,
+      "loss": 1.2788,
+      "theoretical_loss": 3.4798463995216067,
+      "tokens_seen": 1859846144
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044077997111218104,
+      "loss": 1.3323,
+      "theoretical_loss": 3.4798258291333486,
+      "tokens_seen": 1859977216
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004407398491413898,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4798052606004806,
+      "tokens_seen": 1860108288
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044069972717059865,
+      "loss": 1.2611,
+      "theoretical_loss": 3.4797846939227037,
+      "tokens_seen": 1860239360
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004406596051998074,
+      "loss": 1.2963,
+      "theoretical_loss": 3.479764129099721,
+      "tokens_seen": 1860370432
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004406194832290162,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4797435661312344,
+      "tokens_seen": 1860501504
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000440579361258225,
+      "loss": 1.3337,
+      "theoretical_loss": 3.479723005016945,
+      "tokens_seen": 1860632576
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004405392392874338,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4797024457565566,
+      "tokens_seen": 1860763648
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044049911731664263,
+      "loss": 1.3954,
+      "theoretical_loss": 3.47968188834977,
+      "tokens_seen": 1860894720
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004404589953458514,
+      "loss": 1.2251,
+      "theoretical_loss": 3.4796613327962884,
+      "tokens_seen": 1861025792
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004404188733750602,
+      "loss": 1.2743,
+      "theoretical_loss": 3.4796407790958144,
+      "tokens_seen": 1861156864
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000440378751404269,
+      "loss": 1.2547,
+      "theoretical_loss": 3.4796202272480494,
+      "tokens_seen": 1861287936
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004403386294334778,
+      "loss": 1.3109,
+      "theoretical_loss": 3.479599677252697,
+      "tokens_seen": 1861419008
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044029850746268656,
+      "loss": 1.2471,
+      "theoretical_loss": 3.4795791291094593,
+      "tokens_seen": 1861550080
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004402583854918954,
+      "loss": 1.286,
+      "theoretical_loss": 3.479558582818039,
+      "tokens_seen": 1861681152
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044021826352110417,
+      "loss": 1.3821,
+      "theoretical_loss": 3.4795380383781396,
+      "tokens_seen": 1861812224
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004401781415503129,
+      "loss": 1.2488,
+      "theoretical_loss": 3.4795174957894623,
+      "tokens_seen": 1861943296
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004401380195795218,
+      "loss": 1.2991,
+      "theoretical_loss": 3.4794969550517116,
+      "tokens_seen": 1862074368
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044009789760873053,
+      "loss": 1.3208,
+      "theoretical_loss": 3.47947641616459,
+      "tokens_seen": 1862205440
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044005777563793934,
+      "loss": 1.2989,
+      "theoretical_loss": 3.4794558791277996,
+      "tokens_seen": 1862336512
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044001765366714815,
+      "loss": 1.2533,
+      "theoretical_loss": 3.479435343941045,
+      "tokens_seen": 1862467584
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043997753169635695,
+      "loss": 1.2551,
+      "theoretical_loss": 3.4794148106040286,
+      "tokens_seen": 1862598656
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004399374097255657,
+      "loss": 1.2266,
+      "theoretical_loss": 3.479394279116453,
+      "tokens_seen": 1862729728
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004398972877547745,
+      "loss": 1.3293,
+      "theoretical_loss": 3.4793737494780226,
+      "tokens_seen": 1862860800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004398571657839833,
+      "loss": 1.3636,
+      "theoretical_loss": 3.4793532216884406,
+      "tokens_seen": 1862991872
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004398170438131921,
+      "loss": 1.2496,
+      "theoretical_loss": 3.4793326957474093,
+      "tokens_seen": 1863122944
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043977692184240093,
+      "loss": 1.2856,
+      "theoretical_loss": 3.479312171654634,
+      "tokens_seen": 1863254016
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004397367998716097,
+      "loss": 1.2869,
+      "theoretical_loss": 3.4792916494098165,
+      "tokens_seen": 1863385088
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043969667790081854,
+      "loss": 1.2218,
+      "theoretical_loss": 3.479271129012662,
+      "tokens_seen": 1863516160
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004396565559300273,
+      "loss": 1.329,
+      "theoretical_loss": 3.4792506104628735,
+      "tokens_seen": 1863647232
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043961643395923605,
+      "loss": 1.3324,
+      "theoretical_loss": 3.4792300937601546,
+      "tokens_seen": 1863778304
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004395763119884449,
+      "loss": 1.2582,
+      "theoretical_loss": 3.479209578904209,
+      "tokens_seen": 1863909376
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043953619001765366,
+      "loss": 1.244,
+      "theoretical_loss": 3.479189065894741,
+      "tokens_seen": 1864040448
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043949606804686246,
+      "loss": 1.3033,
+      "theoretical_loss": 3.479168554731455,
+      "tokens_seen": 1864171520
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043945594607607127,
+      "loss": 1.2249,
+      "theoretical_loss": 3.479148045414054,
+      "tokens_seen": 1864302592
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004394158241052801,
+      "loss": 1.2121,
+      "theoretical_loss": 3.479127537942243,
+      "tokens_seen": 1864433664
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043937570213448883,
+      "loss": 1.2762,
+      "theoretical_loss": 3.479107032315726,
+      "tokens_seen": 1864564736
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043933558016369764,
+      "loss": 1.3439,
+      "theoretical_loss": 3.4790865285342067,
+      "tokens_seen": 1864695808
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043929545819290644,
+      "loss": 1.2952,
+      "theoretical_loss": 3.4790660265973905,
+      "tokens_seen": 1864826880
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004392553362221152,
+      "loss": 1.2369,
+      "theoretical_loss": 3.479045526504981,
+      "tokens_seen": 1864957952
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043921521425132405,
+      "loss": 1.27,
+      "theoretical_loss": 3.4790250282566824,
+      "tokens_seen": 1865089024
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004391750922805328,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4790045318522,
+      "tokens_seen": 1865220096
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043913497030974167,
+      "loss": 1.2985,
+      "theoretical_loss": 3.478984037291238,
+      "tokens_seen": 1865351168
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004390948483389504,
+      "loss": 1.2916,
+      "theoretical_loss": 3.478963544573501,
+      "tokens_seen": 1865482240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004390547263681592,
+      "loss": 1.3218,
+      "theoretical_loss": 3.478943053698694,
+      "tokens_seen": 1865613312
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043901460439736803,
+      "loss": 1.3876,
+      "theoretical_loss": 3.478922564666522,
+      "tokens_seen": 1865744384
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004389744824265768,
+      "loss": 1.3278,
+      "theoretical_loss": 3.478902077476689,
+      "tokens_seen": 1865875456
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004389343604557856,
+      "loss": 1.2874,
+      "theoretical_loss": 3.478881592128901,
+      "tokens_seen": 1866006528
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004388942384849944,
+      "loss": 1.315,
+      "theoretical_loss": 3.478861108622862,
+      "tokens_seen": 1866137600
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 628509,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.313321590423584,
+      "objective/train/theoretical_loss": 3.4788406269582772,
+      "objective/train/tokens_used": 236727776,
+      "theoretical_loss": 3.4788406269582772,
+      "tokens_seen": 1866268672
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004388541165142032,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4788406269582772,
+      "tokens_seen": 1866268672
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043881399454341195,
+      "loss": 1.2632,
+      "theoretical_loss": 3.4788201471348525,
+      "tokens_seen": 1866399744
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004387738725726208,
+      "loss": 1.3232,
+      "theoretical_loss": 3.478799669152293,
+      "tokens_seen": 1866530816
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043873375060182957,
+      "loss": 1.2823,
+      "theoretical_loss": 3.4787791930103036,
+      "tokens_seen": 1866661888
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004386936286310383,
+      "loss": 1.2846,
+      "theoretical_loss": 3.4787587187085895,
+      "tokens_seen": 1866792960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004386535066602472,
+      "loss": 1.294,
+      "theoretical_loss": 3.4787382462468566,
+      "tokens_seen": 1866924032
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043861338468945593,
+      "loss": 1.273,
+      "theoretical_loss": 3.47871777562481,
+      "tokens_seen": 1867055104
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004385732627186648,
+      "loss": 1.263,
+      "theoretical_loss": 3.4786973068421556,
+      "tokens_seen": 1867186176
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043853314074787354,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4786768398985983,
+      "tokens_seen": 1867317248
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043849301877708235,
+      "loss": 1.3002,
+      "theoretical_loss": 3.478656374793845,
+      "tokens_seen": 1867448320
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043845289680629116,
+      "loss": 1.2963,
+      "theoretical_loss": 3.4786359115275998,
+      "tokens_seen": 1867579392
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004384127748354999,
+      "loss": 1.2337,
+      "theoretical_loss": 3.4786154500995696,
+      "tokens_seen": 1867710464
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004383726528647087,
+      "loss": 1.3299,
+      "theoretical_loss": 3.478594990509461,
+      "tokens_seen": 1867841536
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004383325308939175,
+      "loss": 1.2289,
+      "theoretical_loss": 3.478574532756978,
+      "tokens_seen": 1867972608
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043829240892312633,
+      "loss": 1.3448,
+      "theoretical_loss": 3.478554076841829,
+      "tokens_seen": 1868103680
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004382522869523351,
+      "loss": 1.2988,
+      "theoretical_loss": 3.478533622763718,
+      "tokens_seen": 1868234752
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043821216498154394,
+      "loss": 1.2657,
+      "theoretical_loss": 3.478513170522352,
+      "tokens_seen": 1868365824
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004381720430107527,
+      "loss": 1.3458,
+      "theoretical_loss": 3.478492720117437,
+      "tokens_seen": 1868496896
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043813192103996145,
+      "loss": 1.3726,
+      "theoretical_loss": 3.47847227154868,
+      "tokens_seen": 1868627968
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004380917990691703,
+      "loss": 1.2341,
+      "theoretical_loss": 3.478451824815787,
+      "tokens_seen": 1868759040
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043805167709837906,
+      "loss": 1.259,
+      "theoretical_loss": 3.4784313799184634,
+      "tokens_seen": 1868890112
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043801155512758786,
+      "loss": 1.244,
+      "theoretical_loss": 3.4784109368564176,
+      "tokens_seen": 1869021184
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043797143315679667,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4783904956293545,
+      "tokens_seen": 1869152256
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004379313111860055,
+      "loss": 1.3516,
+      "theoretical_loss": 3.4783700562369813,
+      "tokens_seen": 1869283328
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004378911892152143,
+      "loss": 1.2568,
+      "theoretical_loss": 3.478349618679005,
+      "tokens_seen": 1869414400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043785106724442304,
+      "loss": 1.1763,
+      "theoretical_loss": 3.4783291829551324,
+      "tokens_seen": 1869545472
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043781094527363184,
+      "loss": 1.2471,
+      "theoretical_loss": 3.4783087490650697,
+      "tokens_seen": 1869676544
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043777082330284065,
+      "loss": 1.2738,
+      "theoretical_loss": 3.4782883170085244,
+      "tokens_seen": 1869807616
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043773070133204945,
+      "loss": 1.3047,
+      "theoretical_loss": 3.4782678867852033,
+      "tokens_seen": 1869938688
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004376905793612582,
+      "loss": 1.3373,
+      "theoretical_loss": 3.4782474583948133,
+      "tokens_seen": 1870069760
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043765045739046707,
+      "loss": 1.2504,
+      "theoretical_loss": 3.4782270318370614,
+      "tokens_seen": 1870200832
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004376103354196758,
+      "loss": 1.3512,
+      "theoretical_loss": 3.4782066071116553,
+      "tokens_seen": 1870331904
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043757021344888457,
+      "loss": 1.3033,
+      "theoretical_loss": 3.4781861842183015,
+      "tokens_seen": 1870462976
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043753009147809343,
+      "loss": 1.3173,
+      "theoretical_loss": 3.4781657631567082,
+      "tokens_seen": 1870594048
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004374899695073022,
+      "loss": 1.2744,
+      "theoretical_loss": 3.478145343926582,
+      "tokens_seen": 1870725120
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000437449847536511,
+      "loss": 1.2961,
+      "theoretical_loss": 3.4781249265276304,
+      "tokens_seen": 1870856192
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004374097255657198,
+      "loss": 1.2716,
+      "theoretical_loss": 3.478104510959561,
+      "tokens_seen": 1870987264
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004373696035949286,
+      "loss": 1.2436,
+      "theoretical_loss": 3.4780840972220823,
+      "tokens_seen": 1871118336
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004373294816241374,
+      "loss": 1.3228,
+      "theoretical_loss": 3.4780636853149005,
+      "tokens_seen": 1871249408
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004372893596533462,
+      "loss": 1.327,
+      "theoretical_loss": 3.478043275237724,
+      "tokens_seen": 1871380480
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043724923768255497,
+      "loss": 1.2927,
+      "theoretical_loss": 3.478022866990261,
+      "tokens_seen": 1871511552
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004372091157117638,
+      "loss": 1.3243,
+      "theoretical_loss": 3.4780024605722186,
+      "tokens_seen": 1871642624
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004371689937409726,
+      "loss": 1.2579,
+      "theoretical_loss": 3.477982055983305,
+      "tokens_seen": 1871773696
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043712887177018133,
+      "loss": 1.2896,
+      "theoretical_loss": 3.477961653223228,
+      "tokens_seen": 1871904768
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004370887497993902,
+      "loss": 1.2927,
+      "theoretical_loss": 3.477941252291696,
+      "tokens_seen": 1872035840
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043704862782859894,
+      "loss": 1.3064,
+      "theoretical_loss": 3.477920853188417,
+      "tokens_seen": 1872166912
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043700850585780775,
+      "loss": 1.2831,
+      "theoretical_loss": 3.4779004559130984,
+      "tokens_seen": 1872297984
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043696838388701656,
+      "loss": 1.3036,
+      "theoretical_loss": 3.47788006046545,
+      "tokens_seen": 1872429056
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004369282619162253,
+      "loss": 1.2966,
+      "theoretical_loss": 3.4778596668451787,
+      "tokens_seen": 1872560128
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004368881399454341,
+      "loss": 1.2898,
+      "theoretical_loss": 3.477839275051994,
+      "tokens_seen": 1872691200
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 630672,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.272313117980957,
+      "objective/train/theoretical_loss": 3.477818885085603,
+      "objective/train/tokens_used": 243281376,
+      "theoretical_loss": 3.477818885085603,
+      "tokens_seen": 1872822272
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004368480179746429,
+      "loss": 1.2842,
+      "theoretical_loss": 3.477818885085603,
+      "tokens_seen": 1872822272
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043680789600385173,
+      "loss": 1.3044,
+      "theoretical_loss": 3.4777984969457156,
+      "tokens_seen": 1872953344
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004367677740330605,
+      "loss": 1.3094,
+      "theoretical_loss": 3.47777811063204,
+      "tokens_seen": 1873084416
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043672765206226934,
+      "loss": 1.2328,
+      "theoretical_loss": 3.477757726144284,
+      "tokens_seen": 1873215488
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004366875300914781,
+      "loss": 1.3658,
+      "theoretical_loss": 3.477737343482157,
+      "tokens_seen": 1873346560
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004366474081206869,
+      "loss": 1.2608,
+      "theoretical_loss": 3.477716962645368,
+      "tokens_seen": 1873477632
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004366072861498957,
+      "loss": 1.2405,
+      "theoretical_loss": 3.4776965836336253,
+      "tokens_seen": 1873608704
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043656716417910446,
+      "loss": 1.3631,
+      "theoretical_loss": 3.477676206446638,
+      "tokens_seen": 1873739776
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004365270422083133,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4776558310841157,
+      "tokens_seen": 1873870848
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043648692023752207,
+      "loss": 1.2483,
+      "theoretical_loss": 3.477635457545767,
+      "tokens_seen": 1874001920
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004364467982667309,
+      "loss": 1.2944,
+      "theoretical_loss": 3.4776150858313004,
+      "tokens_seen": 1874132992
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004364066762959397,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4775947159404255,
+      "tokens_seen": 1874264064
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043636655432514843,
+      "loss": 1.2381,
+      "theoretical_loss": 3.477574347872852,
+      "tokens_seen": 1874395136
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043632643235435724,
+      "loss": 1.3518,
+      "theoretical_loss": 3.4775539816282888,
+      "tokens_seen": 1874526208
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043628631038356605,
+      "loss": 1.289,
+      "theoretical_loss": 3.4775336172064453,
+      "tokens_seen": 1874657280
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043624618841277485,
+      "loss": 1.3378,
+      "theoretical_loss": 3.4775132546070306,
+      "tokens_seen": 1874788352
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004362060664419836,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4774928938297553,
+      "tokens_seen": 1874919424
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043616594447119247,
+      "loss": 1.219,
+      "theoretical_loss": 3.477472534874328,
+      "tokens_seen": 1875050496
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004361258225004012,
+      "loss": 1.2285,
+      "theoretical_loss": 3.477452177740459,
+      "tokens_seen": 1875181568
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043608570052960997,
+      "loss": 1.3877,
+      "theoretical_loss": 3.477431822427857,
+      "tokens_seen": 1875312640
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043604557855881883,
+      "loss": 1.2934,
+      "theoretical_loss": 3.477411468936233,
+      "tokens_seen": 1875443712
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004360054565880276,
+      "loss": 1.3059,
+      "theoretical_loss": 3.477391117265296,
+      "tokens_seen": 1875574784
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043596533461723644,
+      "loss": 1.2628,
+      "theoretical_loss": 3.4773707674147563,
+      "tokens_seen": 1875705856
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004359252126464452,
+      "loss": 1.3166,
+      "theoretical_loss": 3.477350419384323,
+      "tokens_seen": 1875836928
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000435885090675654,
+      "loss": 1.393,
+      "theoretical_loss": 3.4773300731737082,
+      "tokens_seen": 1875968000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004358449687048628,
+      "loss": 1.2713,
+      "theoretical_loss": 3.47730972878262,
+      "tokens_seen": 1876099072
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004358048467340716,
+      "loss": 1.2647,
+      "theoretical_loss": 3.47728938621077,
+      "tokens_seen": 1876230144
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043576472476328037,
+      "loss": 1.2843,
+      "theoretical_loss": 3.477269045457867,
+      "tokens_seen": 1876361216
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004357246027924892,
+      "loss": 1.2809,
+      "theoretical_loss": 3.477248706523622,
+      "tokens_seen": 1876492288
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000435684480821698,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4772283694077464,
+      "tokens_seen": 1876623360
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043564435885090673,
+      "loss": 1.3056,
+      "theoretical_loss": 3.4772080341099487,
+      "tokens_seen": 1876754432
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004356042368801156,
+      "loss": 1.3105,
+      "theoretical_loss": 3.4771877006299414,
+      "tokens_seen": 1876885504
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043556411490932434,
+      "loss": 1.2858,
+      "theoretical_loss": 3.477167368967433,
+      "tokens_seen": 1877016576
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043552399293853315,
+      "loss": 1.2549,
+      "theoretical_loss": 3.477147039122136,
+      "tokens_seen": 1877147648
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043548387096774196,
+      "loss": 1.2881,
+      "theoretical_loss": 3.47712671109376,
+      "tokens_seen": 1877278720
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004354437489969507,
+      "loss": 1.3231,
+      "theoretical_loss": 3.4771063848820165,
+      "tokens_seen": 1877409792
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043540362702615957,
+      "loss": 1.2883,
+      "theoretical_loss": 3.4770860604866156,
+      "tokens_seen": 1877540864
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004353635050553683,
+      "loss": 1.2822,
+      "theoretical_loss": 3.477065737907269,
+      "tokens_seen": 1877671936
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043532338308457713,
+      "loss": 1.221,
+      "theoretical_loss": 3.477045417143687,
+      "tokens_seen": 1877803008
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043528326111378593,
+      "loss": 1.2581,
+      "theoretical_loss": 3.477025098195581,
+      "tokens_seen": 1877934080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043524313914299474,
+      "loss": 1.3469,
+      "theoretical_loss": 3.4770047810626616,
+      "tokens_seen": 1878065152
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004352030171722035,
+      "loss": 1.1799,
+      "theoretical_loss": 3.476984465744641,
+      "tokens_seen": 1878196224
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004351628952014123,
+      "loss": 1.3193,
+      "theoretical_loss": 3.476964152241229,
+      "tokens_seen": 1878327296
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004351227732306211,
+      "loss": 1.2921,
+      "theoretical_loss": 3.476943840552139,
+      "tokens_seen": 1878458368
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043508265125982986,
+      "loss": 1.2766,
+      "theoretical_loss": 3.4769235306770803,
+      "tokens_seen": 1878589440
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004350425292890387,
+      "loss": 1.2823,
+      "theoretical_loss": 3.476903222615765,
+      "tokens_seen": 1878720512
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043500240731824747,
+      "loss": 1.286,
+      "theoretical_loss": 3.476882916367905,
+      "tokens_seen": 1878851584
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004349622853474563,
+      "loss": 1.3033,
+      "theoretical_loss": 3.4768626119332113,
+      "tokens_seen": 1878982656
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004349221633766651,
+      "loss": 1.2764,
+      "theoretical_loss": 3.4768423093113965,
+      "tokens_seen": 1879113728
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043488204140587383,
+      "loss": 1.2135,
+      "theoretical_loss": 3.476822008502171,
+      "tokens_seen": 1879244800
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 632825,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2928540706634521,
+      "objective/train/theoretical_loss": 3.4768017095052475,
+      "objective/train/tokens_used": 249834976,
+      "theoretical_loss": 3.4768017095052475,
+      "tokens_seen": 1879375872
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043484191943508264,
+      "loss": 1.3324,
+      "theoretical_loss": 3.4768017095052475,
+      "tokens_seen": 1879375872
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043480179746429145,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4767814123203378,
+      "tokens_seen": 1879506944
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043476167549350025,
+      "loss": 1.3118,
+      "theoretical_loss": 3.476761116947153,
+      "tokens_seen": 1879638016
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043472155352270906,
+      "loss": 1.223,
+      "theoretical_loss": 3.4767408233854065,
+      "tokens_seen": 1879769088
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043468143155191787,
+      "loss": 1.2872,
+      "theoretical_loss": 3.4767205316348084,
+      "tokens_seen": 1879900160
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004346413095811266,
+      "loss": 1.2966,
+      "theoretical_loss": 3.476700241695072,
+      "tokens_seen": 1880031232
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004346011876103354,
+      "loss": 1.254,
+      "theoretical_loss": 3.47667995356591,
+      "tokens_seen": 1880162304
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043456106563954423,
+      "loss": 1.2364,
+      "theoretical_loss": 3.476659667247034,
+      "tokens_seen": 1880293376
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000434520943668753,
+      "loss": 1.2624,
+      "theoretical_loss": 3.4766393827381554,
+      "tokens_seen": 1880424448
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043448082169796184,
+      "loss": 1.2465,
+      "theoretical_loss": 3.476619100038988,
+      "tokens_seen": 1880555520
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004344406997271706,
+      "loss": 1.3092,
+      "theoretical_loss": 3.4765988191492436,
+      "tokens_seen": 1880686592
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004344005777563794,
+      "loss": 1.35,
+      "theoretical_loss": 3.476578540068634,
+      "tokens_seen": 1880817664
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004343604557855882,
+      "loss": 1.2878,
+      "theoretical_loss": 3.476558262796874,
+      "tokens_seen": 1880948736
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000434320333814797,
+      "loss": 1.3079,
+      "theoretical_loss": 3.476537987333673,
+      "tokens_seen": 1881079808
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043428021184400577,
+      "loss": 1.2281,
+      "theoretical_loss": 3.4765177136787466,
+      "tokens_seen": 1881210880
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043424008987321457,
+      "loss": 1.3172,
+      "theoretical_loss": 3.476497441831806,
+      "tokens_seen": 1881341952
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004341999679024234,
+      "loss": 1.3597,
+      "theoretical_loss": 3.476477171792564,
+      "tokens_seen": 1881473024
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043415984593163213,
+      "loss": 1.3413,
+      "theoretical_loss": 3.4764569035607344,
+      "tokens_seen": 1881604096
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000434119723960841,
+      "loss": 1.3882,
+      "theoretical_loss": 3.4764366371360294,
+      "tokens_seen": 1881735168
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043407960199004974,
+      "loss": 1.2857,
+      "theoretical_loss": 3.476416372518162,
+      "tokens_seen": 1881866240
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004340394800192586,
+      "loss": 1.2881,
+      "theoretical_loss": 3.4763961097068456,
+      "tokens_seen": 1881997312
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043399935804846736,
+      "loss": 1.2323,
+      "theoretical_loss": 3.4763758487017937,
+      "tokens_seen": 1882128384
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004339592360776761,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4763555895027185,
+      "tokens_seen": 1882259456
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043391911410688497,
+      "loss": 1.3043,
+      "theoretical_loss": 3.4763353321093344,
+      "tokens_seen": 1882390528
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004338789921360937,
+      "loss": 1.2921,
+      "theoretical_loss": 3.4763150765213537,
+      "tokens_seen": 1882521600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043383887016530253,
+      "loss": 1.3496,
+      "theoretical_loss": 3.4762948227384904,
+      "tokens_seen": 1882652672
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043379874819451133,
+      "loss": 1.337,
+      "theoretical_loss": 3.476274570760458,
+      "tokens_seen": 1882783744
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043375862622372014,
+      "loss": 1.2241,
+      "theoretical_loss": 3.4762543205869703,
+      "tokens_seen": 1882914816
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004337185042529289,
+      "loss": 1.3125,
+      "theoretical_loss": 3.47623407221774,
+      "tokens_seen": 1883045888
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004336783822821377,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4762138256524815,
+      "tokens_seen": 1883176960
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004336382603113465,
+      "loss": 1.2425,
+      "theoretical_loss": 3.476193580890908,
+      "tokens_seen": 1883308032
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043359813834055526,
+      "loss": 1.3231,
+      "theoretical_loss": 3.476173337932734,
+      "tokens_seen": 1883439104
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004335580163697641,
+      "loss": 1.2948,
+      "theoretical_loss": 3.476153096777673,
+      "tokens_seen": 1883570176
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043351789439897287,
+      "loss": 1.3158,
+      "theoretical_loss": 3.476132857425439,
+      "tokens_seen": 1883701248
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043347777242818173,
+      "loss": 1.3569,
+      "theoretical_loss": 3.476112619875746,
+      "tokens_seen": 1883832320
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004334376504573905,
+      "loss": 1.3122,
+      "theoretical_loss": 3.476092384128308,
+      "tokens_seen": 1883963392
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043339752848659923,
+      "loss": 1.2961,
+      "theoretical_loss": 3.476072150182839,
+      "tokens_seen": 1884094464
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004333574065158081,
+      "loss": 1.3205,
+      "theoretical_loss": 3.4760519180390532,
+      "tokens_seen": 1884225536
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043331728454501685,
+      "loss": 1.2642,
+      "theoretical_loss": 3.476031687696665,
+      "tokens_seen": 1884356608
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043327716257422565,
+      "loss": 1.3241,
+      "theoretical_loss": 3.476011459155389,
+      "tokens_seen": 1884487680
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043323704060343446,
+      "loss": 1.3122,
+      "theoretical_loss": 3.4759912324149393,
+      "tokens_seen": 1884618752
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043319691863264327,
+      "loss": 1.3436,
+      "theoretical_loss": 3.47597100747503,
+      "tokens_seen": 1884749824
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000433156796661852,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4759507843353763,
+      "tokens_seen": 1884880896
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004331166746910608,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4759305629956927,
+      "tokens_seen": 1885011968
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043307655272026963,
+      "loss": 1.2632,
+      "theoretical_loss": 3.475910343455693,
+      "tokens_seen": 1885143040
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004330364307494784,
+      "loss": 1.2874,
+      "theoretical_loss": 3.475890125715093,
+      "tokens_seen": 1885274112
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043299630877868724,
+      "loss": 1.21,
+      "theoretical_loss": 3.475869909773607,
+      "tokens_seen": 1885405184
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000432956186807896,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4758496956309495,
+      "tokens_seen": 1885536256
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004329160648371048,
+      "loss": 1.271,
+      "theoretical_loss": 3.4758294832868364,
+      "tokens_seen": 1885667328
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004328759428663136,
+      "loss": 1.3072,
+      "theoretical_loss": 3.4758092727409817,
+      "tokens_seen": 1885798400
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 635135,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2254669666290283,
+      "objective/train/theoretical_loss": 3.475789063993101,
+      "objective/train/tokens_used": 256388576,
+      "theoretical_loss": 3.475789063993101,
+      "tokens_seen": 1885929472
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043283582089552236,
+      "loss": 1.2618,
+      "theoretical_loss": 3.475789063993101,
+      "tokens_seen": 1885929472
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004327956989247312,
+      "loss": 1.3463,
+      "theoretical_loss": 3.475768857042909,
+      "tokens_seen": 1886060544
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043275557695393997,
+      "loss": 1.2854,
+      "theoretical_loss": 3.475748651890121,
+      "tokens_seen": 1886191616
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004327154549831488,
+      "loss": 1.2995,
+      "theoretical_loss": 3.4757284485344524,
+      "tokens_seen": 1886322688
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004326753330123576,
+      "loss": 1.3302,
+      "theoretical_loss": 3.475708246975618,
+      "tokens_seen": 1886453760
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004326352110415664,
+      "loss": 1.2521,
+      "theoretical_loss": 3.4756880472133345,
+      "tokens_seen": 1886584832
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043259508907077514,
+      "loss": 1.3034,
+      "theoretical_loss": 3.4756678492473156,
+      "tokens_seen": 1886715904
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000432554967099984,
+      "loss": 1.2808,
+      "theoretical_loss": 3.475647653077278,
+      "tokens_seen": 1886846976
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043251484512919276,
+      "loss": 1.3079,
+      "theoretical_loss": 3.4756274587029368,
+      "tokens_seen": 1886978048
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004324747231584015,
+      "loss": 1.3396,
+      "theoretical_loss": 3.4756072661240074,
+      "tokens_seen": 1887109120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043243460118761037,
+      "loss": 1.3317,
+      "theoretical_loss": 3.475587075340206,
+      "tokens_seen": 1887240192
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004323944792168191,
+      "loss": 1.2271,
+      "theoretical_loss": 3.475566886351248,
+      "tokens_seen": 1887371264
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043235435724602793,
+      "loss": 1.2725,
+      "theoretical_loss": 3.47554669915685,
+      "tokens_seen": 1887502336
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043231423527523673,
+      "loss": 1.2817,
+      "theoretical_loss": 3.4755265137567264,
+      "tokens_seen": 1887633408
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043227411330444554,
+      "loss": 1.2761,
+      "theoretical_loss": 3.4755063301505946,
+      "tokens_seen": 1887764480
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043223399133365435,
+      "loss": 1.3083,
+      "theoretical_loss": 3.47548614833817,
+      "tokens_seen": 1887895552
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004321938693628631,
+      "loss": 1.2488,
+      "theoretical_loss": 3.4754659683191678,
+      "tokens_seen": 1888026624
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004321537473920719,
+      "loss": 1.2987,
+      "theoretical_loss": 3.475445790093306,
+      "tokens_seen": 1888157696
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004321136254212807,
+      "loss": 1.2552,
+      "theoretical_loss": 3.475425613660299,
+      "tokens_seen": 1888288768
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004320735034504895,
+      "loss": 1.2467,
+      "theoretical_loss": 3.4754054390198648,
+      "tokens_seen": 1888419840
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043203338147969827,
+      "loss": 1.2965,
+      "theoretical_loss": 3.4753852661717186,
+      "tokens_seen": 1888550912
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043199325950890713,
+      "loss": 1.2296,
+      "theoretical_loss": 3.4753650951155763,
+      "tokens_seen": 1888681984
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004319531375381159,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4753449258511555,
+      "tokens_seen": 1888813056
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043191301556732463,
+      "loss": 1.2915,
+      "theoretical_loss": 3.475324758378173,
+      "tokens_seen": 1888944128
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004318728935965335,
+      "loss": 1.2283,
+      "theoretical_loss": 3.475304592696344,
+      "tokens_seen": 1889075200
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043183277162574225,
+      "loss": 1.2967,
+      "theoretical_loss": 3.475284428805386,
+      "tokens_seen": 1889206272
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043179264965495105,
+      "loss": 1.3399,
+      "theoretical_loss": 3.4752642667050155,
+      "tokens_seen": 1889337344
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043175252768415986,
+      "loss": 1.3589,
+      "theoretical_loss": 3.4752441063949497,
+      "tokens_seen": 1889468416
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043171240571336867,
+      "loss": 1.3771,
+      "theoretical_loss": 3.475223947874905,
+      "tokens_seen": 1889599488
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004316722837425774,
+      "loss": 1.286,
+      "theoretical_loss": 3.4752037911445983,
+      "tokens_seen": 1889730560
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004316321617717862,
+      "loss": 1.2757,
+      "theoretical_loss": 3.475183636203747,
+      "tokens_seen": 1889861632
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043159203980099503,
+      "loss": 1.2791,
+      "theoretical_loss": 3.475163483052068,
+      "tokens_seen": 1889992704
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043155191783020384,
+      "loss": 1.2725,
+      "theoretical_loss": 3.475143331689278,
+      "tokens_seen": 1890123776
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043151179585941264,
+      "loss": 1.2453,
+      "theoretical_loss": 3.4751231821150945,
+      "tokens_seen": 1890254848
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004314716738886214,
+      "loss": 1.327,
+      "theoretical_loss": 3.4751030343292344,
+      "tokens_seen": 1890385920
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043143155191783026,
+      "loss": 1.3241,
+      "theoretical_loss": 3.475082888331416,
+      "tokens_seen": 1890516992
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000431391429947039,
+      "loss": 1.3051,
+      "theoretical_loss": 3.4750627441213555,
+      "tokens_seen": 1890648064
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043135130797624776,
+      "loss": 1.3165,
+      "theoretical_loss": 3.475042601698771,
+      "tokens_seen": 1890779136
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004313111860054566,
+      "loss": 1.2951,
+      "theoretical_loss": 3.4750224610633795,
+      "tokens_seen": 1890910208
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043127106403466537,
+      "loss": 1.2297,
+      "theoretical_loss": 3.4750023222148996,
+      "tokens_seen": 1891041280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004312309420638742,
+      "loss": 1.3044,
+      "theoretical_loss": 3.4749821851530474,
+      "tokens_seen": 1891172352
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000431190820093083,
+      "loss": 1.3302,
+      "theoretical_loss": 3.474962049877542,
+      "tokens_seen": 1891303424
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004311506981222918,
+      "loss": 1.2719,
+      "theoretical_loss": 3.4749419163881003,
+      "tokens_seen": 1891434496
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043111057615150054,
+      "loss": 1.2493,
+      "theoretical_loss": 3.4749217846844402,
+      "tokens_seen": 1891565568
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004310704541807094,
+      "loss": 1.2973,
+      "theoretical_loss": 3.4749016547662803,
+      "tokens_seen": 1891696640
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043103033220991816,
+      "loss": 1.3075,
+      "theoretical_loss": 3.4748815266333377,
+      "tokens_seen": 1891827712
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004309902102391269,
+      "loss": 1.2674,
+      "theoretical_loss": 3.47486140028533,
+      "tokens_seen": 1891958784
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043095008826833577,
+      "loss": 1.2755,
+      "theoretical_loss": 3.474841275721977,
+      "tokens_seen": 1892089856
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004309099662975445,
+      "loss": 1.2901,
+      "theoretical_loss": 3.474821152942995,
+      "tokens_seen": 1892220928
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004308698443267534,
+      "loss": 1.3458,
+      "theoretical_loss": 3.474801031948103,
+      "tokens_seen": 1892352000
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 637400,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2908365726470947,
+      "objective/train/theoretical_loss": 3.47478091273702,
+      "objective/train/tokens_used": 262942176,
+      "theoretical_loss": 3.47478091273702,
+      "tokens_seen": 1892483072
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043082972235596213,
+      "loss": 1.3177,
+      "theoretical_loss": 3.47478091273702,
+      "tokens_seen": 1892483072
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043078960038517094,
+      "loss": 1.3174,
+      "theoretical_loss": 3.4747607953094635,
+      "tokens_seen": 1892614144
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043074947841437975,
+      "loss": 1.3044,
+      "theoretical_loss": 3.4747406796651514,
+      "tokens_seen": 1892745216
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004307093564435885,
+      "loss": 1.2713,
+      "theoretical_loss": 3.474720565803803,
+      "tokens_seen": 1892876288
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004306692344727973,
+      "loss": 1.2719,
+      "theoretical_loss": 3.4747004537251365,
+      "tokens_seen": 1893007360
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004306291125020061,
+      "loss": 1.2275,
+      "theoretical_loss": 3.4746803434288704,
+      "tokens_seen": 1893138432
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004305889905312149,
+      "loss": 1.3338,
+      "theoretical_loss": 3.474660234914724,
+      "tokens_seen": 1893269504
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043054886856042367,
+      "loss": 1.2109,
+      "theoretical_loss": 3.4746401281824153,
+      "tokens_seen": 1893400576
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043050874658963253,
+      "loss": 1.2639,
+      "theoretical_loss": 3.474620023231663,
+      "tokens_seen": 1893531648
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004304686246188413,
+      "loss": 1.3237,
+      "theoretical_loss": 3.474599920062187,
+      "tokens_seen": 1893662720
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043042850264805003,
+      "loss": 1.316,
+      "theoretical_loss": 3.4745798186737047,
+      "tokens_seen": 1893793792
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004303883806772589,
+      "loss": 1.2481,
+      "theoretical_loss": 3.474559719065936,
+      "tokens_seen": 1893924864
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043034825870646765,
+      "loss": 1.3465,
+      "theoretical_loss": 3.4745396212385997,
+      "tokens_seen": 1894055936
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004303081367356765,
+      "loss": 1.3158,
+      "theoretical_loss": 3.4745195251914147,
+      "tokens_seen": 1894187008
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043026801476488526,
+      "loss": 1.2292,
+      "theoretical_loss": 3.4744994309241006,
+      "tokens_seen": 1894318080
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043022789279409407,
+      "loss": 1.2333,
+      "theoretical_loss": 3.474479338436377,
+      "tokens_seen": 1894449152
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043018777082330287,
+      "loss": 1.2647,
+      "theoretical_loss": 3.474459247727962,
+      "tokens_seen": 1894580224
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004301476488525116,
+      "loss": 1.3232,
+      "theoretical_loss": 3.4744391587985755,
+      "tokens_seen": 1894711296
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043010752688172043,
+      "loss": 1.3079,
+      "theoretical_loss": 3.474419071647937,
+      "tokens_seen": 1894842368
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043006740491092924,
+      "loss": 1.2455,
+      "theoretical_loss": 3.474398986275766,
+      "tokens_seen": 1894973440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043002728294013804,
+      "loss": 1.3019,
+      "theoretical_loss": 3.474378902681782,
+      "tokens_seen": 1895104512
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004299871609693468,
+      "loss": 1.1847,
+      "theoretical_loss": 3.474358820865705,
+      "tokens_seen": 1895235584
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042994703899855565,
+      "loss": 1.2528,
+      "theoretical_loss": 3.4743387408272537,
+      "tokens_seen": 1895366656
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004299069170277644,
+      "loss": 1.2764,
+      "theoretical_loss": 3.4743186625661484,
+      "tokens_seen": 1895497728
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042986679505697316,
+      "loss": 1.2154,
+      "theoretical_loss": 3.474298586082109,
+      "tokens_seen": 1895628800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000429826673086182,
+      "loss": 1.2418,
+      "theoretical_loss": 3.4742785113748553,
+      "tokens_seen": 1895759872
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042978655111539077,
+      "loss": 1.3012,
+      "theoretical_loss": 3.4742584384441075,
+      "tokens_seen": 1895890944
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004297464291445996,
+      "loss": 1.3326,
+      "theoretical_loss": 3.474238367289585,
+      "tokens_seen": 1896022016
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004297063071738084,
+      "loss": 1.318,
+      "theoretical_loss": 3.474218297911008,
+      "tokens_seen": 1896153088
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004296661852030172,
+      "loss": 1.3108,
+      "theoretical_loss": 3.4741982303080965,
+      "tokens_seen": 1896284160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000429626063232226,
+      "loss": 1.2528,
+      "theoretical_loss": 3.4741781644805716,
+      "tokens_seen": 1896415232
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004295859412614348,
+      "loss": 1.238,
+      "theoretical_loss": 3.474158100428152,
+      "tokens_seen": 1896546304
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042954581929064356,
+      "loss": 1.2486,
+      "theoretical_loss": 3.474138038150559,
+      "tokens_seen": 1896677376
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042950569731985236,
+      "loss": 1.2537,
+      "theoretical_loss": 3.4741179776475137,
+      "tokens_seen": 1896808448
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042946557534906117,
+      "loss": 1.3223,
+      "theoretical_loss": 3.474097918918735,
+      "tokens_seen": 1896939520
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004294254533782699,
+      "loss": 1.264,
+      "theoretical_loss": 3.474077861963944,
+      "tokens_seen": 1897070592
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004293853314074788,
+      "loss": 1.2817,
+      "theoretical_loss": 3.474057806782861,
+      "tokens_seen": 1897201664
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042934520943668753,
+      "loss": 1.3495,
+      "theoretical_loss": 3.4740377533752076,
+      "tokens_seen": 1897332736
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042930508746589634,
+      "loss": 1.1923,
+      "theoretical_loss": 3.4740177017407037,
+      "tokens_seen": 1897463808
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042926496549510515,
+      "loss": 1.3096,
+      "theoretical_loss": 3.4739976518790696,
+      "tokens_seen": 1897594880
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004292248435243139,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4739776037900265,
+      "tokens_seen": 1897725952
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004291847215535227,
+      "loss": 1.289,
+      "theoretical_loss": 3.4739575574732964,
+      "tokens_seen": 1897857024
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004291445995827315,
+      "loss": 1.3308,
+      "theoretical_loss": 3.4739375129285985,
+      "tokens_seen": 1897988096
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004291044776119403,
+      "loss": 1.2572,
+      "theoretical_loss": 3.473917470155655,
+      "tokens_seen": 1898119168
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004290643556411491,
+      "loss": 1.2604,
+      "theoretical_loss": 3.473897429154186,
+      "tokens_seen": 1898250240
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042902423367035793,
+      "loss": 1.2973,
+      "theoretical_loss": 3.4738773899239135,
+      "tokens_seen": 1898381312
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004289841116995667,
+      "loss": 1.3568,
+      "theoretical_loss": 3.4738573524645586,
+      "tokens_seen": 1898512384
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004289439897287755,
+      "loss": 1.2899,
+      "theoretical_loss": 3.473837316775842,
+      "tokens_seen": 1898643456
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004289038677579843,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4738172828574854,
+      "tokens_seen": 1898774528
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042886374578719305,
+      "loss": 1.2824,
+      "theoretical_loss": 3.47379725070921,
+      "tokens_seen": 1898905600
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 639521,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.294398546218872,
+      "objective/train/theoretical_loss": 3.4737772203307373,
+      "objective/train/tokens_used": 269495776,
+      "theoretical_loss": 3.4737772203307373,
+      "tokens_seen": 1899036672
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004288236238164019,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4737772203307373,
+      "tokens_seen": 1899036672
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042878350184561066,
+      "loss": 1.3368,
+      "theoretical_loss": 3.473757191721789,
+      "tokens_seen": 1899167744
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042874337987481946,
+      "loss": 1.2594,
+      "theoretical_loss": 3.4737371648820865,
+      "tokens_seen": 1899298816
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042870325790402827,
+      "loss": 1.3331,
+      "theoretical_loss": 3.4737171398113516,
+      "tokens_seen": 1899429888
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000428663135933237,
+      "loss": 1.3315,
+      "theoretical_loss": 3.4736971165093053,
+      "tokens_seen": 1899560960
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042862301396244583,
+      "loss": 1.3102,
+      "theoretical_loss": 3.4736770949756703,
+      "tokens_seen": 1899692032
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042858289199165464,
+      "loss": 1.2864,
+      "theoretical_loss": 3.473657075210168,
+      "tokens_seen": 1899823104
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042854277002086344,
+      "loss": 1.2899,
+      "theoretical_loss": 3.47363705721252,
+      "tokens_seen": 1899954176
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004285026480500722,
+      "loss": 1.2365,
+      "theoretical_loss": 3.473617040982449,
+      "tokens_seen": 1900085248
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042846252607928105,
+      "loss": 1.3438,
+      "theoretical_loss": 3.473597026519676,
+      "tokens_seen": 1900216320
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004284224041084898,
+      "loss": 1.278,
+      "theoretical_loss": 3.473577013823924,
+      "tokens_seen": 1900347392
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004283822821376986,
+      "loss": 1.2631,
+      "theoretical_loss": 3.473557002894915,
+      "tokens_seen": 1900478464
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004283421601669074,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4735369937323703,
+      "tokens_seen": 1900609536
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042830203819611617,
+      "loss": 1.3402,
+      "theoretical_loss": 3.4735169863360134,
+      "tokens_seen": 1900740608
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042826191622532503,
+      "loss": 1.303,
+      "theoretical_loss": 3.4734969807055656,
+      "tokens_seen": 1900871680
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004282217942545338,
+      "loss": 1.2758,
+      "theoretical_loss": 3.47347697684075,
+      "tokens_seen": 1901002752
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004281816722837426,
+      "loss": 1.3302,
+      "theoretical_loss": 3.4734569747412887,
+      "tokens_seen": 1901133824
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004281415503129514,
+      "loss": 1.3037,
+      "theoretical_loss": 3.473436974406904,
+      "tokens_seen": 1901264896
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042810142834216015,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4734169758373192,
+      "tokens_seen": 1901395968
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042806130637136896,
+      "loss": 1.2848,
+      "theoretical_loss": 3.4733969790322563,
+      "tokens_seen": 1901527040
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042802118440057776,
+      "loss": 1.3577,
+      "theoretical_loss": 3.4733769839914386,
+      "tokens_seen": 1901658112
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042798106242978657,
+      "loss": 1.3368,
+      "theoretical_loss": 3.4733569907145876,
+      "tokens_seen": 1901789184
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004279409404589953,
+      "loss": 1.3159,
+      "theoretical_loss": 3.473336999201428,
+      "tokens_seen": 1901920256
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004279008184882042,
+      "loss": 1.288,
+      "theoretical_loss": 3.473317009451681,
+      "tokens_seen": 1902051328
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042786069651741293,
+      "loss": 1.2973,
+      "theoretical_loss": 3.4732970214650702,
+      "tokens_seen": 1902182400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004278205745466218,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4732770352413187,
+      "tokens_seen": 1902313472
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042778045257583054,
+      "loss": 1.284,
+      "theoretical_loss": 3.4732570507801497,
+      "tokens_seen": 1902444544
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004277403306050393,
+      "loss": 1.3408,
+      "theoretical_loss": 3.473237068081286,
+      "tokens_seen": 1902575616
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042770020863424816,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4732170871444508,
+      "tokens_seen": 1902706688
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004276600866634569,
+      "loss": 1.2569,
+      "theoretical_loss": 3.473197107969368,
+      "tokens_seen": 1902837760
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004276199646926657,
+      "loss": 1.3135,
+      "theoretical_loss": 3.47317713055576,
+      "tokens_seen": 1902968832
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004275798427218745,
+      "loss": 1.3119,
+      "theoretical_loss": 3.473157154903351,
+      "tokens_seen": 1903099904
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042753972075108333,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4731371810118636,
+      "tokens_seen": 1903230976
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004274995987802921,
+      "loss": 1.3054,
+      "theoretical_loss": 3.473117208881022,
+      "tokens_seen": 1903362048
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004274594768095009,
+      "loss": 1.2618,
+      "theoretical_loss": 3.47309723851055,
+      "tokens_seen": 1903493120
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004274193548387097,
+      "loss": 1.2189,
+      "theoretical_loss": 3.47307726990017,
+      "tokens_seen": 1903624192
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042737923286791845,
+      "loss": 1.2328,
+      "theoretical_loss": 3.4730573030496075,
+      "tokens_seen": 1903755264
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004273391108971273,
+      "loss": 1.2139,
+      "theoretical_loss": 3.4730373379585844,
+      "tokens_seen": 1903886336
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042729898892633606,
+      "loss": 1.245,
+      "theoretical_loss": 3.473017374626826,
+      "tokens_seen": 1904017408
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042725886695554486,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4729974130540553,
+      "tokens_seen": 1904148480
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042721874498475367,
+      "loss": 1.2633,
+      "theoretical_loss": 3.472977453239996,
+      "tokens_seen": 1904279552
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004271786230139624,
+      "loss": 1.3145,
+      "theoretical_loss": 3.4729574951843736,
+      "tokens_seen": 1904410624
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004271385010431713,
+      "loss": 1.3004,
+      "theoretical_loss": 3.47293753888691,
+      "tokens_seen": 1904541696
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042709837907238004,
+      "loss": 1.2663,
+      "theoretical_loss": 3.472917584347332,
+      "tokens_seen": 1904672768
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042705825710158884,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4728976315653615,
+      "tokens_seen": 1904803840
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042701813513079765,
+      "loss": 1.376,
+      "theoretical_loss": 3.4728776805407238,
+      "tokens_seen": 1904934912
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042697801316000645,
+      "loss": 1.3099,
+      "theoretical_loss": 3.472857731273143,
+      "tokens_seen": 1905065984
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004269378911892152,
+      "loss": 1.2801,
+      "theoretical_loss": 3.4728377837623436,
+      "tokens_seen": 1905197056
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000426897769218424,
+      "loss": 1.3325,
+      "theoretical_loss": 3.4728178380080497,
+      "tokens_seen": 1905328128
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004268576472476328,
+      "loss": 1.2753,
+      "theoretical_loss": 3.472797894009987,
+      "tokens_seen": 1905459200
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 641709,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3416887521743774,
+      "objective/train/theoretical_loss": 3.472777951767878,
+      "objective/train/tokens_used": 276049376,
+      "theoretical_loss": 3.472777951767878,
+      "tokens_seen": 1905590272
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042681752527684157,
+      "loss": 1.3658,
+      "theoretical_loss": 3.472777951767878,
+      "tokens_seen": 1905590272
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042677740330605043,
+      "loss": 1.2948,
+      "theoretical_loss": 3.472758011281449,
+      "tokens_seen": 1905721344
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004267372813352592,
+      "loss": 1.3323,
+      "theoretical_loss": 3.472738072550424,
+      "tokens_seen": 1905852416
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.000426697159364468,
+      "loss": 1.3665,
+      "theoretical_loss": 3.472718135574528,
+      "tokens_seen": 1905983488
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004266570373936768,
+      "loss": 1.3475,
+      "theoretical_loss": 3.472698200353486,
+      "tokens_seen": 1906114560
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042661691542288555,
+      "loss": 1.3656,
+      "theoretical_loss": 3.472678266887023,
+      "tokens_seen": 1906245632
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042657679345209435,
+      "loss": 1.3965,
+      "theoretical_loss": 3.472658335174863,
+      "tokens_seen": 1906376704
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042653667148130316,
+      "loss": 1.2754,
+      "theoretical_loss": 3.4726384052167316,
+      "tokens_seen": 1906507776
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042649654951051197,
+      "loss": 1.2923,
+      "theoretical_loss": 3.472618477012354,
+      "tokens_seen": 1906638848
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004264564275397208,
+      "loss": 1.3304,
+      "theoretical_loss": 3.472598550561455,
+      "tokens_seen": 1906769920
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004264163055689296,
+      "loss": 1.3434,
+      "theoretical_loss": 3.4725786258637603,
+      "tokens_seen": 1906900992
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042637618359813833,
+      "loss": 1.3114,
+      "theoretical_loss": 3.472558702918995,
+      "tokens_seen": 1907032064
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004263360616273472,
+      "loss": 1.3076,
+      "theoretical_loss": 3.472538781726884,
+      "tokens_seen": 1907163136
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042629593965655594,
+      "loss": 1.2819,
+      "theoretical_loss": 3.472518862287153,
+      "tokens_seen": 1907294208
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004262558176857647,
+      "loss": 1.3325,
+      "theoretical_loss": 3.472498944599527,
+      "tokens_seen": 1907425280
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042621569571497356,
+      "loss": 1.2558,
+      "theoretical_loss": 3.4724790286637326,
+      "tokens_seen": 1907556352
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004261755737441823,
+      "loss": 1.2667,
+      "theoretical_loss": 3.472459114479494,
+      "tokens_seen": 1907687424
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004261354517733911,
+      "loss": 1.3203,
+      "theoretical_loss": 3.4724392020465373,
+      "tokens_seen": 1907818496
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004260953298025999,
+      "loss": 1.3235,
+      "theoretical_loss": 3.4724192913645893,
+      "tokens_seen": 1907949568
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042605520783180873,
+      "loss": 1.2595,
+      "theoretical_loss": 3.472399382433374,
+      "tokens_seen": 1908080640
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004260150858610175,
+      "loss": 1.2212,
+      "theoretical_loss": 3.4723794752526187,
+      "tokens_seen": 1908211712
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004259749638902263,
+      "loss": 1.3038,
+      "theoretical_loss": 3.472359569822048,
+      "tokens_seen": 1908342784
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004259348419194351,
+      "loss": 1.2856,
+      "theoretical_loss": 3.472339666141389,
+      "tokens_seen": 1908473856
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042589471994864385,
+      "loss": 1.3401,
+      "theoretical_loss": 3.4723197642103667,
+      "tokens_seen": 1908604928
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004258545979778527,
+      "loss": 1.3805,
+      "theoretical_loss": 3.4722998640287077,
+      "tokens_seen": 1908736000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042581447600706146,
+      "loss": 1.3108,
+      "theoretical_loss": 3.472279965596138,
+      "tokens_seen": 1908867072
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004257743540362703,
+      "loss": 1.3331,
+      "theoretical_loss": 3.4722600689123837,
+      "tokens_seen": 1908998144
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042573423206547907,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4722401739771716,
+      "tokens_seen": 1909129216
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004256941100946878,
+      "loss": 1.2856,
+      "theoretical_loss": 3.472220280790227,
+      "tokens_seen": 1909260288
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004256539881238967,
+      "loss": 1.2527,
+      "theoretical_loss": 3.4722003893512774,
+      "tokens_seen": 1909391360
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042561386615310543,
+      "loss": 1.3512,
+      "theoretical_loss": 3.4721804996600483,
+      "tokens_seen": 1909522432
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042557374418231424,
+      "loss": 1.3352,
+      "theoretical_loss": 3.472160611716266,
+      "tokens_seen": 1909653504
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042553362221152305,
+      "loss": 1.3586,
+      "theoretical_loss": 3.472140725519658,
+      "tokens_seen": 1909784576
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042549350024073185,
+      "loss": 1.3823,
+      "theoretical_loss": 3.472120841069951,
+      "tokens_seen": 1909915648
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004254533782699406,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4721009583668705,
+      "tokens_seen": 1910046720
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004254132562991494,
+      "loss": 1.3014,
+      "theoretical_loss": 3.4720810774101443,
+      "tokens_seen": 1910177792
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004253731343283582,
+      "loss": 1.2794,
+      "theoretical_loss": 3.4720611981994987,
+      "tokens_seen": 1910308864
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042533301235756697,
+      "loss": 1.2309,
+      "theoretical_loss": 3.4720413207346605,
+      "tokens_seen": 1910439936
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042529289038677583,
+      "loss": 1.2838,
+      "theoretical_loss": 3.472021445015357,
+      "tokens_seen": 1910571008
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004252527684159846,
+      "loss": 1.2688,
+      "theoretical_loss": 3.4720015710413152,
+      "tokens_seen": 1910702080
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042521264644519344,
+      "loss": 1.3565,
+      "theoretical_loss": 3.471981698812261,
+      "tokens_seen": 1910833152
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004251725244744022,
+      "loss": 1.2428,
+      "theoretical_loss": 3.471961828327923,
+      "tokens_seen": 1910964224
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042513240250361095,
+      "loss": 1.317,
+      "theoretical_loss": 3.471941959588028,
+      "tokens_seen": 1911095296
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004250922805328198,
+      "loss": 1.2556,
+      "theoretical_loss": 3.471922092592303,
+      "tokens_seen": 1911226368
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042505215856202856,
+      "loss": 1.2404,
+      "theoretical_loss": 3.471902227340475,
+      "tokens_seen": 1911357440
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042501203659123737,
+      "loss": 1.3849,
+      "theoretical_loss": 3.471882363832272,
+      "tokens_seen": 1911488512
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004249719146204462,
+      "loss": 1.308,
+      "theoretical_loss": 3.4718625020674203,
+      "tokens_seen": 1911619584
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.000424931792649655,
+      "loss": 1.3399,
+      "theoretical_loss": 3.4718426420456487,
+      "tokens_seen": 1911750656
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042489167067886373,
+      "loss": 1.2971,
+      "theoretical_loss": 3.471822783766684,
+      "tokens_seen": 1911881728
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042485154870807254,
+      "loss": 1.2578,
+      "theoretical_loss": 3.471802927230254,
+      "tokens_seen": 1912012800
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 643869,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.348339557647705,
+      "objective/train/theoretical_loss": 3.4717830724360867,
+      "objective/train/tokens_used": 282602976,
+      "theoretical_loss": 3.4717830724360867,
+      "tokens_seen": 1912143872
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042481142673728134,
+      "loss": 1.3258,
+      "theoretical_loss": 3.4717830724360867,
+      "tokens_seen": 1912143872
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004247713047664901,
+      "loss": 1.2803,
+      "theoretical_loss": 3.471763219383909,
+      "tokens_seen": 1912274944
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042473118279569896,
+      "loss": 1.3453,
+      "theoretical_loss": 3.471743368073449,
+      "tokens_seen": 1912406016
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004246910608249077,
+      "loss": 1.2842,
+      "theoretical_loss": 3.471723518504435,
+      "tokens_seen": 1912537088
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004246509388541165,
+      "loss": 1.2942,
+      "theoretical_loss": 3.471703670676595,
+      "tokens_seen": 1912668160
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004246108168833253,
+      "loss": 1.2988,
+      "theoretical_loss": 3.4716838245896557,
+      "tokens_seen": 1912799232
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042457069491253413,
+      "loss": 1.315,
+      "theoretical_loss": 3.4716639802433464,
+      "tokens_seen": 1912930304
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042453057294174293,
+      "loss": 1.3253,
+      "theoretical_loss": 3.4716441376373943,
+      "tokens_seen": 1913061376
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004244904509709517,
+      "loss": 1.2802,
+      "theoretical_loss": 3.4716242967715285,
+      "tokens_seen": 1913192448
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004244503290001605,
+      "loss": 1.3141,
+      "theoretical_loss": 3.4716044576454763,
+      "tokens_seen": 1913323520
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004244102070293693,
+      "loss": 1.2761,
+      "theoretical_loss": 3.471584620258966,
+      "tokens_seen": 1913454592
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004243700850585781,
+      "loss": 1.3001,
+      "theoretical_loss": 3.471564784611727,
+      "tokens_seen": 1913585664
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042432996308778686,
+      "loss": 1.2978,
+      "theoretical_loss": 3.471544950703487,
+      "tokens_seen": 1913716736
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004242898411169957,
+      "loss": 1.2804,
+      "theoretical_loss": 3.471525118533974,
+      "tokens_seen": 1913847808
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042424971914620447,
+      "loss": 1.274,
+      "theoretical_loss": 3.471505288102917,
+      "tokens_seen": 1913978880
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004242095971754132,
+      "loss": 1.2696,
+      "theoretical_loss": 3.4714854594100446,
+      "tokens_seen": 1914109952
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004241694752046221,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4714656324550854,
+      "tokens_seen": 1914241024
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042412935323383083,
+      "loss": 1.2858,
+      "theoretical_loss": 3.4714458072377683,
+      "tokens_seen": 1914372096
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042408923126303964,
+      "loss": 1.297,
+      "theoretical_loss": 3.471425983757821,
+      "tokens_seen": 1914503168
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042404910929224845,
+      "loss": 1.2675,
+      "theoretical_loss": 3.471406162014974,
+      "tokens_seen": 1914634240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042400898732145725,
+      "loss": 1.2532,
+      "theoretical_loss": 3.471386342008955,
+      "tokens_seen": 1914765312
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042396886535066606,
+      "loss": 1.2965,
+      "theoretical_loss": 3.471366523739493,
+      "tokens_seen": 1914896384
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004239287433798748,
+      "loss": 1.3954,
+      "theoretical_loss": 3.471346707206317,
+      "tokens_seen": 1915027456
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004238886214090836,
+      "loss": 1.2604,
+      "theoretical_loss": 3.4713268924091567,
+      "tokens_seen": 1915158528
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004238484994382924,
+      "loss": 1.3406,
+      "theoretical_loss": 3.471307079347741,
+      "tokens_seen": 1915289600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042380837746750123,
+      "loss": 1.3543,
+      "theoretical_loss": 3.4712872680217988,
+      "tokens_seen": 1915420672
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042376825549671,
+      "loss": 1.2,
+      "theoretical_loss": 3.471267458431059,
+      "tokens_seen": 1915551744
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042372813352591884,
+      "loss": 1.3161,
+      "theoretical_loss": 3.4712476505752514,
+      "tokens_seen": 1915682816
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004236880115551276,
+      "loss": 1.2637,
+      "theoretical_loss": 3.4712278444541056,
+      "tokens_seen": 1915813888
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042364788958433635,
+      "loss": 1.3104,
+      "theoretical_loss": 3.471208040067351,
+      "tokens_seen": 1915944960
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004236077676135452,
+      "loss": 1.2168,
+      "theoretical_loss": 3.4711882374147165,
+      "tokens_seen": 1916076032
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042356764564275396,
+      "loss": 1.2888,
+      "theoretical_loss": 3.4711684364959323,
+      "tokens_seen": 1916207104
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042352752367196277,
+      "loss": 1.2841,
+      "theoretical_loss": 3.471148637310727,
+      "tokens_seen": 1916338176
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004234874017011716,
+      "loss": 1.2745,
+      "theoretical_loss": 3.4711288398588316,
+      "tokens_seen": 1916469248
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004234472797303804,
+      "loss": 1.2847,
+      "theoretical_loss": 3.4711090441399746,
+      "tokens_seen": 1916600320
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042340715775958913,
+      "loss": 1.2582,
+      "theoretical_loss": 3.4710892501538866,
+      "tokens_seen": 1916731392
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042336703578879794,
+      "loss": 1.3162,
+      "theoretical_loss": 3.471069457900297,
+      "tokens_seen": 1916862464
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042332691381800674,
+      "loss": 1.2485,
+      "theoretical_loss": 3.4710496673789364,
+      "tokens_seen": 1916993536
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042328679184721555,
+      "loss": 1.2783,
+      "theoretical_loss": 3.471029878589534,
+      "tokens_seen": 1917124608
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042324666987642436,
+      "loss": 1.2122,
+      "theoretical_loss": 3.47101009153182,
+      "tokens_seen": 1917255680
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004232065479056331,
+      "loss": 1.3116,
+      "theoretical_loss": 3.470990306205525,
+      "tokens_seen": 1917386752
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042316642593484197,
+      "loss": 1.3431,
+      "theoretical_loss": 3.4709705226103784,
+      "tokens_seen": 1917517824
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004231263039640507,
+      "loss": 1.2462,
+      "theoretical_loss": 3.470950740746111,
+      "tokens_seen": 1917648896
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042308618199325953,
+      "loss": 1.3089,
+      "theoretical_loss": 3.4709309606124528,
+      "tokens_seen": 1917779968
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042304606002246833,
+      "loss": 1.2751,
+      "theoretical_loss": 3.470911182209134,
+      "tokens_seen": 1917911040
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004230059380516771,
+      "loss": 1.3166,
+      "theoretical_loss": 3.4708914055358857,
+      "tokens_seen": 1918042112
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004229658160808859,
+      "loss": 1.3011,
+      "theoretical_loss": 3.4708716305924376,
+      "tokens_seen": 1918173184
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004229256941100947,
+      "loss": 1.2582,
+      "theoretical_loss": 3.47085185737852,
+      "tokens_seen": 1918304256
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004228855721393035,
+      "loss": 1.2819,
+      "theoretical_loss": 3.470832085893864,
+      "tokens_seen": 1918435328
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042284545016851226,
+      "loss": 1.3003,
+      "theoretical_loss": 3.4708123161382005,
+      "tokens_seen": 1918566400
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 645829,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2091408967971802,
+      "objective/train/theoretical_loss": 3.47079254811126,
+      "objective/train/tokens_used": 289156576,
+      "theoretical_loss": 3.47079254811126,
+      "tokens_seen": 1918697472
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004228053281977211,
+      "loss": 1.2512,
+      "theoretical_loss": 3.47079254811126,
+      "tokens_seen": 1918697472
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042276520622692987,
+      "loss": 1.2855,
+      "theoretical_loss": 3.470772781812773,
+      "tokens_seen": 1918828544
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004227250842561386,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4707530172424703,
+      "tokens_seen": 1918959616
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004226849622853475,
+      "loss": 1.258,
+      "theoretical_loss": 3.4707332544000833,
+      "tokens_seen": 1919090688
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042264484031455623,
+      "loss": 1.2976,
+      "theoretical_loss": 3.470713493285342,
+      "tokens_seen": 1919221760
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004226047183437651,
+      "loss": 1.3078,
+      "theoretical_loss": 3.4706937338979778,
+      "tokens_seen": 1919352832
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042256459637297385,
+      "loss": 1.272,
+      "theoretical_loss": 3.4706739762377223,
+      "tokens_seen": 1919483904
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042252447440218265,
+      "loss": 1.277,
+      "theoretical_loss": 3.4706542203043065,
+      "tokens_seen": 1919614976
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042248435243139146,
+      "loss": 1.2835,
+      "theoretical_loss": 3.4706344660974606,
+      "tokens_seen": 1919746048
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004224442304606002,
+      "loss": 1.2838,
+      "theoretical_loss": 3.470614713616917,
+      "tokens_seen": 1919877120
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.000422404108489809,
+      "loss": 1.2547,
+      "theoretical_loss": 3.470594962862407,
+      "tokens_seen": 1920008192
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004223639865190178,
+      "loss": 1.2933,
+      "theoretical_loss": 3.4705752138336603,
+      "tokens_seen": 1920139264
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042232386454822663,
+      "loss": 1.3381,
+      "theoretical_loss": 3.470555466530411,
+      "tokens_seen": 1920270336
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004222837425774354,
+      "loss": 1.2621,
+      "theoretical_loss": 3.470535720952388,
+      "tokens_seen": 1920401408
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042224362060664424,
+      "loss": 1.3093,
+      "theoretical_loss": 3.470515977099324,
+      "tokens_seen": 1920532480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.000422203498635853,
+      "loss": 1.3087,
+      "theoretical_loss": 3.470496234970951,
+      "tokens_seen": 1920663552
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042216337666506175,
+      "loss": 1.2454,
+      "theoretical_loss": 3.470476494567,
+      "tokens_seen": 1920794624
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004221232546942706,
+      "loss": 1.2456,
+      "theoretical_loss": 3.470456755887203,
+      "tokens_seen": 1920925696
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042208313272347936,
+      "loss": 1.3204,
+      "theoretical_loss": 3.4704370189312916,
+      "tokens_seen": 1921056768
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004220430107526882,
+      "loss": 1.2417,
+      "theoretical_loss": 3.4704172836989975,
+      "tokens_seen": 1921187840
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042200288878189697,
+      "loss": 1.343,
+      "theoretical_loss": 3.470397550190053,
+      "tokens_seen": 1921318912
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004219627668111058,
+      "loss": 1.2678,
+      "theoretical_loss": 3.47037781840419,
+      "tokens_seen": 1921449984
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004219226448403146,
+      "loss": 1.235,
+      "theoretical_loss": 3.4703580883411407,
+      "tokens_seen": 1921581056
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042188252286952334,
+      "loss": 1.2815,
+      "theoretical_loss": 3.4703383600006363,
+      "tokens_seen": 1921712128
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042184240089873214,
+      "loss": 1.3065,
+      "theoretical_loss": 3.47031863338241,
+      "tokens_seen": 1921843200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042180227892794095,
+      "loss": 1.2608,
+      "theoretical_loss": 3.4702989084861935,
+      "tokens_seen": 1921974272
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042176215695714976,
+      "loss": 1.2536,
+      "theoretical_loss": 3.4702791853117185,
+      "tokens_seen": 1922105344
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004217220349863585,
+      "loss": 1.2273,
+      "theoretical_loss": 3.470259463858719,
+      "tokens_seen": 1922236416
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042168191301556737,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4702397441269253,
+      "tokens_seen": 1922367488
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004216417910447761,
+      "loss": 1.2435,
+      "theoretical_loss": 3.4702200261160714,
+      "tokens_seen": 1922498560
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042160166907398493,
+      "loss": 1.3506,
+      "theoretical_loss": 3.4702003098258887,
+      "tokens_seen": 1922629632
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042156154710319373,
+      "loss": 1.2719,
+      "theoretical_loss": 3.470180595256111,
+      "tokens_seen": 1922760704
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004215214251324025,
+      "loss": 1.3506,
+      "theoretical_loss": 3.4701608824064696,
+      "tokens_seen": 1922891776
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004214813031616113,
+      "loss": 1.322,
+      "theoretical_loss": 3.4701411712766985,
+      "tokens_seen": 1923022848
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004214411811908201,
+      "loss": 1.2381,
+      "theoretical_loss": 3.470121461866529,
+      "tokens_seen": 1923153920
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004214010592200289,
+      "loss": 1.2418,
+      "theoretical_loss": 3.470101754175695,
+      "tokens_seen": 1923284992
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004213609372492377,
+      "loss": 1.2545,
+      "theoretical_loss": 3.4700820482039285,
+      "tokens_seen": 1923416064
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004213208152784465,
+      "loss": 1.1683,
+      "theoretical_loss": 3.4700623439509637,
+      "tokens_seen": 1923547136
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042128069330765527,
+      "loss": 1.3398,
+      "theoretical_loss": 3.4700426414165326,
+      "tokens_seen": 1923678208
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004212405713368641,
+      "loss": 1.3322,
+      "theoretical_loss": 3.470022940600368,
+      "tokens_seen": 1923809280
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004212004493660729,
+      "loss": 1.3103,
+      "theoretical_loss": 3.4700032415022033,
+      "tokens_seen": 1923940352
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042116032739528163,
+      "loss": 1.255,
+      "theoretical_loss": 3.4699835441217726,
+      "tokens_seen": 1924071424
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004211202054244905,
+      "loss": 1.198,
+      "theoretical_loss": 3.4699638484588076,
+      "tokens_seen": 1924202496
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042108008345369925,
+      "loss": 1.2168,
+      "theoretical_loss": 3.4699441545130423,
+      "tokens_seen": 1924333568
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042103996148290805,
+      "loss": 1.2989,
+      "theoretical_loss": 3.46992446228421,
+      "tokens_seen": 1924464640
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042099983951211686,
+      "loss": 1.2575,
+      "theoretical_loss": 3.469904771772044,
+      "tokens_seen": 1924595712
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004209597175413256,
+      "loss": 1.2305,
+      "theoretical_loss": 3.4698850829762784,
+      "tokens_seen": 1924726784
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004209195955705344,
+      "loss": 1.2944,
+      "theoretical_loss": 3.4698653958966457,
+      "tokens_seen": 1924857856
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004208794735997432,
+      "loss": 1.2416,
+      "theoretical_loss": 3.46984571053288,
+      "tokens_seen": 1924988928
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042083935162895203,
+      "loss": 1.2177,
+      "theoretical_loss": 3.469826026884715,
+      "tokens_seen": 1925120000
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 648016,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2386829853057861,
+      "objective/train/theoretical_loss": 3.469806344951884,
+      "objective/train/tokens_used": 295710176,
+      "theoretical_loss": 3.469806344951884,
+      "tokens_seen": 1925251072
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004207992296581608,
+      "loss": 1.2449,
+      "theoretical_loss": 3.469806344951884,
+      "tokens_seen": 1925251072
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042075910768736964,
+      "loss": 1.2098,
+      "theoretical_loss": 3.469786664734121,
+      "tokens_seen": 1925382144
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004207189857165784,
+      "loss": 1.2457,
+      "theoretical_loss": 3.46976698623116,
+      "tokens_seen": 1925513216
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004206788637457872,
+      "loss": 1.2868,
+      "theoretical_loss": 3.469747309442735,
+      "tokens_seen": 1925644288
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.000420638741774996,
+      "loss": 1.2314,
+      "theoretical_loss": 3.4697276343685792,
+      "tokens_seen": 1925775360
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042059861980420476,
+      "loss": 1.2625,
+      "theoretical_loss": 3.469707961008427,
+      "tokens_seen": 1925906432
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004205584978334136,
+      "loss": 1.2815,
+      "theoretical_loss": 3.4696882893620127,
+      "tokens_seen": 1926037504
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042051837586262237,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4696686194290702,
+      "tokens_seen": 1926168576
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004204782538918312,
+      "loss": 1.2587,
+      "theoretical_loss": 3.469648951209333,
+      "tokens_seen": 1926299648
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042043813192104,
+      "loss": 1.3229,
+      "theoretical_loss": 3.4696292847025365,
+      "tokens_seen": 1926430720
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042039800995024874,
+      "loss": 1.2715,
+      "theoretical_loss": 3.4696096199084145,
+      "tokens_seen": 1926561792
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042035788797945754,
+      "loss": 1.2891,
+      "theoretical_loss": 3.469589956826701,
+      "tokens_seen": 1926692864
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042031776600866635,
+      "loss": 1.2996,
+      "theoretical_loss": 3.469570295457131,
+      "tokens_seen": 1926823936
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042027764403787516,
+      "loss": 1.2852,
+      "theoretical_loss": 3.469550635799438,
+      "tokens_seen": 1926955008
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004202375220670839,
+      "loss": 1.2649,
+      "theoretical_loss": 3.469530977853358,
+      "tokens_seen": 1927086080
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042019740009629277,
+      "loss": 1.24,
+      "theoretical_loss": 3.4695113216186244,
+      "tokens_seen": 1927217152
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004201572781255015,
+      "loss": 1.3189,
+      "theoretical_loss": 3.469491667094972,
+      "tokens_seen": 1927348224
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004201171561547103,
+      "loss": 1.2969,
+      "theoretical_loss": 3.469472014282136,
+      "tokens_seen": 1927479296
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042007703418391913,
+      "loss": 1.311,
+      "theoretical_loss": 3.4694523631798506,
+      "tokens_seen": 1927610368
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004200369122131279,
+      "loss": 1.3128,
+      "theoretical_loss": 3.4694327137878513,
+      "tokens_seen": 1927741440
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041999679024233675,
+      "loss": 1.2648,
+      "theoretical_loss": 3.469413066105872,
+      "tokens_seen": 1927872512
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004199566682715455,
+      "loss": 1.2984,
+      "theoretical_loss": 3.469393420133649,
+      "tokens_seen": 1928003584
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004199165463007543,
+      "loss": 1.2446,
+      "theoretical_loss": 3.469373775870916,
+      "tokens_seen": 1928134656
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004198764243299631,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4693541333174083,
+      "tokens_seen": 1928265728
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004198363023591719,
+      "loss": 1.2425,
+      "theoretical_loss": 3.4693344924728615,
+      "tokens_seen": 1928396800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041979618038838067,
+      "loss": 1.2761,
+      "theoretical_loss": 3.4693148533370106,
+      "tokens_seen": 1928527872
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004197560584175895,
+      "loss": 1.2226,
+      "theoretical_loss": 3.4692952159095904,
+      "tokens_seen": 1928658944
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004197159364467983,
+      "loss": 1.2546,
+      "theoretical_loss": 3.469275580190337,
+      "tokens_seen": 1928790016
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041967581447600703,
+      "loss": 1.2714,
+      "theoretical_loss": 3.4692559461789854,
+      "tokens_seen": 1928921088
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004196356925052159,
+      "loss": 1.2881,
+      "theoretical_loss": 3.46923631387527,
+      "tokens_seen": 1929052160
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041959557053442465,
+      "loss": 1.3148,
+      "theoretical_loss": 3.4692166832789284,
+      "tokens_seen": 1929183232
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041955544856363345,
+      "loss": 1.3567,
+      "theoretical_loss": 3.469197054389694,
+      "tokens_seen": 1929314304
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041951532659284226,
+      "loss": 1.2467,
+      "theoretical_loss": 3.469177427207304,
+      "tokens_seen": 1929445376
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.000419475204622051,
+      "loss": 1.2393,
+      "theoretical_loss": 3.469157801731493,
+      "tokens_seen": 1929576448
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041943508265125987,
+      "loss": 1.2563,
+      "theoretical_loss": 3.469138177961997,
+      "tokens_seen": 1929707520
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004193949606804686,
+      "loss": 1.2951,
+      "theoretical_loss": 3.469118555898552,
+      "tokens_seen": 1929838592
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041935483870967743,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4690989355408934,
+      "tokens_seen": 1929969664
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041931471673888624,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4690793168887577,
+      "tokens_seen": 1930100736
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041927459476809504,
+      "loss": 1.2743,
+      "theoretical_loss": 3.46905969994188,
+      "tokens_seen": 1930231808
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004192344727973038,
+      "loss": 1.2562,
+      "theoretical_loss": 3.469040084699997,
+      "tokens_seen": 1930362880
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004191943508265126,
+      "loss": 1.296,
+      "theoretical_loss": 3.4690204711628443,
+      "tokens_seen": 1930493952
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004191542288557214,
+      "loss": 1.3882,
+      "theoretical_loss": 3.4690008593301584,
+      "tokens_seen": 1930625024
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041911410688493016,
+      "loss": 1.263,
+      "theoretical_loss": 3.4689812492016756,
+      "tokens_seen": 1930756096
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.000419073984914139,
+      "loss": 1.1987,
+      "theoretical_loss": 3.4689616407771315,
+      "tokens_seen": 1930887168
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041903386294334777,
+      "loss": 1.2769,
+      "theoretical_loss": 3.468942034056263,
+      "tokens_seen": 1931018240
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004189937409725566,
+      "loss": 1.2669,
+      "theoretical_loss": 3.4689224290388054,
+      "tokens_seen": 1931149312
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004189536190017654,
+      "loss": 1.294,
+      "theoretical_loss": 3.4689028257244967,
+      "tokens_seen": 1931280384
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041891349703097414,
+      "loss": 1.3228,
+      "theoretical_loss": 3.468883224113072,
+      "tokens_seen": 1931411456
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.000418873375060183,
+      "loss": 1.2705,
+      "theoretical_loss": 3.468863624204269,
+      "tokens_seen": 1931542528
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041883325308939175,
+      "loss": 1.2871,
+      "theoretical_loss": 3.4688440259978233,
+      "tokens_seen": 1931673600
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 650108,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2642157077789307,
+      "objective/train/theoretical_loss": 3.4688244294934716,
+      "objective/train/tokens_used": 302263776,
+      "theoretical_loss": 3.4688244294934716,
+      "tokens_seen": 1931804672
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041879313111860056,
+      "loss": 1.2604,
+      "theoretical_loss": 3.4688244294934716,
+      "tokens_seen": 1931804672
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041875300914780936,
+      "loss": 1.236,
+      "theoretical_loss": 3.4688048346909515,
+      "tokens_seen": 1931935744
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041871288717701817,
+      "loss": 1.301,
+      "theoretical_loss": 3.468785241589999,
+      "tokens_seen": 1932066816
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004186727652062269,
+      "loss": 1.3718,
+      "theoretical_loss": 3.4687656501903508,
+      "tokens_seen": 1932197888
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004186326432354357,
+      "loss": 1.2406,
+      "theoretical_loss": 3.4687460604917444,
+      "tokens_seen": 1932328960
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041859252126464453,
+      "loss": 1.2542,
+      "theoretical_loss": 3.468726472493916,
+      "tokens_seen": 1932460032
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004185523992938533,
+      "loss": 1.3461,
+      "theoretical_loss": 3.4687068861966037,
+      "tokens_seen": 1932591104
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041851227732306215,
+      "loss": 1.2967,
+      "theoretical_loss": 3.4686873015995436,
+      "tokens_seen": 1932722176
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004184721553522709,
+      "loss": 1.2405,
+      "theoretical_loss": 3.4686677187024726,
+      "tokens_seen": 1932853248
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004184320333814797,
+      "loss": 1.2155,
+      "theoretical_loss": 3.468648137505129,
+      "tokens_seen": 1932984320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004183919114106885,
+      "loss": 1.2305,
+      "theoretical_loss": 3.468628558007249,
+      "tokens_seen": 1933115392
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004183517894398973,
+      "loss": 1.2017,
+      "theoretical_loss": 3.4686089802085704,
+      "tokens_seen": 1933246464
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041831166746910607,
+      "loss": 1.2954,
+      "theoretical_loss": 3.46858940410883,
+      "tokens_seen": 1933377536
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004182715454983149,
+      "loss": 1.3475,
+      "theoretical_loss": 3.468569829707766,
+      "tokens_seen": 1933508608
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004182314235275237,
+      "loss": 1.3321,
+      "theoretical_loss": 3.4685502570051154,
+      "tokens_seen": 1933639680
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004181913015567325,
+      "loss": 1.2751,
+      "theoretical_loss": 3.468530686000616,
+      "tokens_seen": 1933770752
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004181511795859413,
+      "loss": 1.234,
+      "theoretical_loss": 3.468511116694005,
+      "tokens_seen": 1933901824
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041811105761515005,
+      "loss": 1.258,
+      "theoretical_loss": 3.4684915490850203,
+      "tokens_seen": 1934032896
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004180709356443589,
+      "loss": 1.2105,
+      "theoretical_loss": 3.4684719831733997,
+      "tokens_seen": 1934163968
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041803081367356766,
+      "loss": 1.3133,
+      "theoretical_loss": 3.46845241895888,
+      "tokens_seen": 1934295040
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004179906917027764,
+      "loss": 1.2996,
+      "theoretical_loss": 3.4684328564412006,
+      "tokens_seen": 1934426112
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041795056973198527,
+      "loss": 1.3049,
+      "theoretical_loss": 3.468413295620098,
+      "tokens_seen": 1934557184
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.000417910447761194,
+      "loss": 1.308,
+      "theoretical_loss": 3.468393736495311,
+      "tokens_seen": 1934688256
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041787032579040283,
+      "loss": 1.2955,
+      "theoretical_loss": 3.4683741790665774,
+      "tokens_seen": 1934819328
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041783020381961164,
+      "loss": 1.3392,
+      "theoretical_loss": 3.4683546233336346,
+      "tokens_seen": 1934950400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041779008184882044,
+      "loss": 1.3072,
+      "theoretical_loss": 3.468335069296222,
+      "tokens_seen": 1935081472
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004177499598780292,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4683155169540765,
+      "tokens_seen": 1935212544
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.000417709837907238,
+      "loss": 1.2969,
+      "theoretical_loss": 3.4682959663069366,
+      "tokens_seen": 1935343616
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004176697159364468,
+      "loss": 1.2611,
+      "theoretical_loss": 3.4682764173545406,
+      "tokens_seen": 1935474688
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041762959396565556,
+      "loss": 1.3009,
+      "theoretical_loss": 3.468256870096628,
+      "tokens_seen": 1935605760
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004175894719948644,
+      "loss": 1.3187,
+      "theoretical_loss": 3.468237324532935,
+      "tokens_seen": 1935736832
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041754935002407317,
+      "loss": 1.262,
+      "theoretical_loss": 3.468217780663202,
+      "tokens_seen": 1935867904
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041750922805328203,
+      "loss": 1.2704,
+      "theoretical_loss": 3.468198238487167,
+      "tokens_seen": 1935998976
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004174691060824908,
+      "loss": 1.2769,
+      "theoretical_loss": 3.468178698004568,
+      "tokens_seen": 1936130048
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041742898411169954,
+      "loss": 1.3086,
+      "theoretical_loss": 3.4681591592151437,
+      "tokens_seen": 1936261120
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004173888621409084,
+      "loss": 1.3293,
+      "theoretical_loss": 3.468139622118634,
+      "tokens_seen": 1936392192
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041734874017011715,
+      "loss": 1.3141,
+      "theoretical_loss": 3.4681200867147757,
+      "tokens_seen": 1936523264
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041730861819932596,
+      "loss": 1.2421,
+      "theoretical_loss": 3.4681005530033087,
+      "tokens_seen": 1936654336
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041726849622853476,
+      "loss": 1.3289,
+      "theoretical_loss": 3.4680810209839716,
+      "tokens_seen": 1936785408
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041722837425774357,
+      "loss": 1.273,
+      "theoretical_loss": 3.4680614906565044,
+      "tokens_seen": 1936916480
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004171882522869523,
+      "loss": 1.315,
+      "theoretical_loss": 3.468041962020645,
+      "tokens_seen": 1937047552
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004171481303161611,
+      "loss": 1.2106,
+      "theoretical_loss": 3.468022435076132,
+      "tokens_seen": 1937178624
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041710800834536993,
+      "loss": 1.2825,
+      "theoretical_loss": 3.468002909822705,
+      "tokens_seen": 1937309696
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004170678863745787,
+      "loss": 1.2787,
+      "theoretical_loss": 3.4679833862601033,
+      "tokens_seen": 1937440768
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041702776440378755,
+      "loss": 1.3726,
+      "theoretical_loss": 3.467963864388066,
+      "tokens_seen": 1937571840
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004169876424329963,
+      "loss": 1.3197,
+      "theoretical_loss": 3.467944344206333,
+      "tokens_seen": 1937702912
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041694752046220516,
+      "loss": 1.1687,
+      "theoretical_loss": 3.467924825714643,
+      "tokens_seen": 1937833984
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004169073984914139,
+      "loss": 1.3418,
+      "theoretical_loss": 3.4679053089127345,
+      "tokens_seen": 1937965056
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004168672765206227,
+      "loss": 1.2661,
+      "theoretical_loss": 3.467885793800349,
+      "tokens_seen": 1938096128
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004168271545498315,
+      "loss": 1.374,
+      "theoretical_loss": 3.467866280377224,
+      "tokens_seen": 1938227200
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 652422,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3257834911346436,
+      "objective/train/theoretical_loss": 3.4678467686431,
+      "objective/train/tokens_used": 308817376,
+      "theoretical_loss": 3.4678467686431,
+      "tokens_seen": 1938358272
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004167870325790403,
+      "loss": 1.306,
+      "theoretical_loss": 3.4678467686431,
+      "tokens_seen": 1938358272
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004167469106082491,
+      "loss": 1.2281,
+      "theoretical_loss": 3.467827258597717,
+      "tokens_seen": 1938489344
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004167067886374579,
+      "loss": 1.2433,
+      "theoretical_loss": 3.4678077502408136,
+      "tokens_seen": 1938620416
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004166666666666667,
+      "loss": 1.2247,
+      "theoretical_loss": 3.46778824357213,
+      "tokens_seen": 1938751488
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041662654469587545,
+      "loss": 1.2749,
+      "theoretical_loss": 3.4677687385914067,
+      "tokens_seen": 1938882560
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004165864227250843,
+      "loss": 1.2404,
+      "theoretical_loss": 3.467749235298383,
+      "tokens_seen": 1939013632
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041654630075429306,
+      "loss": 1.3126,
+      "theoretical_loss": 3.467729733692798,
+      "tokens_seen": 1939144704
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004165061787835018,
+      "loss": 1.27,
+      "theoretical_loss": 3.467710233774393,
+      "tokens_seen": 1939275776
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041646605681271067,
+      "loss": 1.3321,
+      "theoretical_loss": 3.4676907355429067,
+      "tokens_seen": 1939406848
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004164259348419194,
+      "loss": 1.2845,
+      "theoretical_loss": 3.46767123899808,
+      "tokens_seen": 1939537920
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041638581287112823,
+      "loss": 1.3132,
+      "theoretical_loss": 3.467651744139654,
+      "tokens_seen": 1939668992
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041634569090033704,
+      "loss": 1.3445,
+      "theoretical_loss": 3.4676322509673665,
+      "tokens_seen": 1939800064
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041630556892954584,
+      "loss": 1.2834,
+      "theoretical_loss": 3.46761275948096,
+      "tokens_seen": 1939931136
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041626544695875465,
+      "loss": 1.243,
+      "theoretical_loss": 3.4675932696801732,
+      "tokens_seen": 1940062208
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004162253249879634,
+      "loss": 1.332,
+      "theoretical_loss": 3.467573781564747,
+      "tokens_seen": 1940193280
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004161852030171722,
+      "loss": 1.3705,
+      "theoretical_loss": 3.4675542951344225,
+      "tokens_seen": 1940324352
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000416145081046381,
+      "loss": 1.3298,
+      "theoretical_loss": 3.4675348103889387,
+      "tokens_seen": 1940455424
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004161049590755898,
+      "loss": 1.1867,
+      "theoretical_loss": 3.4675153273280377,
+      "tokens_seen": 1940586496
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041606483710479857,
+      "loss": 1.2456,
+      "theoretical_loss": 3.4674958459514595,
+      "tokens_seen": 1940717568
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041602471513400743,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4674763662589445,
+      "tokens_seen": 1940848640
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004159845931632162,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4674568882502337,
+      "tokens_seen": 1940979712
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041594447119242494,
+      "loss": 1.251,
+      "theoretical_loss": 3.4674374119250673,
+      "tokens_seen": 1941110784
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004159043492216338,
+      "loss": 1.2967,
+      "theoretical_loss": 3.4674179372831873,
+      "tokens_seen": 1941241856
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041586422725084255,
+      "loss": 1.2537,
+      "theoretical_loss": 3.467398464324333,
+      "tokens_seen": 1941372928
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041582410528005135,
+      "loss": 1.2443,
+      "theoretical_loss": 3.4673789930482464,
+      "tokens_seen": 1941504000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041578398330926016,
+      "loss": 1.3465,
+      "theoretical_loss": 3.4673595234546686,
+      "tokens_seen": 1941635072
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041574386133846897,
+      "loss": 1.1962,
+      "theoretical_loss": 3.46734005554334,
+      "tokens_seen": 1941766144
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004157037393676778,
+      "loss": 1.2323,
+      "theoretical_loss": 3.467320589314002,
+      "tokens_seen": 1941897216
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004156636173968865,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4673011247663963,
+      "tokens_seen": 1942028288
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041562349542609533,
+      "loss": 1.29,
+      "theoretical_loss": 3.4672816619002633,
+      "tokens_seen": 1942159360
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041558337345530414,
+      "loss": 1.2001,
+      "theoretical_loss": 3.4672622007153446,
+      "tokens_seen": 1942290432
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041554325148451294,
+      "loss": 1.3005,
+      "theoretical_loss": 3.467242741211381,
+      "tokens_seen": 1942421504
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004155031295137217,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4672232833881154,
+      "tokens_seen": 1942552576
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041546300754293056,
+      "loss": 1.3275,
+      "theoretical_loss": 3.467203827245287,
+      "tokens_seen": 1942683648
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004154228855721393,
+      "loss": 1.2447,
+      "theoretical_loss": 3.46718437278264,
+      "tokens_seen": 1942814720
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041538276360134806,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4671649199999135,
+      "tokens_seen": 1942945792
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004153426416305569,
+      "loss": 1.3684,
+      "theoretical_loss": 3.4671454688968506,
+      "tokens_seen": 1943076864
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004153025196597657,
+      "loss": 1.2106,
+      "theoretical_loss": 3.4671260194731923,
+      "tokens_seen": 1943207936
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004152623976889745,
+      "loss": 1.2602,
+      "theoretical_loss": 3.4671065717286806,
+      "tokens_seen": 1943339008
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004152222757181833,
+      "loss": 1.2677,
+      "theoretical_loss": 3.4670871256630575,
+      "tokens_seen": 1943470080
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004151821537473921,
+      "loss": 1.2646,
+      "theoretical_loss": 3.4670676812760646,
+      "tokens_seen": 1943601152
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041514203177660085,
+      "loss": 1.218,
+      "theoretical_loss": 3.4670482385674437,
+      "tokens_seen": 1943732224
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004151019098058097,
+      "loss": 1.2169,
+      "theoretical_loss": 3.467028797536937,
+      "tokens_seen": 1943863296
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041506178783501846,
+      "loss": 1.2343,
+      "theoretical_loss": 3.4670093581842867,
+      "tokens_seen": 1943994368
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041502166586422726,
+      "loss": 1.2169,
+      "theoretical_loss": 3.466989920509234,
+      "tokens_seen": 1944125440
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041498154389343607,
+      "loss": 1.1921,
+      "theoretical_loss": 3.4669704845115223,
+      "tokens_seen": 1944256512
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004149414219226448,
+      "loss": 1.1847,
+      "theoretical_loss": 3.4669510501908922,
+      "tokens_seen": 1944387584
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004149012999518537,
+      "loss": 1.2862,
+      "theoretical_loss": 3.466931617547088,
+      "tokens_seen": 1944518656
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041486117798106244,
+      "loss": 1.2391,
+      "theoretical_loss": 3.46691218657985,
+      "tokens_seen": 1944649728
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041482105601027124,
+      "loss": 1.2926,
+      "theoretical_loss": 3.466892757288922,
+      "tokens_seen": 1944780800
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/docs_used": 654435,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.20259428024292,
+      "objective/train/theoretical_loss": 3.466873329674046,
+      "objective/train/tokens_used": 315370976,
+      "theoretical_loss": 3.466873329674046,
+      "tokens_seen": 1944911872
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041478093403948005,
+      "loss": 1.2466,
+      "theoretical_loss": 3.466873329674046,
+      "tokens_seen": 1944911872
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004147408120686888,
+      "loss": 1.3374,
+      "theoretical_loss": 3.4668539037349637,
+      "tokens_seen": 1945042944
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004147006900978976,
+      "loss": 1.1613,
+      "theoretical_loss": 3.4668344794714185,
+      "tokens_seen": 1945174016
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004146605681271064,
+      "loss": 1.2649,
+      "theoretical_loss": 3.466815056883153,
+      "tokens_seen": 1945305088
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004146204461563152,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4667956359699095,
+      "tokens_seen": 1945436160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041458032418552397,
+      "loss": 1.2538,
+      "theoretical_loss": 3.466776216731431,
+      "tokens_seen": 1945567232
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041454020221473283,
+      "loss": 1.3552,
+      "theoretical_loss": 3.4667567991674595,
+      "tokens_seen": 1945698304
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004145000802439416,
+      "loss": 1.3154,
+      "theoretical_loss": 3.4667373832777395,
+      "tokens_seen": 1945829376
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041445995827315034,
+      "loss": 1.1976,
+      "theoretical_loss": 3.466717969062012,
+      "tokens_seen": 1945960448
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004144198363023592,
+      "loss": 1.2327,
+      "theoretical_loss": 3.4666985565200212,
+      "tokens_seen": 1946091520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041437971433156795,
+      "loss": 1.2287,
+      "theoretical_loss": 3.466679145651509,
+      "tokens_seen": 1946222592
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004143395923607768,
+      "loss": 1.2871,
+      "theoretical_loss": 3.4666597364562195,
+      "tokens_seen": 1946353664
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041429947038998556,
+      "loss": 1.2125,
+      "theoretical_loss": 3.466640328933895,
+      "tokens_seen": 1946484736
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041425934841919437,
+      "loss": 1.2367,
+      "theoretical_loss": 3.4666209230842795,
+      "tokens_seen": 1946615808
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004142192264484032,
+      "loss": 1.2916,
+      "theoretical_loss": 3.4666015189071158,
+      "tokens_seen": 1946746880
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004141791044776119,
+      "loss": 1.3165,
+      "theoretical_loss": 3.466582116402147,
+      "tokens_seen": 1946877952
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041413898250682073,
+      "loss": 1.2393,
+      "theoretical_loss": 3.4665627155691165,
+      "tokens_seen": 1947009024
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041409886053602954,
+      "loss": 1.3005,
+      "theoretical_loss": 3.4665433164077673,
+      "tokens_seen": 1947140096
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041405873856523834,
+      "loss": 1.295,
+      "theoretical_loss": 3.4665239189178436,
+      "tokens_seen": 1947271168
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004140186165944471,
+      "loss": 1.2625,
+      "theoretical_loss": 3.466504523099089,
+      "tokens_seen": 1947402240
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041397849462365596,
+      "loss": 1.314,
+      "theoretical_loss": 3.466485128951246,
+      "tokens_seen": 1947533312
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004139383726528647,
+      "loss": 1.1485,
+      "theoretical_loss": 3.4664657364740594,
+      "tokens_seen": 1947664384
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041389825068207346,
+      "loss": 1.2128,
+      "theoretical_loss": 3.466446345667272,
+      "tokens_seen": 1947795456
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004138581287112823,
+      "loss": 1.2929,
+      "theoretical_loss": 3.4664269565306283,
+      "tokens_seen": 1947926528
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004138180067404911,
+      "loss": 1.2843,
+      "theoretical_loss": 3.466407569063871,
+      "tokens_seen": 1948057600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041377788476969993,
+      "loss": 1.292,
+      "theoretical_loss": 3.466388183266745,
+      "tokens_seen": 1948188672
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004137377627989087,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4663687991389933,
+      "tokens_seen": 1948319744
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004136976408281175,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4663494166803606,
+      "tokens_seen": 1948450816
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004136575188573263,
+      "loss": 1.3015,
+      "theoretical_loss": 3.466330035890591,
+      "tokens_seen": 1948581888
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004136173968865351,
+      "loss": 1.3777,
+      "theoretical_loss": 3.4663106567694277,
+      "tokens_seen": 1948712960
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041357727491574386,
+      "loss": 1.3042,
+      "theoretical_loss": 3.4662912793166156,
+      "tokens_seen": 1948844032
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041353715294495266,
+      "loss": 1.3329,
+      "theoretical_loss": 3.4662719035318985,
+      "tokens_seen": 1948975104
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041349703097416147,
+      "loss": 1.2896,
+      "theoretical_loss": 3.466252529415021,
+      "tokens_seen": 1949106176
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004134569090033702,
+      "loss": 1.3137,
+      "theoretical_loss": 3.4662331569657265,
+      "tokens_seen": 1949237248
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004134167870325791,
+      "loss": 1.3343,
+      "theoretical_loss": 3.4662137861837605,
+      "tokens_seen": 1949368320
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041337666506178783,
+      "loss": 1.2307,
+      "theoretical_loss": 3.4661944170688668,
+      "tokens_seen": 1949499392
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041333654309099664,
+      "loss": 1.3599,
+      "theoretical_loss": 3.46617504962079,
+      "tokens_seen": 1949630464
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041329642112020545,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4661556838392746,
+      "tokens_seen": 1949761536
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004132562991494142,
+      "loss": 1.2366,
+      "theoretical_loss": 3.4661363197240647,
+      "tokens_seen": 1949892608
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000413216177178623,
+      "loss": 1.3103,
+      "theoretical_loss": 3.4661169572749055,
+      "tokens_seen": 1950023680
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004131760552078318,
+      "loss": 1.2987,
+      "theoretical_loss": 3.466097596491542,
+      "tokens_seen": 1950154752
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004131359332370406,
+      "loss": 1.2629,
+      "theoretical_loss": 3.466078237373718,
+      "tokens_seen": 1950285824
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004130958112662494,
+      "loss": 1.2107,
+      "theoretical_loss": 3.466058879921179,
+      "tokens_seen": 1950416896
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041305568929545823,
+      "loss": 1.2279,
+      "theoretical_loss": 3.4660395241336697,
+      "tokens_seen": 1950547968
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000413015567324667,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4660201700109345,
+      "tokens_seen": 1950679040
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004129754453538758,
+      "loss": 1.2927,
+      "theoretical_loss": 3.4660008175527195,
+      "tokens_seen": 1950810112
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004129353233830846,
+      "loss": 1.2148,
+      "theoretical_loss": 3.4659814667587687,
+      "tokens_seen": 1950941184
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041289520141229335,
+      "loss": 1.2687,
+      "theoretical_loss": 3.4659621176288278,
+      "tokens_seen": 1951072256
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004128550794415022,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4659427701626413,
+      "tokens_seen": 1951203328
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041281495747071096,
+      "loss": 1.2611,
+      "theoretical_loss": 3.465923424359955,
+      "tokens_seen": 1951334400
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/docs_used": 656717,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1931488513946533,
+      "objective/train/theoretical_loss": 3.465904080220514,
+      "objective/train/tokens_used": 321924576,
+      "theoretical_loss": 3.465904080220514,
+      "tokens_seen": 1951465472
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041277483549991977,
+      "loss": 1.2371,
+      "theoretical_loss": 3.465904080220514,
+      "tokens_seen": 1951465472
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004127347135291286,
+      "loss": 1.2106,
+      "theoretical_loss": 3.465884737744063,
+      "tokens_seen": 1951596544
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004126945915583373,
+      "loss": 1.3026,
+      "theoretical_loss": 3.465865396930348,
+      "tokens_seen": 1951727616
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041265446958754613,
+      "loss": 1.2422,
+      "theoretical_loss": 3.465846057779115,
+      "tokens_seen": 1951858688
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041261434761675494,
+      "loss": 1.2876,
+      "theoretical_loss": 3.465826720290108,
+      "tokens_seen": 1951989760
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041257422564596374,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4658073844630732,
+      "tokens_seen": 1952120832
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004125341036751725,
+      "loss": 1.2289,
+      "theoretical_loss": 3.465788050297757,
+      "tokens_seen": 1952251904
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041249398170438136,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4657687177939036,
+      "tokens_seen": 1952382976
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004124538597335901,
+      "loss": 1.4118,
+      "theoretical_loss": 3.4657493869512597,
+      "tokens_seen": 1952514048
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004124137377627989,
+      "loss": 1.2423,
+      "theoretical_loss": 3.465730057769571,
+      "tokens_seen": 1952645120
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004123736157920077,
+      "loss": 1.2387,
+      "theoretical_loss": 3.465710730248583,
+      "tokens_seen": 1952776192
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004123334938212165,
+      "loss": 1.1998,
+      "theoretical_loss": 3.465691404388041,
+      "tokens_seen": 1952907264
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041229337185042533,
+      "loss": 1.2565,
+      "theoretical_loss": 3.465672080187692,
+      "tokens_seen": 1953038336
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004122532498796341,
+      "loss": 1.4095,
+      "theoretical_loss": 3.4656527576472818,
+      "tokens_seen": 1953169408
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004122131279088429,
+      "loss": 1.257,
+      "theoretical_loss": 3.4656334367665558,
+      "tokens_seen": 1953300480
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004121730059380517,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4656141175452606,
+      "tokens_seen": 1953431552
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004121328839672605,
+      "loss": 1.248,
+      "theoretical_loss": 3.4655947999831422,
+      "tokens_seen": 1953562624
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041209276199646926,
+      "loss": 1.3446,
+      "theoretical_loss": 3.465575484079947,
+      "tokens_seen": 1953693696
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041205264002567806,
+      "loss": 1.2894,
+      "theoretical_loss": 3.4655561698354207,
+      "tokens_seen": 1953824768
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041201251805488687,
+      "loss": 1.2596,
+      "theoretical_loss": 3.46553685724931,
+      "tokens_seen": 1953955840
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004119723960840956,
+      "loss": 1.2028,
+      "theoretical_loss": 3.4655175463213617,
+      "tokens_seen": 1954086912
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004119322741133045,
+      "loss": 1.3464,
+      "theoretical_loss": 3.465498237051321,
+      "tokens_seen": 1954217984
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041189215214251323,
+      "loss": 1.206,
+      "theoretical_loss": 3.4654789294389357,
+      "tokens_seen": 1954349056
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004118520301717221,
+      "loss": 1.2917,
+      "theoretical_loss": 3.4654596234839516,
+      "tokens_seen": 1954480128
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041181190820093085,
+      "loss": 1.2773,
+      "theoretical_loss": 3.465440319186115,
+      "tokens_seen": 1954611200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004117717862301396,
+      "loss": 1.2671,
+      "theoretical_loss": 3.4654210165451738,
+      "tokens_seen": 1954742272
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041173166425934846,
+      "loss": 1.2225,
+      "theoretical_loss": 3.4654017155608736,
+      "tokens_seen": 1954873344
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004116915422885572,
+      "loss": 1.2878,
+      "theoretical_loss": 3.4653824162329614,
+      "tokens_seen": 1955004416
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000411651420317766,
+      "loss": 1.2498,
+      "theoretical_loss": 3.465363118561184,
+      "tokens_seen": 1955135488
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004116112983469748,
+      "loss": 1.2052,
+      "theoretical_loss": 3.4653438225452886,
+      "tokens_seen": 1955266560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041157117637618363,
+      "loss": 1.1849,
+      "theoretical_loss": 3.465324528185022,
+      "tokens_seen": 1955397632
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004115310544053924,
+      "loss": 1.2744,
+      "theoretical_loss": 3.4653052354801304,
+      "tokens_seen": 1955528704
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004114909324346012,
+      "loss": 1.145,
+      "theoretical_loss": 3.465285944430362,
+      "tokens_seen": 1955659776
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041145081046381,
+      "loss": 1.2778,
+      "theoretical_loss": 3.465266655035463,
+      "tokens_seen": 1955790848
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041141068849301875,
+      "loss": 1.2806,
+      "theoretical_loss": 3.4652473672951816,
+      "tokens_seen": 1955921920
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004113705665222276,
+      "loss": 1.2384,
+      "theoretical_loss": 3.465228081209264,
+      "tokens_seen": 1956052992
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041133044455143636,
+      "loss": 1.2554,
+      "theoretical_loss": 3.4652087967774574,
+      "tokens_seen": 1956184064
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041129032258064517,
+      "loss": 1.3051,
+      "theoretical_loss": 3.46518951399951,
+      "tokens_seen": 1956315136
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041125020060985397,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4651702328751686,
+      "tokens_seen": 1956446208
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004112100786390627,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4651509534041804,
+      "tokens_seen": 1956577280
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004111699566682716,
+      "loss": 1.3102,
+      "theoretical_loss": 3.4651316755862935,
+      "tokens_seen": 1956708352
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041112983469748034,
+      "loss": 1.292,
+      "theoretical_loss": 3.4651123994212547,
+      "tokens_seen": 1956839424
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041108971272668914,
+      "loss": 1.3562,
+      "theoretical_loss": 3.4650931249088126,
+      "tokens_seen": 1956970496
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041104959075589795,
+      "loss": 1.2664,
+      "theoretical_loss": 3.465073852048714,
+      "tokens_seen": 1957101568
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041100946878510676,
+      "loss": 1.355,
+      "theoretical_loss": 3.4650545808407065,
+      "tokens_seen": 1957232640
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004109693468143155,
+      "loss": 1.1833,
+      "theoretical_loss": 3.465035311284539,
+      "tokens_seen": 1957363712
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004109292248435243,
+      "loss": 1.2476,
+      "theoretical_loss": 3.4650160433799577,
+      "tokens_seen": 1957494784
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004108891028727331,
+      "loss": 1.2903,
+      "theoretical_loss": 3.4649967771267116,
+      "tokens_seen": 1957625856
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004108489809019419,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4649775125245483,
+      "tokens_seen": 1957756928
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041080885893115073,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4649582495732165,
+      "tokens_seen": 1957888000
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 658749,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2225360870361328,
+      "objective/train/theoretical_loss": 3.4649389882724626,
+      "objective/train/tokens_used": 328478176,
+      "theoretical_loss": 3.4649389882724626,
+      "tokens_seen": 1958019072
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004107687369603595,
+      "loss": 1.2578,
+      "theoretical_loss": 3.4649389882724626,
+      "tokens_seen": 1958019072
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004107286149895683,
+      "loss": 1.2702,
+      "theoretical_loss": 3.4649197286220357,
+      "tokens_seen": 1958150144
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004106884930187771,
+      "loss": 1.3221,
+      "theoretical_loss": 3.4649004706216844,
+      "tokens_seen": 1958281216
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041064837104798585,
+      "loss": 1.3537,
+      "theoretical_loss": 3.4648812142711565,
+      "tokens_seen": 1958412288
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004106082490771947,
+      "loss": 1.2577,
+      "theoretical_loss": 3.4648619595701993,
+      "tokens_seen": 1958543360
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041056812710640346,
+      "loss": 1.3088,
+      "theoretical_loss": 3.4648427065185627,
+      "tokens_seen": 1958674432
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041052800513561227,
+      "loss": 1.2419,
+      "theoretical_loss": 3.4648234551159938,
+      "tokens_seen": 1958805504
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004104878831648211,
+      "loss": 1.2524,
+      "theoretical_loss": 3.464804205362242,
+      "tokens_seen": 1958936576
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004104477611940299,
+      "loss": 1.2578,
+      "theoretical_loss": 3.464784957257055,
+      "tokens_seen": 1959067648
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041040763922323863,
+      "loss": 1.2936,
+      "theoretical_loss": 3.4647657108001813,
+      "tokens_seen": 1959198720
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004103675172524475,
+      "loss": 1.2698,
+      "theoretical_loss": 3.46474646599137,
+      "tokens_seen": 1959329792
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041032739528165625,
+      "loss": 1.2251,
+      "theoretical_loss": 3.46472722283037,
+      "tokens_seen": 1959460864
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000410287273310865,
+      "loss": 1.288,
+      "theoretical_loss": 3.464707981316929,
+      "tokens_seen": 1959591936
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041024715134007386,
+      "loss": 1.253,
+      "theoretical_loss": 3.4646887414507965,
+      "tokens_seen": 1959723008
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004102070293692826,
+      "loss": 1.2195,
+      "theoretical_loss": 3.4646695032317214,
+      "tokens_seen": 1959854080
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004101669073984914,
+      "loss": 1.2991,
+      "theoretical_loss": 3.4646502666594516,
+      "tokens_seen": 1959985152
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004101267854277002,
+      "loss": 1.2371,
+      "theoretical_loss": 3.464631031733737,
+      "tokens_seen": 1960116224
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041008666345690903,
+      "loss": 1.215,
+      "theoretical_loss": 3.464611798454327,
+      "tokens_seen": 1960247296
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004100465414861178,
+      "loss": 1.2961,
+      "theoretical_loss": 3.4645925668209694,
+      "tokens_seen": 1960378368
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004100064195153266,
+      "loss": 1.3492,
+      "theoretical_loss": 3.4645733368334137,
+      "tokens_seen": 1960509440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004099662975445354,
+      "loss": 1.2946,
+      "theoretical_loss": 3.4645541084914093,
+      "tokens_seen": 1960640512
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004099261755737442,
+      "loss": 1.3028,
+      "theoretical_loss": 3.4645348817947053,
+      "tokens_seen": 1960771584
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000409886053602953,
+      "loss": 1.256,
+      "theoretical_loss": 3.4645156567430506,
+      "tokens_seen": 1960902656
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040984593163216176,
+      "loss": 1.257,
+      "theoretical_loss": 3.464496433336195,
+      "tokens_seen": 1961033728
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004098058096613706,
+      "loss": 1.2759,
+      "theoretical_loss": 3.464477211573887,
+      "tokens_seen": 1961164800
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040976568769057937,
+      "loss": 1.2606,
+      "theoretical_loss": 3.464457991455877,
+      "tokens_seen": 1961295872
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004097255657197881,
+      "loss": 1.2318,
+      "theoretical_loss": 3.4644387729819144,
+      "tokens_seen": 1961426944
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000409685443748997,
+      "loss": 1.2693,
+      "theoretical_loss": 3.464419556151748,
+      "tokens_seen": 1961558016
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040964532177820574,
+      "loss": 1.268,
+      "theoretical_loss": 3.4644003409651285,
+      "tokens_seen": 1961689088
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040960519980741454,
+      "loss": 1.2889,
+      "theoretical_loss": 3.464381127421804,
+      "tokens_seen": 1961820160
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040956507783662335,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4643619155215255,
+      "tokens_seen": 1961951232
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040952495586583216,
+      "loss": 1.2995,
+      "theoretical_loss": 3.464342705264042,
+      "tokens_seen": 1962082304
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004094848338950409,
+      "loss": 1.2664,
+      "theoretical_loss": 3.464323496649104,
+      "tokens_seen": 1962213376
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004094447119242497,
+      "loss": 1.2723,
+      "theoretical_loss": 3.464304289676461,
+      "tokens_seen": 1962344448
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004094045899534585,
+      "loss": 1.2295,
+      "theoretical_loss": 3.464285084345862,
+      "tokens_seen": 1962475520
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040936446798266727,
+      "loss": 1.2141,
+      "theoretical_loss": 3.4642658806570585,
+      "tokens_seen": 1962606592
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040932434601187613,
+      "loss": 1.2946,
+      "theoretical_loss": 3.4642466786097996,
+      "tokens_seen": 1962737664
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004092842240410849,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4642274782038354,
+      "tokens_seen": 1962868736
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040924410207029375,
+      "loss": 1.2803,
+      "theoretical_loss": 3.464208279438916,
+      "tokens_seen": 1962999808
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004092039800995025,
+      "loss": 1.2702,
+      "theoretical_loss": 3.4641890823147925,
+      "tokens_seen": 1963130880
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040916385812871125,
+      "loss": 1.301,
+      "theoretical_loss": 3.464169886831214,
+      "tokens_seen": 1963261952
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004091237361579201,
+      "loss": 1.3358,
+      "theoretical_loss": 3.464150692987931,
+      "tokens_seen": 1963393024
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040908361418712886,
+      "loss": 1.2697,
+      "theoretical_loss": 3.4641315007846947,
+      "tokens_seen": 1963524096
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040904349221633767,
+      "loss": 1.3342,
+      "theoretical_loss": 3.464112310221254,
+      "tokens_seen": 1963655168
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004090033702455465,
+      "loss": 1.212,
+      "theoretical_loss": 3.4640931212973607,
+      "tokens_seen": 1963786240
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004089632482747553,
+      "loss": 1.2426,
+      "theoretical_loss": 3.464073934012765,
+      "tokens_seen": 1963917312
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040892312630396403,
+      "loss": 1.2561,
+      "theoretical_loss": 3.464054748367217,
+      "tokens_seen": 1964048384
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004088830043331729,
+      "loss": 1.3638,
+      "theoretical_loss": 3.4640355643604677,
+      "tokens_seen": 1964179456
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040884288236238165,
+      "loss": 1.2729,
+      "theoretical_loss": 3.464016381992267,
+      "tokens_seen": 1964310528
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004088027603915904,
+      "loss": 1.2187,
+      "theoretical_loss": 3.463997201262367,
+      "tokens_seen": 1964441600
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 660816,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3220582008361816,
+      "objective/train/theoretical_loss": 3.463978022170518,
+      "objective/train/tokens_used": 335031776,
+      "theoretical_loss": 3.463978022170518,
+      "tokens_seen": 1964572672
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040876263842079926,
+      "loss": 1.2352,
+      "theoretical_loss": 3.463978022170518,
+      "tokens_seen": 1964572672
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000408722516450008,
+      "loss": 1.2548,
+      "theoretical_loss": 3.46395884471647,
+      "tokens_seen": 1964703744
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040868239447921687,
+      "loss": 1.218,
+      "theoretical_loss": 3.463939668899975,
+      "tokens_seen": 1964834816
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004086422725084256,
+      "loss": 1.3536,
+      "theoretical_loss": 3.463920494720783,
+      "tokens_seen": 1964965888
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040860215053763443,
+      "loss": 1.256,
+      "theoretical_loss": 3.4639013221786454,
+      "tokens_seen": 1965096960
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040856202856684324,
+      "loss": 1.2803,
+      "theoretical_loss": 3.463882151273314,
+      "tokens_seen": 1965228032
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000408521906596052,
+      "loss": 1.3184,
+      "theoretical_loss": 3.4638629820045383,
+      "tokens_seen": 1965359104
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004084817846252608,
+      "loss": 1.2732,
+      "theoretical_loss": 3.463843814372071,
+      "tokens_seen": 1965490176
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004084416626544696,
+      "loss": 1.2652,
+      "theoretical_loss": 3.4638246483756623,
+      "tokens_seen": 1965621248
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004084015406836784,
+      "loss": 1.3271,
+      "theoretical_loss": 3.4638054840150643,
+      "tokens_seen": 1965752320
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040836141871288716,
+      "loss": 1.2681,
+      "theoretical_loss": 3.4637863212900273,
+      "tokens_seen": 1965883392
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000408321296742096,
+      "loss": 1.2591,
+      "theoretical_loss": 3.4637671602003035,
+      "tokens_seen": 1966014464
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040828117477130477,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4637480007456443,
+      "tokens_seen": 1966145536
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004082410528005135,
+      "loss": 1.19,
+      "theoretical_loss": 3.463728842925801,
+      "tokens_seen": 1966276608
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004082009308297224,
+      "loss": 1.2914,
+      "theoretical_loss": 3.4637096867405246,
+      "tokens_seen": 1966407680
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040816080885893114,
+      "loss": 1.3065,
+      "theoretical_loss": 3.4636905321895677,
+      "tokens_seen": 1966538752
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040812068688813994,
+      "loss": 1.2973,
+      "theoretical_loss": 3.4636713792726814,
+      "tokens_seen": 1966669824
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040808056491734875,
+      "loss": 1.2826,
+      "theoretical_loss": 3.463652227989618,
+      "tokens_seen": 1966800896
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040804044294655756,
+      "loss": 1.2996,
+      "theoretical_loss": 3.463633078340128,
+      "tokens_seen": 1966931968
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040800032097576636,
+      "loss": 1.3576,
+      "theoretical_loss": 3.463613930323964,
+      "tokens_seen": 1967063040
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004079601990049751,
+      "loss": 1.2358,
+      "theoretical_loss": 3.463594783940878,
+      "tokens_seen": 1967194112
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004079200770341839,
+      "loss": 1.3321,
+      "theoretical_loss": 3.463575639190622,
+      "tokens_seen": 1967325184
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004078799550633927,
+      "loss": 1.2143,
+      "theoretical_loss": 3.4635564960729472,
+      "tokens_seen": 1967456256
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040783983309260153,
+      "loss": 1.3044,
+      "theoretical_loss": 3.463537354587607,
+      "tokens_seen": 1967587328
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004077997111218103,
+      "loss": 1.3029,
+      "theoretical_loss": 3.4635182147343517,
+      "tokens_seen": 1967718400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040775958915101915,
+      "loss": 1.2925,
+      "theoretical_loss": 3.4634990765129348,
+      "tokens_seen": 1967849472
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004077194671802279,
+      "loss": 1.318,
+      "theoretical_loss": 3.463479939923108,
+      "tokens_seen": 1967980544
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040767934520943665,
+      "loss": 1.2101,
+      "theoretical_loss": 3.463460804964624,
+      "tokens_seen": 1968111616
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004076392232386455,
+      "loss": 1.2739,
+      "theoretical_loss": 3.4634416716372343,
+      "tokens_seen": 1968242688
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040759910126785426,
+      "loss": 1.2767,
+      "theoretical_loss": 3.4634225399406917,
+      "tokens_seen": 1968373760
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040755897929706307,
+      "loss": 1.2856,
+      "theoretical_loss": 3.463403409874749,
+      "tokens_seen": 1968504832
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004075188573262719,
+      "loss": 1.2338,
+      "theoretical_loss": 3.4633842814391578,
+      "tokens_seen": 1968635904
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004074787353554807,
+      "loss": 1.2472,
+      "theoretical_loss": 3.4633651546336712,
+      "tokens_seen": 1968766976
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004074386133846895,
+      "loss": 1.2956,
+      "theoretical_loss": 3.463346029458042,
+      "tokens_seen": 1968898048
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004073984914138983,
+      "loss": 1.2889,
+      "theoretical_loss": 3.463326905912022,
+      "tokens_seen": 1969029120
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040735836944310705,
+      "loss": 1.2382,
+      "theoretical_loss": 3.463307783995365,
+      "tokens_seen": 1969160192
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040731824747231585,
+      "loss": 1.3312,
+      "theoretical_loss": 3.4632886637078224,
+      "tokens_seen": 1969291264
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040727812550152466,
+      "loss": 1.2859,
+      "theoretical_loss": 3.463269545049148,
+      "tokens_seen": 1969422336
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004072380035307334,
+      "loss": 1.1796,
+      "theoretical_loss": 3.4632504280190943,
+      "tokens_seen": 1969553408
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040719788155994227,
+      "loss": 1.2695,
+      "theoretical_loss": 3.463231312617414,
+      "tokens_seen": 1969684480
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000407157759589151,
+      "loss": 1.2754,
+      "theoretical_loss": 3.463212198843861,
+      "tokens_seen": 1969815552
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040711763761835983,
+      "loss": 1.2182,
+      "theoretical_loss": 3.463193086698187,
+      "tokens_seen": 1969946624
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040707751564756864,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4631739761801454,
+      "tokens_seen": 1970077696
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004070373936767774,
+      "loss": 1.4053,
+      "theoretical_loss": 3.4631548672894903,
+      "tokens_seen": 1970208768
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004069972717059862,
+      "loss": 1.2478,
+      "theoretical_loss": 3.463135760025973,
+      "tokens_seen": 1970339840
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000406957149735195,
+      "loss": 1.2948,
+      "theoretical_loss": 3.4631166543893492,
+      "tokens_seen": 1970470912
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004069170277644038,
+      "loss": 1.3322,
+      "theoretical_loss": 3.46309755037937,
+      "tokens_seen": 1970601984
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040687690579361256,
+      "loss": 1.2199,
+      "theoretical_loss": 3.4630784479957897,
+      "tokens_seen": 1970733056
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004068367838228214,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4630593472383615,
+      "tokens_seen": 1970864128
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040679666185203017,
+      "loss": 1.2558,
+      "theoretical_loss": 3.463040248106839,
+      "tokens_seen": 1970995200
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 662847,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.30068838596344,
+      "objective/train/theoretical_loss": 3.4630211506009756,
+      "objective/train/tokens_used": 341585376,
+      "theoretical_loss": 3.4630211506009756,
+      "tokens_seen": 1971126272
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000406756539881239,
+      "loss": 1.2857,
+      "theoretical_loss": 3.4630211506009756,
+      "tokens_seen": 1971126272
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004067164179104478,
+      "loss": 1.3721,
+      "theoretical_loss": 3.4630020547205245,
+      "tokens_seen": 1971257344
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040667629593965654,
+      "loss": 1.2946,
+      "theoretical_loss": 3.46298296046524,
+      "tokens_seen": 1971388416
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004066361739688654,
+      "loss": 1.3207,
+      "theoretical_loss": 3.462963867834875,
+      "tokens_seen": 1971519488
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040659605199807415,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4629447768291834,
+      "tokens_seen": 1971650560
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040655593002728296,
+      "loss": 1.3111,
+      "theoretical_loss": 3.4629256874479193,
+      "tokens_seen": 1971781632
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040651580805649176,
+      "loss": 1.2995,
+      "theoretical_loss": 3.4629065996908364,
+      "tokens_seen": 1971912704
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004064756860857005,
+      "loss": 1.2542,
+      "theoretical_loss": 3.462887513557688,
+      "tokens_seen": 1972043776
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004064355641149093,
+      "loss": 1.2586,
+      "theoretical_loss": 3.4628684290482292,
+      "tokens_seen": 1972174848
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004063954421441181,
+      "loss": 1.3374,
+      "theoretical_loss": 3.462849346162213,
+      "tokens_seen": 1972305920
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040635532017332693,
+      "loss": 1.2565,
+      "theoretical_loss": 3.4628302648993934,
+      "tokens_seen": 1972436992
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004063151982025357,
+      "loss": 1.215,
+      "theoretical_loss": 3.4628111852595254,
+      "tokens_seen": 1972568064
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040627507623174455,
+      "loss": 1.2475,
+      "theoretical_loss": 3.462792107242362,
+      "tokens_seen": 1972699136
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004062349542609533,
+      "loss": 1.2257,
+      "theoretical_loss": 3.462773030847658,
+      "tokens_seen": 1972830208
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040619483229016205,
+      "loss": 1.221,
+      "theoretical_loss": 3.4627539560751677,
+      "tokens_seen": 1972961280
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004061547103193709,
+      "loss": 1.2746,
+      "theoretical_loss": 3.4627348829246447,
+      "tokens_seen": 1973092352
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040611458834857966,
+      "loss": 1.2275,
+      "theoretical_loss": 3.4627158113958445,
+      "tokens_seen": 1973223424
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004060744663777885,
+      "loss": 1.3715,
+      "theoretical_loss": 3.4626967414885206,
+      "tokens_seen": 1973354496
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004060343444069973,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4626776732024283,
+      "tokens_seen": 1973485568
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004059942224362061,
+      "loss": 1.2829,
+      "theoretical_loss": 3.4626586065373206,
+      "tokens_seen": 1973616640
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004059541004654149,
+      "loss": 1.3163,
+      "theoretical_loss": 3.4626395414929534,
+      "tokens_seen": 1973747712
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040591397849462364,
+      "loss": 1.2807,
+      "theoretical_loss": 3.462620478069081,
+      "tokens_seen": 1973878784
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040587385652383245,
+      "loss": 1.252,
+      "theoretical_loss": 3.462601416265458,
+      "tokens_seen": 1974009856
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040583373455304125,
+      "loss": 1.2743,
+      "theoretical_loss": 3.462582356081839,
+      "tokens_seen": 1974140928
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040579361258225006,
+      "loss": 1.3131,
+      "theoretical_loss": 3.4625632975179785,
+      "tokens_seen": 1974272000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004057534906114588,
+      "loss": 1.2615,
+      "theoretical_loss": 3.4625442405736324,
+      "tokens_seen": 1974403072
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040571336864066767,
+      "loss": 1.2982,
+      "theoretical_loss": 3.4625251852485546,
+      "tokens_seen": 1974534144
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004056732466698764,
+      "loss": 1.2366,
+      "theoretical_loss": 3.4625061315425008,
+      "tokens_seen": 1974665216
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040563312469908523,
+      "loss": 1.298,
+      "theoretical_loss": 3.462487079455225,
+      "tokens_seen": 1974796288
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040559300272829404,
+      "loss": 1.2949,
+      "theoretical_loss": 3.4624680289864824,
+      "tokens_seen": 1974927360
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004055528807575028,
+      "loss": 1.3161,
+      "theoretical_loss": 3.4624489801360294,
+      "tokens_seen": 1975058432
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040551275878671165,
+      "loss": 1.3841,
+      "theoretical_loss": 3.4624299329036194,
+      "tokens_seen": 1975189504
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004054726368159204,
+      "loss": 1.2089,
+      "theoretical_loss": 3.4624108872890087,
+      "tokens_seen": 1975320576
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004054325148451292,
+      "loss": 1.3336,
+      "theoretical_loss": 3.462391843291952,
+      "tokens_seen": 1975451648
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.000405392392874338,
+      "loss": 1.3389,
+      "theoretical_loss": 3.4623728009122052,
+      "tokens_seen": 1975582720
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004053522709035468,
+      "loss": 1.3242,
+      "theoretical_loss": 3.462353760149523,
+      "tokens_seen": 1975713792
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040531214893275557,
+      "loss": 1.1941,
+      "theoretical_loss": 3.4623347210036615,
+      "tokens_seen": 1975844864
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004052720269619644,
+      "loss": 1.2251,
+      "theoretical_loss": 3.462315683474376,
+      "tokens_seen": 1975975936
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004052319049911732,
+      "loss": 1.3319,
+      "theoretical_loss": 3.4622966475614216,
+      "tokens_seen": 1976107008
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040519178302038194,
+      "loss": 1.2273,
+      "theoretical_loss": 3.462277613264554,
+      "tokens_seen": 1976238080
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004051516610495908,
+      "loss": 1.2931,
+      "theoretical_loss": 3.462258580583529,
+      "tokens_seen": 1976369152
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040511153907879955,
+      "loss": 1.2831,
+      "theoretical_loss": 3.462239549518102,
+      "tokens_seen": 1976500224
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040507141710800835,
+      "loss": 1.2563,
+      "theoretical_loss": 3.462220520068029,
+      "tokens_seen": 1976631296
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040503129513721716,
+      "loss": 1.3671,
+      "theoretical_loss": 3.462201492233066,
+      "tokens_seen": 1976762368
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004049911731664259,
+      "loss": 1.2223,
+      "theoretical_loss": 3.462182466012969,
+      "tokens_seen": 1976893440
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004049510511956347,
+      "loss": 1.2591,
+      "theoretical_loss": 3.4621634414074927,
+      "tokens_seen": 1977024512
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004049109292248435,
+      "loss": 1.3478,
+      "theoretical_loss": 3.4621444184163943,
+      "tokens_seen": 1977155584
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040487080725405233,
+      "loss": 1.2953,
+      "theoretical_loss": 3.462125397039429,
+      "tokens_seen": 1977286656
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040483068528326114,
+      "loss": 1.3297,
+      "theoretical_loss": 3.4621063772763527,
+      "tokens_seen": 1977417728
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040479056331246994,
+      "loss": 1.2759,
+      "theoretical_loss": 3.462087359126923,
+      "tokens_seen": 1977548800
+    },
+    {
+      "epoch": 0.2,
+      "objective/train/docs_used": 664930,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2153178453445435,
+      "objective/train/theoretical_loss": 3.4620683425908947,
+      "objective/train/tokens_used": 348138976,
+      "theoretical_loss": 3.4620683425908947,
+      "tokens_seen": 1977679872
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004047504413416787,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4620683425908947,
+      "tokens_seen": 1977679872
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004047103193708875,
+      "loss": 1.2375,
+      "theoretical_loss": 3.4620493276680238,
+      "tokens_seen": 1977810944
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004046701974000963,
+      "loss": 1.2854,
+      "theoretical_loss": 3.462030314358068,
+      "tokens_seen": 1977942016
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040463007542930506,
+      "loss": 1.3488,
+      "theoretical_loss": 3.462011302660782,
+      "tokens_seen": 1978073088
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004045899534585139,
+      "loss": 1.2996,
+      "theoretical_loss": 3.461992292575923,
+      "tokens_seen": 1978204160
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004045498314877227,
+      "loss": 1.2423,
+      "theoretical_loss": 3.461973284103248,
+      "tokens_seen": 1978335232
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004045097095169315,
+      "loss": 1.2839,
+      "theoretical_loss": 3.461954277242512,
+      "tokens_seen": 1978466304
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004044695875461403,
+      "loss": 1.2452,
+      "theoretical_loss": 3.4619352719934726,
+      "tokens_seen": 1978597376
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040442946557534904,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4619162683558864,
+      "tokens_seen": 1978728448
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040438934360455785,
+      "loss": 1.2811,
+      "theoretical_loss": 3.461897266329509,
+      "tokens_seen": 1978859520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040434922163376665,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4618782659140983,
+      "tokens_seen": 1978990592
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040430909966297546,
+      "loss": 1.255,
+      "theoretical_loss": 3.461859267109411,
+      "tokens_seen": 1979121664
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004042689776921842,
+      "loss": 1.2539,
+      "theoretical_loss": 3.461840269915203,
+      "tokens_seen": 1979252736
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040422885572139307,
+      "loss": 1.2524,
+      "theoretical_loss": 3.461821274331232,
+      "tokens_seen": 1979383808
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004041887337506018,
+      "loss": 1.274,
+      "theoretical_loss": 3.461802280357254,
+      "tokens_seen": 1979514880
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004041486117798107,
+      "loss": 1.3125,
+      "theoretical_loss": 3.461783287993027,
+      "tokens_seen": 1979645952
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040410848980901944,
+      "loss": 1.2609,
+      "theoretical_loss": 3.461764297238307,
+      "tokens_seen": 1979777024
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004040683678382282,
+      "loss": 1.2217,
+      "theoretical_loss": 3.4617453080928513,
+      "tokens_seen": 1979908096
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040402824586743705,
+      "loss": 1.2465,
+      "theoretical_loss": 3.4617263205564175,
+      "tokens_seen": 1980039168
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004039881238966458,
+      "loss": 1.2771,
+      "theoretical_loss": 3.461707334628763,
+      "tokens_seen": 1980170240
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004039480019258546,
+      "loss": 1.2164,
+      "theoretical_loss": 3.461688350309644,
+      "tokens_seen": 1980301312
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004039078799550634,
+      "loss": 1.3622,
+      "theoretical_loss": 3.4616693675988186,
+      "tokens_seen": 1980432384
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004038677579842722,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4616503864960437,
+      "tokens_seen": 1980563456
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040382763601348097,
+      "loss": 1.2628,
+      "theoretical_loss": 3.461631407001077,
+      "tokens_seen": 1980694528
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004037875140426898,
+      "loss": 1.2233,
+      "theoretical_loss": 3.461612429113675,
+      "tokens_seen": 1980825600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004037473920718986,
+      "loss": 1.3287,
+      "theoretical_loss": 3.4615934528335965,
+      "tokens_seen": 1980956672
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040370727010110734,
+      "loss": 1.3483,
+      "theoretical_loss": 3.4615744781605984,
+      "tokens_seen": 1981087744
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004036671481303162,
+      "loss": 1.2706,
+      "theoretical_loss": 3.461555505094438,
+      "tokens_seen": 1981218816
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040362702615952495,
+      "loss": 1.2216,
+      "theoretical_loss": 3.461536533634874,
+      "tokens_seen": 1981349888
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004035869041887338,
+      "loss": 1.4017,
+      "theoretical_loss": 3.4615175637816624,
+      "tokens_seen": 1981480960
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040354678221794256,
+      "loss": 1.3307,
+      "theoretical_loss": 3.4614985955345623,
+      "tokens_seen": 1981612032
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004035066602471513,
+      "loss": 1.3279,
+      "theoretical_loss": 3.461479628893331,
+      "tokens_seen": 1981743104
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004034665382763602,
+      "loss": 1.2339,
+      "theoretical_loss": 3.4614606638577263,
+      "tokens_seen": 1981874176
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004034264163055689,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4614417004275064,
+      "tokens_seen": 1982005248
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040338629433477773,
+      "loss": 1.2813,
+      "theoretical_loss": 3.461422738602429,
+      "tokens_seen": 1982136320
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040334617236398654,
+      "loss": 1.3083,
+      "theoretical_loss": 3.461403778382252,
+      "tokens_seen": 1982267392
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040330605039319534,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4613848197667343,
+      "tokens_seen": 1982398464
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004032659284224041,
+      "loss": 1.2906,
+      "theoretical_loss": 3.461365862755633,
+      "tokens_seen": 1982529536
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004032258064516129,
+      "loss": 1.2646,
+      "theoretical_loss": 3.461346907348706,
+      "tokens_seen": 1982660608
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004031856844808217,
+      "loss": 1.2573,
+      "theoretical_loss": 3.461327953545713,
+      "tokens_seen": 1982791680
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040314556251003046,
+      "loss": 1.2598,
+      "theoretical_loss": 3.461309001346411,
+      "tokens_seen": 1982922752
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004031054405392393,
+      "loss": 1.2576,
+      "theoretical_loss": 3.4612900507505584,
+      "tokens_seen": 1983053824
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004030653185684481,
+      "loss": 1.2403,
+      "theoretical_loss": 3.461271101757914,
+      "tokens_seen": 1983184896
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004030251965976569,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4612521543682364,
+      "tokens_seen": 1983315968
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004029850746268657,
+      "loss": 1.2349,
+      "theoretical_loss": 3.461233208581284,
+      "tokens_seen": 1983447040
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040294495265607444,
+      "loss": 1.3047,
+      "theoretical_loss": 3.4612142643968147,
+      "tokens_seen": 1983578112
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004029048306852833,
+      "loss": 1.1653,
+      "theoretical_loss": 3.4611953218145874,
+      "tokens_seen": 1983709184
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040286470871449205,
+      "loss": 1.191,
+      "theoretical_loss": 3.4611763808343614,
+      "tokens_seen": 1983840256
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040282458674370086,
+      "loss": 1.2887,
+      "theoretical_loss": 3.461157441455894,
+      "tokens_seen": 1983971328
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040278446477290966,
+      "loss": 1.3249,
+      "theoretical_loss": 3.4611385036789457,
+      "tokens_seen": 1984102400
+    },
+    {
+      "epoch": 0.2,
+      "objective/train/docs_used": 666996,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2839837074279785,
+      "objective/train/theoretical_loss": 3.4611195675032738,
+      "objective/train/tokens_used": 354692576,
+      "theoretical_loss": 3.4611195675032738,
+      "tokens_seen": 1984233472
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040274434280211847,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4611195675032738,
+      "tokens_seen": 1984233472
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004027042208313272,
+      "loss": 1.2781,
+      "theoretical_loss": 3.4611006329286376,
+      "tokens_seen": 1984364544
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004026640988605361,
+      "loss": 1.203,
+      "theoretical_loss": 3.461081699954796,
+      "tokens_seen": 1984495616
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040262397688974483,
+      "loss": 1.2936,
+      "theoretical_loss": 3.4610627685815083,
+      "tokens_seen": 1984626688
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004025838549189536,
+      "loss": 1.2146,
+      "theoretical_loss": 3.461043838808533,
+      "tokens_seen": 1984757760
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040254373294816245,
+      "loss": 1.2372,
+      "theoretical_loss": 3.4610249106356292,
+      "tokens_seen": 1984888832
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004025036109773712,
+      "loss": 1.3284,
+      "theoretical_loss": 3.461005984062557,
+      "tokens_seen": 1985019904
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040246348900658,
+      "loss": 1.2506,
+      "theoretical_loss": 3.4609870590890743,
+      "tokens_seen": 1985150976
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004024233670357888,
+      "loss": 1.2847,
+      "theoretical_loss": 3.4609681357149404,
+      "tokens_seen": 1985282048
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004023832450649976,
+      "loss": 1.2518,
+      "theoretical_loss": 3.460949213939915,
+      "tokens_seen": 1985413120
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004023431230942064,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4609302937637576,
+      "tokens_seen": 1985544192
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004023030011234152,
+      "loss": 1.3513,
+      "theoretical_loss": 3.4609113751862273,
+      "tokens_seen": 1985675264
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.000402262879152624,
+      "loss": 1.3023,
+      "theoretical_loss": 3.4608924582070832,
+      "tokens_seen": 1985806336
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004022227571818328,
+      "loss": 1.3207,
+      "theoretical_loss": 3.4608735428260857,
+      "tokens_seen": 1985937408
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004021826352110416,
+      "loss": 1.1869,
+      "theoretical_loss": 3.460854629042993,
+      "tokens_seen": 1986068480
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040214251324025035,
+      "loss": 1.248,
+      "theoretical_loss": 3.4608357168575656,
+      "tokens_seen": 1986199552
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004021023912694592,
+      "loss": 1.3622,
+      "theoretical_loss": 3.4608168062695626,
+      "tokens_seen": 1986330624
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040206226929866796,
+      "loss": 1.2982,
+      "theoretical_loss": 3.4607978972787445,
+      "tokens_seen": 1986461696
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004020221473278767,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4607789898848704,
+      "tokens_seen": 1986592768
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004019820253570856,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4607600840877,
+      "tokens_seen": 1986723840
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004019419033862943,
+      "loss": 1.3014,
+      "theoretical_loss": 3.460741179886993,
+      "tokens_seen": 1986854912
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040190178141550313,
+      "loss": 1.2996,
+      "theoretical_loss": 3.4607222772825104,
+      "tokens_seen": 1986985984
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040186165944471194,
+      "loss": 1.2451,
+      "theoretical_loss": 3.4607033762740107,
+      "tokens_seen": 1987117056
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040182153747392074,
+      "loss": 1.3068,
+      "theoretical_loss": 3.4606844768612546,
+      "tokens_seen": 1987248128
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004017814155031295,
+      "loss": 1.245,
+      "theoretical_loss": 3.4606655790440017,
+      "tokens_seen": 1987379200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004017412935323383,
+      "loss": 1.2553,
+      "theoretical_loss": 3.4606466828220133,
+      "tokens_seen": 1987510272
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004017011715615471,
+      "loss": 1.3492,
+      "theoretical_loss": 3.4606277881950485,
+      "tokens_seen": 1987641344
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004016610495907559,
+      "loss": 1.2491,
+      "theoretical_loss": 3.460608895162867,
+      "tokens_seen": 1987772416
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004016209276199647,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4605900037252306,
+      "tokens_seen": 1987903488
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004015808056491735,
+      "loss": 1.2872,
+      "theoretical_loss": 3.4605711138818984,
+      "tokens_seen": 1988034560
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040154068367838233,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4605522256326307,
+      "tokens_seen": 1988165632
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004015005617075911,
+      "loss": 1.2413,
+      "theoretical_loss": 3.460533338977189,
+      "tokens_seen": 1988296704
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040146043973679984,
+      "loss": 1.2712,
+      "theoretical_loss": 3.4605144539153327,
+      "tokens_seen": 1988427776
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004014203177660087,
+      "loss": 1.2846,
+      "theoretical_loss": 3.4604955704468225,
+      "tokens_seen": 1988558848
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040138019579521745,
+      "loss": 1.2936,
+      "theoretical_loss": 3.4604766885714193,
+      "tokens_seen": 1988689920
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040134007382442626,
+      "loss": 1.2348,
+      "theoretical_loss": 3.4604578082888837,
+      "tokens_seen": 1988820992
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040129995185363506,
+      "loss": 1.2155,
+      "theoretical_loss": 3.460438929598976,
+      "tokens_seen": 1988952064
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040125982988284387,
+      "loss": 1.3163,
+      "theoretical_loss": 3.460420052501457,
+      "tokens_seen": 1989083136
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004012197079120526,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4604011769960876,
+      "tokens_seen": 1989214208
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040117958594126143,
+      "loss": 1.2121,
+      "theoretical_loss": 3.4603823030826284,
+      "tokens_seen": 1989345280
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040113946397047023,
+      "loss": 1.1806,
+      "theoretical_loss": 3.4603634307608413,
+      "tokens_seen": 1989476352
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000401099341999679,
+      "loss": 1.2979,
+      "theoretical_loss": 3.4603445600304856,
+      "tokens_seen": 1989607424
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040105922002888785,
+      "loss": 1.2611,
+      "theoretical_loss": 3.4603256908913234,
+      "tokens_seen": 1989738496
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004010190980580966,
+      "loss": 1.2609,
+      "theoretical_loss": 3.460306823343115,
+      "tokens_seen": 1989869568
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040097897608730546,
+      "loss": 1.3454,
+      "theoretical_loss": 3.4602879573856224,
+      "tokens_seen": 1990000640
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004009388541165142,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4602690930186055,
+      "tokens_seen": 1990131712
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000400898732145723,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4602502302418268,
+      "tokens_seen": 1990262784
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004008586101749318,
+      "loss": 1.2782,
+      "theoretical_loss": 3.4602313690550464,
+      "tokens_seen": 1990393856
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004008184882041406,
+      "loss": 1.3636,
+      "theoretical_loss": 3.4602125094580267,
+      "tokens_seen": 1990524928
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004007783662333494,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4601936514505276,
+      "tokens_seen": 1990656000
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 669188,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3386015892028809,
+      "objective/train/theoretical_loss": 3.4601747950323123,
+      "objective/train/tokens_used": 361246176,
+      "theoretical_loss": 3.4601747950323123,
+      "tokens_seen": 1990787072
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004007382442625582,
+      "loss": 1.2959,
+      "theoretical_loss": 3.4601747950323123,
+      "tokens_seen": 1990787072
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000400698122291767,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4601559402031405,
+      "tokens_seen": 1990918144
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040065800032097575,
+      "loss": 1.3556,
+      "theoretical_loss": 3.4601370869627743,
+      "tokens_seen": 1991049216
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004006178783501846,
+      "loss": 1.2507,
+      "theoretical_loss": 3.4601182353109756,
+      "tokens_seen": 1991180288
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040057775637939336,
+      "loss": 1.1895,
+      "theoretical_loss": 3.4600993852475064,
+      "tokens_seen": 1991311360
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004005376344086021,
+      "loss": 1.2753,
+      "theoretical_loss": 3.460080536772127,
+      "tokens_seen": 1991442432
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040049751243781097,
+      "loss": 1.2934,
+      "theoretical_loss": 3.4600616898845997,
+      "tokens_seen": 1991573504
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004004573904670197,
+      "loss": 1.262,
+      "theoretical_loss": 3.4600428445846867,
+      "tokens_seen": 1991704576
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004004172684962286,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4600240008721492,
+      "tokens_seen": 1991835648
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040037714652543734,
+      "loss": 1.2839,
+      "theoretical_loss": 3.4600051587467493,
+      "tokens_seen": 1991966720
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040033702455464614,
+      "loss": 1.3099,
+      "theoretical_loss": 3.459986318208249,
+      "tokens_seen": 1992097792
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040029690258385495,
+      "loss": 1.3224,
+      "theoretical_loss": 3.4599674792564103,
+      "tokens_seen": 1992228864
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004002567806130637,
+      "loss": 1.1905,
+      "theoretical_loss": 3.4599486418909944,
+      "tokens_seen": 1992359936
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004002166586422725,
+      "loss": 1.2464,
+      "theoretical_loss": 3.4599298061117647,
+      "tokens_seen": 1992491008
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004001765366714813,
+      "loss": 1.3258,
+      "theoretical_loss": 3.4599109719184824,
+      "tokens_seen": 1992622080
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004001364147006901,
+      "loss": 1.2298,
+      "theoretical_loss": 3.4598921393109094,
+      "tokens_seen": 1992753152
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004000962927298989,
+      "loss": 1.2422,
+      "theoretical_loss": 3.459873308288809,
+      "tokens_seen": 1992884224
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040005617075910773,
+      "loss": 1.3043,
+      "theoretical_loss": 3.4598544788519425,
+      "tokens_seen": 1993015296
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004000160487883165,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4598356510000725,
+      "tokens_seen": 1993146368
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039997592681752524,
+      "loss": 1.2514,
+      "theoretical_loss": 3.459816824732962,
+      "tokens_seen": 1993277440
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003999358048467341,
+      "loss": 1.2969,
+      "theoretical_loss": 3.459798000050372,
+      "tokens_seen": 1993408512
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039989568287594285,
+      "loss": 1.3554,
+      "theoretical_loss": 3.4597791769520656,
+      "tokens_seen": 1993539584
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039985556090515166,
+      "loss": 1.2501,
+      "theoretical_loss": 3.4597603554378065,
+      "tokens_seen": 1993670656
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039981543893436046,
+      "loss": 1.3706,
+      "theoretical_loss": 3.4597415355073555,
+      "tokens_seen": 1993801728
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039977531696356927,
+      "loss": 1.2317,
+      "theoretical_loss": 3.459722717160476,
+      "tokens_seen": 1993932800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003997351949927781,
+      "loss": 1.2494,
+      "theoretical_loss": 3.459703900396931,
+      "tokens_seen": 1994063872
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039969507302198683,
+      "loss": 1.2254,
+      "theoretical_loss": 3.459685085216483,
+      "tokens_seen": 1994194944
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039965495105119563,
+      "loss": 1.2409,
+      "theoretical_loss": 3.459666271618894,
+      "tokens_seen": 1994326016
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039961482908040444,
+      "loss": 1.3383,
+      "theoretical_loss": 3.4596474596039277,
+      "tokens_seen": 1994457088
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039957470710961325,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4596286491713473,
+      "tokens_seen": 1994588160
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000399534585138822,
+      "loss": 1.311,
+      "theoretical_loss": 3.4596098403209146,
+      "tokens_seen": 1994719232
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039949446316803086,
+      "loss": 1.1866,
+      "theoretical_loss": 3.4595910330523933,
+      "tokens_seen": 1994850304
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003994543411972396,
+      "loss": 1.3242,
+      "theoretical_loss": 3.459572227365546,
+      "tokens_seen": 1994981376
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003994142192264484,
+      "loss": 1.3759,
+      "theoretical_loss": 3.4595534232601364,
+      "tokens_seen": 1995112448
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003993740972556572,
+      "loss": 1.255,
+      "theoretical_loss": 3.459534620735927,
+      "tokens_seen": 1995243520
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000399333975284866,
+      "loss": 1.2713,
+      "theoretical_loss": 3.4595158197926814,
+      "tokens_seen": 1995374592
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003992938533140748,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4594970204301623,
+      "tokens_seen": 1995505664
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003992537313432836,
+      "loss": 1.3131,
+      "theoretical_loss": 3.459478222648134,
+      "tokens_seen": 1995636736
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003992136093724924,
+      "loss": 1.2542,
+      "theoretical_loss": 3.459459426446359,
+      "tokens_seen": 1995767808
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039917348740170115,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4594406318246005,
+      "tokens_seen": 1995898880
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039913336543091,
+      "loss": 1.2012,
+      "theoretical_loss": 3.4594218387826228,
+      "tokens_seen": 1996029952
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039909324346011876,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4594030473201887,
+      "tokens_seen": 1996161024
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039905312148932757,
+      "loss": 1.2598,
+      "theoretical_loss": 3.459384257437062,
+      "tokens_seen": 1996292096
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039901299951853637,
+      "loss": 1.2061,
+      "theoretical_loss": 3.4593654691330062,
+      "tokens_seen": 1996423168
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003989728775477451,
+      "loss": 1.2282,
+      "theoretical_loss": 3.459346682407785,
+      "tokens_seen": 1996554240
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000398932755576954,
+      "loss": 1.2829,
+      "theoretical_loss": 3.4593278972611623,
+      "tokens_seen": 1996685312
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039889263360616274,
+      "loss": 1.2076,
+      "theoretical_loss": 3.459309113692901,
+      "tokens_seen": 1996816384
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039885251163537154,
+      "loss": 1.2084,
+      "theoretical_loss": 3.459290331702766,
+      "tokens_seen": 1996947456
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039881238966458035,
+      "loss": 1.3096,
+      "theoretical_loss": 3.45927155129052,
+      "tokens_seen": 1997078528
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003987722676937891,
+      "loss": 1.3098,
+      "theoretical_loss": 3.4592527724559283,
+      "tokens_seen": 1997209600
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 671479,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3098700046539307,
+      "objective/train/theoretical_loss": 3.459233995198754,
+      "objective/train/tokens_used": 367799776,
+      "theoretical_loss": 3.459233995198754,
+      "tokens_seen": 1997340672
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003987321457229979,
+      "loss": 1.2562,
+      "theoretical_loss": 3.459233995198754,
+      "tokens_seen": 1997340672
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003986920237522067,
+      "loss": 1.2789,
+      "theoretical_loss": 3.459215219518761,
+      "tokens_seen": 1997471744
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003986519017814155,
+      "loss": 1.205,
+      "theoretical_loss": 3.459196445415713,
+      "tokens_seen": 1997602816
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039861177981062427,
+      "loss": 1.3005,
+      "theoretical_loss": 3.4591776728893753,
+      "tokens_seen": 1997733888
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039857165783983313,
+      "loss": 1.2751,
+      "theoretical_loss": 3.459158901939511,
+      "tokens_seen": 1997864960
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003985315358690419,
+      "loss": 1.2294,
+      "theoretical_loss": 3.4591401325658846,
+      "tokens_seen": 1997996032
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003984914138982507,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4591213647682606,
+      "tokens_seen": 1998127104
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003984512919274595,
+      "loss": 1.305,
+      "theoretical_loss": 3.4591025985464032,
+      "tokens_seen": 1998258176
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039841116995666825,
+      "loss": 1.2918,
+      "theoretical_loss": 3.4590838339000767,
+      "tokens_seen": 1998389248
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003983710479858771,
+      "loss": 1.1624,
+      "theoretical_loss": 3.459065070829045,
+      "tokens_seen": 1998520320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039833092601508586,
+      "loss": 1.2293,
+      "theoretical_loss": 3.4590463093330737,
+      "tokens_seen": 1998651392
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039829080404429467,
+      "loss": 1.176,
+      "theoretical_loss": 3.4590275494119256,
+      "tokens_seen": 1998782464
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003982506820735035,
+      "loss": 1.2695,
+      "theoretical_loss": 3.459008791065367,
+      "tokens_seen": 1998913536
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039821056010271223,
+      "loss": 1.2457,
+      "theoretical_loss": 3.4589900342931617,
+      "tokens_seen": 1999044608
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039817043813192103,
+      "loss": 1.2743,
+      "theoretical_loss": 3.458971279095074,
+      "tokens_seen": 1999175680
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039813031616112984,
+      "loss": 1.2986,
+      "theoretical_loss": 3.4589525254708695,
+      "tokens_seen": 1999306752
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039809019419033865,
+      "loss": 1.3267,
+      "theoretical_loss": 3.4589337734203127,
+      "tokens_seen": 1999437824
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003980500722195474,
+      "loss": 1.2794,
+      "theoretical_loss": 3.4589150229431675,
+      "tokens_seen": 1999568896
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039800995024875626,
+      "loss": 1.315,
+      "theoretical_loss": 3.4588962740392,
+      "tokens_seen": 1999699968
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000397969828277965,
+      "loss": 1.3248,
+      "theoretical_loss": 3.458877526708174,
+      "tokens_seen": 1999831040
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003979297063071738,
+      "loss": 1.2034,
+      "theoretical_loss": 3.4588587809498557,
+      "tokens_seen": 1999962112
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003978895843363826,
+      "loss": 1.2171,
+      "theoretical_loss": 3.458840036764009,
+      "tokens_seen": 2000093184
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003978494623655914,
+      "loss": 1.2886,
+      "theoretical_loss": 3.4588212941503995,
+      "tokens_seen": 2000224256
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039780934039480024,
+      "loss": 1.198,
+      "theoretical_loss": 3.458802553108792,
+      "tokens_seen": 2000355328
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000397769218424009,
+      "loss": 1.3387,
+      "theoretical_loss": 3.458783813638952,
+      "tokens_seen": 2000486400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003977290964532178,
+      "loss": 1.2365,
+      "theoretical_loss": 3.4587650757406445,
+      "tokens_seen": 2000617472
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003976889744824266,
+      "loss": 1.3228,
+      "theoretical_loss": 3.458746339413635,
+      "tokens_seen": 2000748544
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003976488525116354,
+      "loss": 1.2944,
+      "theoretical_loss": 3.4587276046576885,
+      "tokens_seen": 2000879616
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039760873054084416,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4587088714725702,
+      "tokens_seen": 2001010688
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039756860857005297,
+      "loss": 1.2365,
+      "theoretical_loss": 3.4586901398580467,
+      "tokens_seen": 2001141760
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039752848659926177,
+      "loss": 1.2811,
+      "theoretical_loss": 3.4586714098138813,
+      "tokens_seen": 2001272832
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003974883646284705,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4586526813398417,
+      "tokens_seen": 2001403904
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003974482426576794,
+      "loss": 1.2553,
+      "theoretical_loss": 3.458633954435692,
+      "tokens_seen": 2001534976
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039740812068688814,
+      "loss": 1.2766,
+      "theoretical_loss": 3.458615229101199,
+      "tokens_seen": 2001666048
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039736799871609694,
+      "loss": 1.236,
+      "theoretical_loss": 3.458596505336127,
+      "tokens_seen": 2001797120
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039732787674530575,
+      "loss": 1.2896,
+      "theoretical_loss": 3.458577783140243,
+      "tokens_seen": 2001928192
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003972877547745145,
+      "loss": 1.3138,
+      "theoretical_loss": 3.458559062513312,
+      "tokens_seen": 2002059264
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039724763280372336,
+      "loss": 1.2644,
+      "theoretical_loss": 3.4585403434551,
+      "tokens_seen": 2002190336
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003972075108329321,
+      "loss": 1.3287,
+      "theoretical_loss": 3.4585216259653726,
+      "tokens_seen": 2002321408
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003971673888621409,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4585029100438964,
+      "tokens_seen": 2002452480
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003971272668913497,
+      "loss": 1.3084,
+      "theoretical_loss": 3.4584841956904366,
+      "tokens_seen": 2002583552
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039708714492055853,
+      "loss": 1.2908,
+      "theoretical_loss": 3.4584654829047596,
+      "tokens_seen": 2002714624
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003970470229497673,
+      "loss": 1.29,
+      "theoretical_loss": 3.4584467716866314,
+      "tokens_seen": 2002845696
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003970069009789761,
+      "loss": 1.2806,
+      "theoretical_loss": 3.4584280620358188,
+      "tokens_seen": 2002976768
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003969667790081849,
+      "loss": 1.2944,
+      "theoretical_loss": 3.458409353952087,
+      "tokens_seen": 2003107840
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039692665703739365,
+      "loss": 1.2308,
+      "theoretical_loss": 3.458390647435202,
+      "tokens_seen": 2003238912
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003968865350666025,
+      "loss": 1.2985,
+      "theoretical_loss": 3.458371942484931,
+      "tokens_seen": 2003369984
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039684641309581126,
+      "loss": 1.2819,
+      "theoretical_loss": 3.45835323910104,
+      "tokens_seen": 2003501056
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039680629112502007,
+      "loss": 1.2611,
+      "theoretical_loss": 3.458334537283295,
+      "tokens_seen": 2003632128
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003967661691542289,
+      "loss": 1.2504,
+      "theoretical_loss": 3.458315837031463,
+      "tokens_seen": 2003763200
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 673566,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3838492631912231,
+      "objective/train/theoretical_loss": 3.4582971383453103,
+      "objective/train/tokens_used": 374353376,
+      "theoretical_loss": 3.4582971383453103,
+      "tokens_seen": 2003894272
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039672604718343763,
+      "loss": 1.3177,
+      "theoretical_loss": 3.4582971383453103,
+      "tokens_seen": 2003894272
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039668592521264643,
+      "loss": 1.2994,
+      "theoretical_loss": 3.4582784412246026,
+      "tokens_seen": 2004025344
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039664580324185524,
+      "loss": 1.298,
+      "theoretical_loss": 3.4582597456691078,
+      "tokens_seen": 2004156416
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039660568127106405,
+      "loss": 1.3054,
+      "theoretical_loss": 3.4582410516785917,
+      "tokens_seen": 2004287488
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039656555930027285,
+      "loss": 1.3247,
+      "theoretical_loss": 3.4582223592528214,
+      "tokens_seen": 2004418560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039652543732948166,
+      "loss": 1.3053,
+      "theoretical_loss": 3.4582036683915627,
+      "tokens_seen": 2004549632
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003964853153586904,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4581849790945833,
+      "tokens_seen": 2004680704
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003964451933878992,
+      "loss": 1.241,
+      "theoretical_loss": 3.4581662913616498,
+      "tokens_seen": 2004811776
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.000396405071417108,
+      "loss": 1.3008,
+      "theoretical_loss": 3.4581476051925293,
+      "tokens_seen": 2004942848
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003963649494463168,
+      "loss": 1.2137,
+      "theoretical_loss": 3.4581289205869883,
+      "tokens_seen": 2005073920
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039632482747552564,
+      "loss": 1.2858,
+      "theoretical_loss": 3.458110237544794,
+      "tokens_seen": 2005204992
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003962847055047344,
+      "loss": 1.2284,
+      "theoretical_loss": 3.458091556065713,
+      "tokens_seen": 2005336064
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003962445835339432,
+      "loss": 1.2726,
+      "theoretical_loss": 3.4580728761495125,
+      "tokens_seen": 2005467136
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.000396204461563152,
+      "loss": 1.2788,
+      "theoretical_loss": 3.45805419779596,
+      "tokens_seen": 2005598208
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003961643395923608,
+      "loss": 1.2768,
+      "theoretical_loss": 3.458035521004823,
+      "tokens_seen": 2005729280
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039612421762156956,
+      "loss": 1.2858,
+      "theoretical_loss": 3.4580168457758673,
+      "tokens_seen": 2005860352
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039608409565077837,
+      "loss": 1.2724,
+      "theoretical_loss": 3.4579981721088613,
+      "tokens_seen": 2005991424
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039604397367998717,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4579795000035727,
+      "tokens_seen": 2006122496
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003960038517091959,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4579608294597675,
+      "tokens_seen": 2006253568
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003959637297384048,
+      "loss": 1.3097,
+      "theoretical_loss": 3.4579421604772143,
+      "tokens_seen": 2006384640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039592360776761354,
+      "loss": 1.2389,
+      "theoretical_loss": 3.45792349305568,
+      "tokens_seen": 2006515712
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003958834857968224,
+      "loss": 1.2795,
+      "theoretical_loss": 3.457904827194932,
+      "tokens_seen": 2006646784
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039584336382603115,
+      "loss": 1.2686,
+      "theoretical_loss": 3.457886162894739,
+      "tokens_seen": 2006777856
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003958032418552399,
+      "loss": 1.2095,
+      "theoretical_loss": 3.457867500154866,
+      "tokens_seen": 2006908928
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039576311988444876,
+      "loss": 1.2482,
+      "theoretical_loss": 3.457848838975084,
+      "tokens_seen": 2007040000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003957229979136575,
+      "loss": 1.3062,
+      "theoretical_loss": 3.4578301793551587,
+      "tokens_seen": 2007171072
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003956828759428663,
+      "loss": 1.2386,
+      "theoretical_loss": 3.4578115212948575,
+      "tokens_seen": 2007302144
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003956427539720751,
+      "loss": 1.2356,
+      "theoretical_loss": 3.4577928647939498,
+      "tokens_seen": 2007433216
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039560263200128393,
+      "loss": 1.3431,
+      "theoretical_loss": 3.4577742098522024,
+      "tokens_seen": 2007564288
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003955625100304927,
+      "loss": 1.3319,
+      "theoretical_loss": 3.457755556469383,
+      "tokens_seen": 2007695360
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003955223880597015,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4577369046452606,
+      "tokens_seen": 2007826432
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003954822660889103,
+      "loss": 1.3228,
+      "theoretical_loss": 3.4577182543796026,
+      "tokens_seen": 2007957504
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039544214411811905,
+      "loss": 1.2736,
+      "theoretical_loss": 3.457699605672177,
+      "tokens_seen": 2008088576
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003954020221473279,
+      "loss": 1.3411,
+      "theoretical_loss": 3.457680958522752,
+      "tokens_seen": 2008219648
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039536190017653666,
+      "loss": 1.2851,
+      "theoretical_loss": 3.457662312931096,
+      "tokens_seen": 2008350720
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003953217782057455,
+      "loss": 1.253,
+      "theoretical_loss": 3.457643668896977,
+      "tokens_seen": 2008481792
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003952816562349543,
+      "loss": 1.2866,
+      "theoretical_loss": 3.457625026420163,
+      "tokens_seen": 2008612864
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.000395241534264163,
+      "loss": 1.3105,
+      "theoretical_loss": 3.4576063855004224,
+      "tokens_seen": 2008743936
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003952014122933719,
+      "loss": 1.212,
+      "theoretical_loss": 3.457587746137524,
+      "tokens_seen": 2008875008
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039516129032258064,
+      "loss": 1.1827,
+      "theoretical_loss": 3.457569108331236,
+      "tokens_seen": 2009006080
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039512116835178945,
+      "loss": 1.2563,
+      "theoretical_loss": 3.4575504720813264,
+      "tokens_seen": 2009137152
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039508104638099825,
+      "loss": 1.2666,
+      "theoretical_loss": 3.457531837387564,
+      "tokens_seen": 2009268224
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039504092441020706,
+      "loss": 1.3007,
+      "theoretical_loss": 3.457513204249718,
+      "tokens_seen": 2009399296
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003950008024394158,
+      "loss": 1.2656,
+      "theoretical_loss": 3.4574945726675566,
+      "tokens_seen": 2009530368
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003949606804686246,
+      "loss": 1.2894,
+      "theoretical_loss": 3.4574759426408477,
+      "tokens_seen": 2009661440
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003949205584978334,
+      "loss": 1.2845,
+      "theoretical_loss": 3.457457314169361,
+      "tokens_seen": 2009792512
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003948804365270422,
+      "loss": 1.2505,
+      "theoretical_loss": 3.457438687252865,
+      "tokens_seen": 2009923584
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039484031455625104,
+      "loss": 1.2526,
+      "theoretical_loss": 3.457420061891128,
+      "tokens_seen": 2010054656
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003948001925854598,
+      "loss": 1.2761,
+      "theoretical_loss": 3.4574014380839198,
+      "tokens_seen": 2010185728
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003947600706146686,
+      "loss": 1.2995,
+      "theoretical_loss": 3.4573828158310085,
+      "tokens_seen": 2010316800
+    },
+    {
+      "epoch": 0.22,
+      "objective/train/docs_used": 675820,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3124701976776123,
+      "objective/train/theoretical_loss": 3.457364195132163,
+      "objective/train/tokens_used": 380906976,
+      "theoretical_loss": 3.457364195132163,
+      "tokens_seen": 2010447872
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003947199486438774,
+      "loss": 1.2905,
+      "theoretical_loss": 3.457364195132163,
+      "tokens_seen": 2010447872
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003946798266730862,
+      "loss": 1.2615,
+      "theoretical_loss": 3.457345575987153,
+      "tokens_seen": 2010578944
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.000394639704702295,
+      "loss": 1.3587,
+      "theoretical_loss": 3.4573269583957473,
+      "tokens_seen": 2010710016
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039459958273150377,
+      "loss": 1.2758,
+      "theoretical_loss": 3.457308342357715,
+      "tokens_seen": 2010841088
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039455946076071257,
+      "loss": 1.2468,
+      "theoretical_loss": 3.4572897278728245,
+      "tokens_seen": 2010972160
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003945193387899214,
+      "loss": 1.2857,
+      "theoretical_loss": 3.4572711149408457,
+      "tokens_seen": 2011103232
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003944792168191302,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4572525035615485,
+      "tokens_seen": 2011234304
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039443909484833894,
+      "loss": 1.3217,
+      "theoretical_loss": 3.457233893734701,
+      "tokens_seen": 2011365376
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003943989728775478,
+      "loss": 1.3185,
+      "theoretical_loss": 3.457215285460073,
+      "tokens_seen": 2011496448
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039435885090675655,
+      "loss": 1.2363,
+      "theoretical_loss": 3.457196678737434,
+      "tokens_seen": 2011627520
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003943187289359653,
+      "loss": 1.2857,
+      "theoretical_loss": 3.4571780735665536,
+      "tokens_seen": 2011758592
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039427860696517416,
+      "loss": 1.3605,
+      "theoretical_loss": 3.457159469947201,
+      "tokens_seen": 2011889664
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003942384849943829,
+      "loss": 1.2647,
+      "theoretical_loss": 3.4571408678791453,
+      "tokens_seen": 2012020736
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003941983630235917,
+      "loss": 1.3025,
+      "theoretical_loss": 3.4571222673621573,
+      "tokens_seen": 2012151808
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003941582410528005,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4571036683960057,
+      "tokens_seen": 2012282880
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039411811908200933,
+      "loss": 1.3302,
+      "theoretical_loss": 3.457085070980461,
+      "tokens_seen": 2012413952
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039407799711121814,
+      "loss": 1.2165,
+      "theoretical_loss": 3.4570664751152913,
+      "tokens_seen": 2012545024
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003940378751404269,
+      "loss": 1.2236,
+      "theoretical_loss": 3.4570478808002685,
+      "tokens_seen": 2012676096
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003939977531696357,
+      "loss": 1.2614,
+      "theoretical_loss": 3.457029288035161,
+      "tokens_seen": 2012807168
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003939576311988445,
+      "loss": 1.1849,
+      "theoretical_loss": 3.457010696819739,
+      "tokens_seen": 2012938240
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003939175092280533,
+      "loss": 1.2211,
+      "theoretical_loss": 3.4569921071537726,
+      "tokens_seen": 2013069312
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039387738725726206,
+      "loss": 1.3133,
+      "theoretical_loss": 3.4569735190370325,
+      "tokens_seen": 2013200384
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003938372652864709,
+      "loss": 1.1837,
+      "theoretical_loss": 3.456954932469287,
+      "tokens_seen": 2013331456
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003937971433156797,
+      "loss": 1.2329,
+      "theoretical_loss": 3.4569363474503083,
+      "tokens_seen": 2013462528
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003937570213448884,
+      "loss": 1.2699,
+      "theoretical_loss": 3.4569177639798645,
+      "tokens_seen": 2013593600
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003937168993740973,
+      "loss": 1.2179,
+      "theoretical_loss": 3.456899182057727,
+      "tokens_seen": 2013724672
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039367677740330604,
+      "loss": 1.2256,
+      "theoretical_loss": 3.456880601683666,
+      "tokens_seen": 2013855744
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039363665543251485,
+      "loss": 1.2763,
+      "theoretical_loss": 3.456862022857451,
+      "tokens_seen": 2013986816
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039359653346172365,
+      "loss": 1.217,
+      "theoretical_loss": 3.4568434455788535,
+      "tokens_seen": 2014117888
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039355641149093246,
+      "loss": 1.3444,
+      "theoretical_loss": 3.456824869847643,
+      "tokens_seen": 2014248960
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003935162895201412,
+      "loss": 1.2793,
+      "theoretical_loss": 3.4568062956635903,
+      "tokens_seen": 2014380032
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039347616754935,
+      "loss": 1.2781,
+      "theoretical_loss": 3.4567877230264656,
+      "tokens_seen": 2014511104
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003934360455785588,
+      "loss": 1.3119,
+      "theoretical_loss": 3.4567691519360393,
+      "tokens_seen": 2014642176
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039339592360776763,
+      "loss": 1.335,
+      "theoretical_loss": 3.456750582392083,
+      "tokens_seen": 2014773248
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039335580163697644,
+      "loss": 1.2721,
+      "theoretical_loss": 3.4567320143943663,
+      "tokens_seen": 2014904320
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003933156796661852,
+      "loss": 1.285,
+      "theoretical_loss": 3.45671344794266,
+      "tokens_seen": 2015035392
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039327555769539405,
+      "loss": 1.2817,
+      "theoretical_loss": 3.456694883036735,
+      "tokens_seen": 2015166464
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003932354357246028,
+      "loss": 1.2994,
+      "theoretical_loss": 3.4566763196763626,
+      "tokens_seen": 2015297536
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003931953137538116,
+      "loss": 1.3214,
+      "theoretical_loss": 3.4566577578613122,
+      "tokens_seen": 2015428608
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003931551917830204,
+      "loss": 1.2558,
+      "theoretical_loss": 3.456639197591356,
+      "tokens_seen": 2015559680
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039311506981222916,
+      "loss": 1.3185,
+      "theoretical_loss": 3.456620638866265,
+      "tokens_seen": 2015690752
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039307494784143797,
+      "loss": 1.2966,
+      "theoretical_loss": 3.456602081685809,
+      "tokens_seen": 2015821824
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003930348258706468,
+      "loss": 1.2322,
+      "theoretical_loss": 3.45658352604976,
+      "tokens_seen": 2015952896
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003929947038998556,
+      "loss": 1.2826,
+      "theoretical_loss": 3.4565649719578886,
+      "tokens_seen": 2016083968
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039295458192906434,
+      "loss": 1.2935,
+      "theoretical_loss": 3.4565464194099658,
+      "tokens_seen": 2016215040
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003929144599582732,
+      "loss": 1.2944,
+      "theoretical_loss": 3.4565278684057636,
+      "tokens_seen": 2016346112
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039287433798748195,
+      "loss": 1.2651,
+      "theoretical_loss": 3.456509318945052,
+      "tokens_seen": 2016477184
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003928342160166907,
+      "loss": 1.2111,
+      "theoretical_loss": 3.4564907710276036,
+      "tokens_seen": 2016608256
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039279409404589956,
+      "loss": 1.2817,
+      "theoretical_loss": 3.4564722246531887,
+      "tokens_seen": 2016739328
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003927539720751083,
+      "loss": 1.2817,
+      "theoretical_loss": 3.4564536798215793,
+      "tokens_seen": 2016870400
+    },
+    {
+      "epoch": 0.22,
+      "objective/train/docs_used": 678408,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2401456832885742,
+      "objective/train/theoretical_loss": 3.456435136532546,
+      "objective/train/tokens_used": 387460576,
+      "theoretical_loss": 3.456435136532546,
+      "tokens_seen": 2017001472
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003927138501043172,
+      "loss": 1.2314,
+      "theoretical_loss": 3.456435136532546,
+      "tokens_seen": 2017001472
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003926737281335259,
+      "loss": 1.2732,
+      "theoretical_loss": 3.456416594785861,
+      "tokens_seen": 2017132544
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039263360616273473,
+      "loss": 1.2745,
+      "theoretical_loss": 3.456398054581295,
+      "tokens_seen": 2017263616
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039259348419194354,
+      "loss": 1.2546,
+      "theoretical_loss": 3.456379515918621,
+      "tokens_seen": 2017394688
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003925533622211523,
+      "loss": 1.3091,
+      "theoretical_loss": 3.4563609787976093,
+      "tokens_seen": 2017525760
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003925132402503611,
+      "loss": 1.2891,
+      "theoretical_loss": 3.456342443218032,
+      "tokens_seen": 2017656832
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003924731182795699,
+      "loss": 1.295,
+      "theoretical_loss": 3.4563239091796607,
+      "tokens_seen": 2017787904
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003924329963087787,
+      "loss": 1.3104,
+      "theoretical_loss": 3.456305376682267,
+      "tokens_seen": 2017918976
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039239287433798746,
+      "loss": 1.28,
+      "theoretical_loss": 3.4562868457256233,
+      "tokens_seen": 2018050048
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003923527523671963,
+      "loss": 1.3142,
+      "theoretical_loss": 3.456268316309501,
+      "tokens_seen": 2018181120
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003923126303964051,
+      "loss": 1.2327,
+      "theoretical_loss": 3.456249788433672,
+      "tokens_seen": 2018312192
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003922725084256138,
+      "loss": 1.2388,
+      "theoretical_loss": 3.456231262097908,
+      "tokens_seen": 2018443264
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003922323864548227,
+      "loss": 1.2262,
+      "theoretical_loss": 3.4562127373019815,
+      "tokens_seen": 2018574336
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039219226448403144,
+      "loss": 1.3029,
+      "theoretical_loss": 3.4561942140456647,
+      "tokens_seen": 2018705408
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003921521425132403,
+      "loss": 1.2184,
+      "theoretical_loss": 3.4561756923287286,
+      "tokens_seen": 2018836480
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039211202054244905,
+      "loss": 1.2255,
+      "theoretical_loss": 3.4561571721509465,
+      "tokens_seen": 2018967552
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039207189857165786,
+      "loss": 1.313,
+      "theoretical_loss": 3.45613865351209,
+      "tokens_seen": 2019098624
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039203177660086666,
+      "loss": 1.3257,
+      "theoretical_loss": 3.456120136411932,
+      "tokens_seen": 2019229696
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003919916546300754,
+      "loss": 1.266,
+      "theoretical_loss": 3.4561016208502435,
+      "tokens_seen": 2019360768
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003919515326592842,
+      "loss": 1.2408,
+      "theoretical_loss": 3.4560831068267976,
+      "tokens_seen": 2019491840
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039191141068849303,
+      "loss": 1.2544,
+      "theoretical_loss": 3.4560645943413673,
+      "tokens_seen": 2019622912
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039187128871770183,
+      "loss": 1.3057,
+      "theoretical_loss": 3.4560460833937237,
+      "tokens_seen": 2019753984
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003918311667469106,
+      "loss": 1.3359,
+      "theoretical_loss": 3.456027573983641,
+      "tokens_seen": 2019885056
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039179104477611945,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4560090661108895,
+      "tokens_seen": 2020016128
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003917509228053282,
+      "loss": 1.2523,
+      "theoretical_loss": 3.455990559775244,
+      "tokens_seen": 2020147200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039171080083453695,
+      "loss": 1.2837,
+      "theoretical_loss": 3.455972054976475,
+      "tokens_seen": 2020278272
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003916706788637458,
+      "loss": 1.2814,
+      "theoretical_loss": 3.455953551714357,
+      "tokens_seen": 2020409344
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039163055689295456,
+      "loss": 1.2726,
+      "theoretical_loss": 3.4559350499886623,
+      "tokens_seen": 2020540416
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039159043492216337,
+      "loss": 1.3147,
+      "theoretical_loss": 3.455916549799163,
+      "tokens_seen": 2020671488
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003915503129513722,
+      "loss": 1.2908,
+      "theoretical_loss": 3.455898051145632,
+      "tokens_seen": 2020802560
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.000391510190980581,
+      "loss": 1.286,
+      "theoretical_loss": 3.4558795540278426,
+      "tokens_seen": 2020933632
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003914700690097898,
+      "loss": 1.3473,
+      "theoretical_loss": 3.4558610584455676,
+      "tokens_seen": 2021064704
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003914299470389986,
+      "loss": 1.2725,
+      "theoretical_loss": 3.45584256439858,
+      "tokens_seen": 2021195776
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039138982506820735,
+      "loss": 1.2571,
+      "theoretical_loss": 3.455824071886653,
+      "tokens_seen": 2021326848
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039134970309741615,
+      "loss": 1.2444,
+      "theoretical_loss": 3.4558055809095585,
+      "tokens_seen": 2021457920
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039130958112662496,
+      "loss": 1.2836,
+      "theoretical_loss": 3.455787091467071,
+      "tokens_seen": 2021588992
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003912694591558337,
+      "loss": 1.3118,
+      "theoretical_loss": 3.4557686035589636,
+      "tokens_seen": 2021720064
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003912293371850426,
+      "loss": 1.2572,
+      "theoretical_loss": 3.4557501171850085,
+      "tokens_seen": 2021851136
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003911892152142513,
+      "loss": 1.3217,
+      "theoretical_loss": 3.455731632344979,
+      "tokens_seen": 2021982208
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039114909324346013,
+      "loss": 1.2814,
+      "theoretical_loss": 3.45571314903865,
+      "tokens_seen": 2022113280
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039110897127266894,
+      "loss": 1.2663,
+      "theoretical_loss": 3.455694667265793,
+      "tokens_seen": 2022244352
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003910688493018777,
+      "loss": 1.2881,
+      "theoretical_loss": 3.4556761870261825,
+      "tokens_seen": 2022375424
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003910287273310865,
+      "loss": 1.2831,
+      "theoretical_loss": 3.455657708319592,
+      "tokens_seen": 2022506496
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003909886053602953,
+      "loss": 1.1924,
+      "theoretical_loss": 3.4556392311457937,
+      "tokens_seen": 2022637568
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003909484833895041,
+      "loss": 1.2905,
+      "theoretical_loss": 3.455620755504562,
+      "tokens_seen": 2022768640
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039090836141871286,
+      "loss": 1.3437,
+      "theoretical_loss": 3.455602281395671,
+      "tokens_seen": 2022899712
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003908682394479217,
+      "loss": 1.2332,
+      "theoretical_loss": 3.4555838088188935,
+      "tokens_seen": 2023030784
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003908281174771305,
+      "loss": 1.2425,
+      "theoretical_loss": 3.455565337774004,
+      "tokens_seen": 2023161856
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003907879955063393,
+      "loss": 1.2258,
+      "theoretical_loss": 3.4555468682607753,
+      "tokens_seen": 2023292928
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003907478735355481,
+      "loss": 1.1871,
+      "theoretical_loss": 3.4555284002789817,
+      "tokens_seen": 2023424000
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 680686,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2464842796325684,
+      "objective/train/theoretical_loss": 3.455509933828397,
+      "objective/train/tokens_used": 394014176,
+      "theoretical_loss": 3.455509933828397,
+      "tokens_seen": 2023555072
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039070775156475684,
+      "loss": 1.24,
+      "theoretical_loss": 3.455509933828397,
+      "tokens_seen": 2023555072
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003906676295939657,
+      "loss": 1.2859,
+      "theoretical_loss": 3.455491468908795,
+      "tokens_seen": 2023686144
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039062750762317445,
+      "loss": 1.1977,
+      "theoretical_loss": 3.45547300551995,
+      "tokens_seen": 2023817216
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039058738565238326,
+      "loss": 1.3184,
+      "theoretical_loss": 3.4554545436616353,
+      "tokens_seen": 2023948288
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039054726368159206,
+      "loss": 1.2169,
+      "theoretical_loss": 3.455436083333625,
+      "tokens_seen": 2024079360
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003905071417108008,
+      "loss": 1.2567,
+      "theoretical_loss": 3.455417624535694,
+      "tokens_seen": 2024210432
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003904670197400096,
+      "loss": 1.2439,
+      "theoretical_loss": 3.4553991672676156,
+      "tokens_seen": 2024341504
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039042689776921843,
+      "loss": 1.3132,
+      "theoretical_loss": 3.4553807115291644,
+      "tokens_seen": 2024472576
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039038677579842723,
+      "loss": 1.2774,
+      "theoretical_loss": 3.4553622573201137,
+      "tokens_seen": 2024603648
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000390346653827636,
+      "loss": 1.2463,
+      "theoretical_loss": 3.455343804640239,
+      "tokens_seen": 2024734720
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039030653185684485,
+      "loss": 1.2626,
+      "theoretical_loss": 3.455325353489314,
+      "tokens_seen": 2024865792
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003902664098860536,
+      "loss": 1.2911,
+      "theoretical_loss": 3.455306903867113,
+      "tokens_seen": 2024996864
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003902262879152624,
+      "loss": 1.3132,
+      "theoretical_loss": 3.455288455773411,
+      "tokens_seen": 2025127936
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003901861659444712,
+      "loss": 1.2731,
+      "theoretical_loss": 3.4552700092079816,
+      "tokens_seen": 2025259008
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039014604397367996,
+      "loss": 1.2577,
+      "theoretical_loss": 3.4552515641705996,
+      "tokens_seen": 2025390080
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003901059220028888,
+      "loss": 1.2342,
+      "theoretical_loss": 3.45523312066104,
+      "tokens_seen": 2025521152
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003900658000320976,
+      "loss": 1.2688,
+      "theoretical_loss": 3.4552146786790767,
+      "tokens_seen": 2025652224
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003900256780613064,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4551962382244845,
+      "tokens_seen": 2025783296
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003899855560905152,
+      "loss": 1.2165,
+      "theoretical_loss": 3.455177799297039,
+      "tokens_seen": 2025914368
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000389945434119724,
+      "loss": 1.2988,
+      "theoretical_loss": 3.4551593618965137,
+      "tokens_seen": 2026045440
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038990531214893275,
+      "loss": 1.3551,
+      "theoretical_loss": 3.455140926022684,
+      "tokens_seen": 2026176512
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038986519017814155,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4551224916753247,
+      "tokens_seen": 2026307584
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038982506820735036,
+      "loss": 1.3055,
+      "theoretical_loss": 3.4551040588542103,
+      "tokens_seen": 2026438656
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003897849462365591,
+      "loss": 1.3495,
+      "theoretical_loss": 3.455085627559116,
+      "tokens_seen": 2026569728
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038974482426576797,
+      "loss": 1.263,
+      "theoretical_loss": 3.4550671977898175,
+      "tokens_seen": 2026700800
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003897047022949767,
+      "loss": 1.2828,
+      "theoretical_loss": 3.4550487695460887,
+      "tokens_seen": 2026831872
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038966458032418553,
+      "loss": 1.239,
+      "theoretical_loss": 3.455030342827705,
+      "tokens_seen": 2026962944
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038962445835339434,
+      "loss": 1.3464,
+      "theoretical_loss": 3.455011917634442,
+      "tokens_seen": 2027094016
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003895843363826031,
+      "loss": 1.3172,
+      "theoretical_loss": 3.4549934939660742,
+      "tokens_seen": 2027225088
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038954421441181195,
+      "loss": 1.3281,
+      "theoretical_loss": 3.4549750718223775,
+      "tokens_seen": 2027356160
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003895040924410207,
+      "loss": 1.3276,
+      "theoretical_loss": 3.4549566512031262,
+      "tokens_seen": 2027487232
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003894639704702295,
+      "loss": 1.2909,
+      "theoretical_loss": 3.4549382321080966,
+      "tokens_seen": 2027618304
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003894238484994383,
+      "loss": 1.2933,
+      "theoretical_loss": 3.4549198145370634,
+      "tokens_seen": 2027749376
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003893837265286471,
+      "loss": 1.2868,
+      "theoretical_loss": 3.4549013984898025,
+      "tokens_seen": 2027880448
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003893436045578559,
+      "loss": 1.2853,
+      "theoretical_loss": 3.454882983966089,
+      "tokens_seen": 2028011520
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003893034825870647,
+      "loss": 1.2963,
+      "theoretical_loss": 3.4548645709656984,
+      "tokens_seen": 2028142592
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003892633606162735,
+      "loss": 1.2509,
+      "theoretical_loss": 3.454846159488407,
+      "tokens_seen": 2028273664
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038922323864548224,
+      "loss": 1.252,
+      "theoretical_loss": 3.454827749533989,
+      "tokens_seen": 2028404736
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003891831166746911,
+      "loss": 1.3153,
+      "theoretical_loss": 3.454809341102221,
+      "tokens_seen": 2028535808
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038914299470389985,
+      "loss": 1.3048,
+      "theoretical_loss": 3.4547909341928786,
+      "tokens_seen": 2028666880
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038910287273310866,
+      "loss": 1.3197,
+      "theoretical_loss": 3.4547725288057376,
+      "tokens_seen": 2028797952
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038906275076231746,
+      "loss": 1.3113,
+      "theoretical_loss": 3.4547541249405738,
+      "tokens_seen": 2028929024
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003890226287915262,
+      "loss": 1.257,
+      "theoretical_loss": 3.454735722597162,
+      "tokens_seen": 2029060096
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003889825068207351,
+      "loss": 1.3224,
+      "theoretical_loss": 3.45471732177528,
+      "tokens_seen": 2029191168
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038894238484994383,
+      "loss": 1.3099,
+      "theoretical_loss": 3.454698922474702,
+      "tokens_seen": 2029322240
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038890226287915263,
+      "loss": 1.2859,
+      "theoretical_loss": 3.454680524695205,
+      "tokens_seen": 2029453312
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038886214090836144,
+      "loss": 1.3683,
+      "theoretical_loss": 3.4546621284365644,
+      "tokens_seen": 2029584384
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038882201893757025,
+      "loss": 1.3615,
+      "theoretical_loss": 3.454643733698557,
+      "tokens_seen": 2029715456
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000388781896966779,
+      "loss": 1.2834,
+      "theoretical_loss": 3.4546253404809577,
+      "tokens_seen": 2029846528
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003887417749959878,
+      "loss": 1.2983,
+      "theoretical_loss": 3.454606948783544,
+      "tokens_seen": 2029977600
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 682912,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.4097011089324951,
+      "objective/train/theoretical_loss": 3.4545885586060914,
+      "objective/train/tokens_used": 400567776,
+      "theoretical_loss": 3.4545885586060914,
+      "tokens_seen": 2030108672
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003887016530251966,
+      "loss": 1.3558,
+      "theoretical_loss": 3.4545885586060914,
+      "tokens_seen": 2030108672
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038866153105440536,
+      "loss": 1.3413,
+      "theoretical_loss": 3.4545701699483766,
+      "tokens_seen": 2030239744
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003886214090836142,
+      "loss": 1.2564,
+      "theoretical_loss": 3.4545517828101753,
+      "tokens_seen": 2030370816
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000388581287112823,
+      "loss": 1.2929,
+      "theoretical_loss": 3.454533397191264,
+      "tokens_seen": 2030501888
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003885411651420318,
+      "loss": 1.259,
+      "theoretical_loss": 3.4545150130914197,
+      "tokens_seen": 2030632960
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003885010431712406,
+      "loss": 1.2823,
+      "theoretical_loss": 3.4544966305104183,
+      "tokens_seen": 2030764032
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038846092120044934,
+      "loss": 1.2433,
+      "theoretical_loss": 3.4544782494480364,
+      "tokens_seen": 2030895104
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038842079922965815,
+      "loss": 1.3055,
+      "theoretical_loss": 3.4544598699040505,
+      "tokens_seen": 2031026176
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038838067725886695,
+      "loss": 1.3354,
+      "theoretical_loss": 3.454441491878238,
+      "tokens_seen": 2031157248
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038834055528807576,
+      "loss": 1.2525,
+      "theoretical_loss": 3.454423115370374,
+      "tokens_seen": 2031288320
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038830043331728457,
+      "loss": 1.304,
+      "theoretical_loss": 3.4544047403802356,
+      "tokens_seen": 2031419392
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038826031134649337,
+      "loss": 1.3098,
+      "theoretical_loss": 3.4543863669076007,
+      "tokens_seen": 2031550464
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003882201893757021,
+      "loss": 1.3209,
+      "theoretical_loss": 3.4543679949522454,
+      "tokens_seen": 2031681536
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000388180067404911,
+      "loss": 1.4001,
+      "theoretical_loss": 3.454349624513946,
+      "tokens_seen": 2031812608
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038813994543411974,
+      "loss": 1.3386,
+      "theoretical_loss": 3.4543312555924803,
+      "tokens_seen": 2031943680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003880998234633285,
+      "loss": 1.2469,
+      "theoretical_loss": 3.454312888187624,
+      "tokens_seen": 2032074752
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038805970149253735,
+      "loss": 1.3156,
+      "theoretical_loss": 3.4542945222991555,
+      "tokens_seen": 2032205824
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003880195795217461,
+      "loss": 1.2348,
+      "theoretical_loss": 3.4542761579268513,
+      "tokens_seen": 2032336896
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003879794575509549,
+      "loss": 1.2645,
+      "theoretical_loss": 3.454257795070488,
+      "tokens_seen": 2032467968
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003879393355801637,
+      "loss": 1.2462,
+      "theoretical_loss": 3.4542394337298425,
+      "tokens_seen": 2032599040
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003878992136093725,
+      "loss": 1.2988,
+      "theoretical_loss": 3.454221073904693,
+      "tokens_seen": 2032730112
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003878590916385813,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4542027155948163,
+      "tokens_seen": 2032861184
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003878189696677901,
+      "loss": 1.2748,
+      "theoretical_loss": 3.4541843587999894,
+      "tokens_seen": 2032992256
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003877788476969989,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4541660035199895,
+      "tokens_seen": 2033123328
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038773872572620764,
+      "loss": 1.3544,
+      "theoretical_loss": 3.454147649754595,
+      "tokens_seen": 2033254400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003876986037554165,
+      "loss": 1.3015,
+      "theoretical_loss": 3.4541292975035818,
+      "tokens_seen": 2033385472
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038765848178462525,
+      "loss": 1.3149,
+      "theoretical_loss": 3.4541109467667277,
+      "tokens_seen": 2033516544
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003876183598138341,
+      "loss": 1.3081,
+      "theoretical_loss": 3.454092597543811,
+      "tokens_seen": 2033647616
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038757823784304286,
+      "loss": 1.3307,
+      "theoretical_loss": 3.4540742498346084,
+      "tokens_seen": 2033778688
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003875381158722516,
+      "loss": 1.2927,
+      "theoretical_loss": 3.4540559036388983,
+      "tokens_seen": 2033909760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003874979939014605,
+      "loss": 1.3677,
+      "theoretical_loss": 3.4540375589564576,
+      "tokens_seen": 2034040832
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038745787193066923,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4540192157870644,
+      "tokens_seen": 2034171904
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038741774995987803,
+      "loss": 1.3257,
+      "theoretical_loss": 3.4540008741304966,
+      "tokens_seen": 2034302976
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038737762798908684,
+      "loss": 1.381,
+      "theoretical_loss": 3.453982533986531,
+      "tokens_seen": 2034434048
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038733750601829565,
+      "loss": 1.3225,
+      "theoretical_loss": 3.4539641953549465,
+      "tokens_seen": 2034565120
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003872973840475044,
+      "loss": 1.2968,
+      "theoretical_loss": 3.45394585823552,
+      "tokens_seen": 2034696192
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003872572620767132,
+      "loss": 1.2775,
+      "theoretical_loss": 3.45392752262803,
+      "tokens_seen": 2034827264
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000387217140105922,
+      "loss": 1.3025,
+      "theoretical_loss": 3.453909188532255,
+      "tokens_seen": 2034958336
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038717701813513076,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4538908559479715,
+      "tokens_seen": 2035089408
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003871368961643396,
+      "loss": 1.256,
+      "theoretical_loss": 3.453872524874959,
+      "tokens_seen": 2035220480
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003870967741935484,
+      "loss": 1.2433,
+      "theoretical_loss": 3.4538541953129953,
+      "tokens_seen": 2035351552
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038705665222275724,
+      "loss": 1.3016,
+      "theoretical_loss": 3.453835867261858,
+      "tokens_seen": 2035482624
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000387016530251966,
+      "loss": 1.2986,
+      "theoretical_loss": 3.453817540721326,
+      "tokens_seen": 2035613696
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038697640828117474,
+      "loss": 1.2835,
+      "theoretical_loss": 3.4537992156911765,
+      "tokens_seen": 2035744768
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003869362863103836,
+      "loss": 1.3337,
+      "theoretical_loss": 3.4537808921711886,
+      "tokens_seen": 2035875840
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038689616433959235,
+      "loss": 1.2588,
+      "theoretical_loss": 3.4537625701611407,
+      "tokens_seen": 2036006912
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038685604236880116,
+      "loss": 1.3598,
+      "theoretical_loss": 3.4537442496608106,
+      "tokens_seen": 2036137984
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038681592039800997,
+      "loss": 1.2306,
+      "theoretical_loss": 3.4537259306699775,
+      "tokens_seen": 2036269056
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038677579842721877,
+      "loss": 1.2619,
+      "theoretical_loss": 3.453707613188419,
+      "tokens_seen": 2036400128
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003867356764564275,
+      "loss": 1.3081,
+      "theoretical_loss": 3.4536892972159143,
+      "tokens_seen": 2036531200
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 684955,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2143110036849976,
+      "objective/train/theoretical_loss": 3.4536709827522416,
+      "objective/train/tokens_used": 407121376,
+      "theoretical_loss": 3.4536709827522416,
+      "tokens_seen": 2036662272
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003866955544856364,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4536709827522416,
+      "tokens_seen": 2036662272
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038665543251484514,
+      "loss": 1.2265,
+      "theoretical_loss": 3.4536526697971794,
+      "tokens_seen": 2036793344
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003866153105440539,
+      "loss": 1.2263,
+      "theoretical_loss": 3.453634358350507,
+      "tokens_seen": 2036924416
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038657518857326275,
+      "loss": 1.2977,
+      "theoretical_loss": 3.4536160484120026,
+      "tokens_seen": 2037055488
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003865350666024715,
+      "loss": 1.2532,
+      "theoretical_loss": 3.4535977399814453,
+      "tokens_seen": 2037186560
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003864949446316803,
+      "loss": 1.2937,
+      "theoretical_loss": 3.4535794330586134,
+      "tokens_seen": 2037317632
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003864548226608891,
+      "loss": 1.2718,
+      "theoretical_loss": 3.453561127643286,
+      "tokens_seen": 2037448704
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003864147006900979,
+      "loss": 1.3261,
+      "theoretical_loss": 3.4535428237352424,
+      "tokens_seen": 2037579776
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003863745787193067,
+      "loss": 1.2768,
+      "theoretical_loss": 3.453524521334261,
+      "tokens_seen": 2037710848
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003863344567485155,
+      "loss": 1.2724,
+      "theoretical_loss": 3.4535062204401203,
+      "tokens_seen": 2037841920
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003862943347777243,
+      "loss": 1.2715,
+      "theoretical_loss": 3.4534879210526004,
+      "tokens_seen": 2037972992
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003862542128069331,
+      "loss": 1.2574,
+      "theoretical_loss": 3.45346962317148,
+      "tokens_seen": 2038104064
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003862140908361419,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4534513267965385,
+      "tokens_seen": 2038235136
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038617396886535065,
+      "loss": 1.3003,
+      "theoretical_loss": 3.4534330319275544,
+      "tokens_seen": 2038366208
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003861338468945595,
+      "loss": 1.3071,
+      "theoretical_loss": 3.453414738564307,
+      "tokens_seen": 2038497280
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038609372492376826,
+      "loss": 1.2306,
+      "theoretical_loss": 3.4533964467065763,
+      "tokens_seen": 2038628352
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.000386053602952977,
+      "loss": 1.2528,
+      "theoretical_loss": 3.453378156354141,
+      "tokens_seen": 2038759424
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003860134809821859,
+      "loss": 1.3397,
+      "theoretical_loss": 3.453359867506781,
+      "tokens_seen": 2038890496
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038597335901139463,
+      "loss": 1.2999,
+      "theoretical_loss": 3.453341580164275,
+      "tokens_seen": 2039021568
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038593323704060343,
+      "loss": 1.3137,
+      "theoretical_loss": 3.4533232943264025,
+      "tokens_seen": 2039152640
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038589311506981224,
+      "loss": 1.1872,
+      "theoretical_loss": 3.4533050099929437,
+      "tokens_seen": 2039283712
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038585299309902105,
+      "loss": 1.3216,
+      "theoretical_loss": 3.453286727163677,
+      "tokens_seen": 2039414784
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038581287112822985,
+      "loss": 1.2855,
+      "theoretical_loss": 3.4532684458383835,
+      "tokens_seen": 2039545856
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003857727491574386,
+      "loss": 1.2426,
+      "theoretical_loss": 3.4532501660168418,
+      "tokens_seen": 2039676928
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003857326271866474,
+      "loss": 1.2519,
+      "theoretical_loss": 3.4532318876988324,
+      "tokens_seen": 2039808000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003856925052158562,
+      "loss": 1.2674,
+      "theoretical_loss": 3.453213610884134,
+      "tokens_seen": 2039939072
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.000385652383245065,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4531953355725267,
+      "tokens_seen": 2040070144
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003856122612742738,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4531770617637907,
+      "tokens_seen": 2040201216
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038557213930348264,
+      "loss": 1.2624,
+      "theoretical_loss": 3.4531587894577056,
+      "tokens_seen": 2040332288
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003855320173326914,
+      "loss": 1.305,
+      "theoretical_loss": 3.4531405186540516,
+      "tokens_seen": 2040463360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038549189536190014,
+      "loss": 1.3242,
+      "theoretical_loss": 3.453122249352608,
+      "tokens_seen": 2040594432
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.000385451773391109,
+      "loss": 1.2202,
+      "theoretical_loss": 3.4531039815531557,
+      "tokens_seen": 2040725504
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038541165142031775,
+      "loss": 1.2418,
+      "theoretical_loss": 3.4530857152554737,
+      "tokens_seen": 2040856576
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038537152944952656,
+      "loss": 1.283,
+      "theoretical_loss": 3.453067450459343,
+      "tokens_seen": 2040987648
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038533140747873537,
+      "loss": 1.2814,
+      "theoretical_loss": 3.4530491871645435,
+      "tokens_seen": 2041118720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038529128550794417,
+      "loss": 1.2821,
+      "theoretical_loss": 3.4530309253708555,
+      "tokens_seen": 2041249792
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003852511635371529,
+      "loss": 1.3024,
+      "theoretical_loss": 3.453012665078059,
+      "tokens_seen": 2041380864
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003852110415663618,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4529944062859346,
+      "tokens_seen": 2041511936
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038517091959557054,
+      "loss": 1.3001,
+      "theoretical_loss": 3.452976148994262,
+      "tokens_seen": 2041643008
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038513079762477934,
+      "loss": 1.2332,
+      "theoretical_loss": 3.4529578932028224,
+      "tokens_seen": 2041774080
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038509067565398815,
+      "loss": 1.2937,
+      "theoretical_loss": 3.4529396389113955,
+      "tokens_seen": 2041905152
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003850505536831969,
+      "loss": 1.366,
+      "theoretical_loss": 3.4529213861197627,
+      "tokens_seen": 2042036224
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038501043171240576,
+      "loss": 1.2921,
+      "theoretical_loss": 3.4529031348277033,
+      "tokens_seen": 2042167296
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003849703097416145,
+      "loss": 1.2423,
+      "theoretical_loss": 3.4528848850349987,
+      "tokens_seen": 2042298368
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003849301877708233,
+      "loss": 1.2974,
+      "theoretical_loss": 3.4528666367414296,
+      "tokens_seen": 2042429440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003848900658000321,
+      "loss": 1.24,
+      "theoretical_loss": 3.4528483899467757,
+      "tokens_seen": 2042560512
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003848499438292409,
+      "loss": 1.2387,
+      "theoretical_loss": 3.452830144650819,
+      "tokens_seen": 2042691584
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003848098218584497,
+      "loss": 1.2211,
+      "theoretical_loss": 3.452811900853339,
+      "tokens_seen": 2042822656
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003847696998876585,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4527936585541177,
+      "tokens_seen": 2042953728
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003847295779168673,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4527754177529353,
+      "tokens_seen": 2043084800
+    },
+    {
+      "epoch": 0.24,
+      "objective/train/docs_used": 686989,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3142861127853394,
+      "objective/train/theoretical_loss": 3.4527571784495725,
+      "objective/train/tokens_used": 413674976,
+      "theoretical_loss": 3.4527571784495725,
+      "tokens_seen": 2043215872
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038468945594607605,
+      "loss": 1.3413,
+      "theoretical_loss": 3.4527571784495725,
+      "tokens_seen": 2043215872
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003846493339752849,
+      "loss": 1.2959,
+      "theoretical_loss": 3.452738940643811,
+      "tokens_seen": 2043346944
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038460921200449366,
+      "loss": 1.3563,
+      "theoretical_loss": 3.45272070433543,
+      "tokens_seen": 2043478016
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003845690900337024,
+      "loss": 1.2122,
+      "theoretical_loss": 3.452702469524213,
+      "tokens_seen": 2043609088
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003845289680629113,
+      "loss": 1.3024,
+      "theoretical_loss": 3.4526842362099397,
+      "tokens_seen": 2043740160
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038448884609212,
+      "loss": 1.2062,
+      "theoretical_loss": 3.452666004392391,
+      "tokens_seen": 2043871232
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003844487241213289,
+      "loss": 1.2428,
+      "theoretical_loss": 3.452647774071349,
+      "tokens_seen": 2044002304
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038440860215053764,
+      "loss": 1.243,
+      "theoretical_loss": 3.4526295452465936,
+      "tokens_seen": 2044133376
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038436848017974645,
+      "loss": 1.2579,
+      "theoretical_loss": 3.452611317917907,
+      "tokens_seen": 2044264448
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038432835820895525,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4525930920850714,
+      "tokens_seen": 2044395520
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.000384288236238164,
+      "loss": 1.3152,
+      "theoretical_loss": 3.452574867747866,
+      "tokens_seen": 2044526592
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003842481142673728,
+      "loss": 1.2523,
+      "theoretical_loss": 3.452556644906074,
+      "tokens_seen": 2044657664
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003842079922965816,
+      "loss": 1.267,
+      "theoretical_loss": 3.452538423559475,
+      "tokens_seen": 2044788736
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003841678703257904,
+      "loss": 1.2487,
+      "theoretical_loss": 3.452520203707852,
+      "tokens_seen": 2044919808
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003841277483549992,
+      "loss": 1.2326,
+      "theoretical_loss": 3.452501985350987,
+      "tokens_seen": 2045050880
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038408762638420804,
+      "loss": 1.3069,
+      "theoretical_loss": 3.45248376848866,
+      "tokens_seen": 2045181952
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003840475044134168,
+      "loss": 1.3618,
+      "theoretical_loss": 3.4524655531206534,
+      "tokens_seen": 2045313024
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038400738244262554,
+      "loss": 1.3237,
+      "theoretical_loss": 3.4524473392467487,
+      "tokens_seen": 2045444096
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003839672604718344,
+      "loss": 1.3132,
+      "theoretical_loss": 3.452429126866728,
+      "tokens_seen": 2045575168
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038392713850104315,
+      "loss": 1.2888,
+      "theoretical_loss": 3.452410915980373,
+      "tokens_seen": 2045706240
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.000383887016530252,
+      "loss": 1.2662,
+      "theoretical_loss": 3.4523927065874647,
+      "tokens_seen": 2045837312
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038384689455946077,
+      "loss": 1.3328,
+      "theoretical_loss": 3.4523744986877856,
+      "tokens_seen": 2045968384
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038380677258866957,
+      "loss": 1.2205,
+      "theoretical_loss": 3.452356292281118,
+      "tokens_seen": 2046099456
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003837666506178784,
+      "loss": 1.2761,
+      "theoretical_loss": 3.452338087367243,
+      "tokens_seen": 2046230528
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038372652864708713,
+      "loss": 1.3239,
+      "theoretical_loss": 3.4523198839459432,
+      "tokens_seen": 2046361600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038368640667629594,
+      "loss": 1.2194,
+      "theoretical_loss": 3.4523016820170005,
+      "tokens_seen": 2046492672
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038364628470550474,
+      "loss": 1.1811,
+      "theoretical_loss": 3.4522834815801966,
+      "tokens_seen": 2046623744
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038360616273471355,
+      "loss": 1.2946,
+      "theoretical_loss": 3.4522652826353144,
+      "tokens_seen": 2046754816
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003835660407639223,
+      "loss": 1.3684,
+      "theoretical_loss": 3.4522470851821354,
+      "tokens_seen": 2046885888
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038352591879313116,
+      "loss": 1.249,
+      "theoretical_loss": 3.452228889220442,
+      "tokens_seen": 2047016960
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003834857968223399,
+      "loss": 1.3186,
+      "theoretical_loss": 3.4522106947500166,
+      "tokens_seen": 2047148032
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003834456748515487,
+      "loss": 1.2804,
+      "theoretical_loss": 3.452192501770641,
+      "tokens_seen": 2047279104
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003834055528807575,
+      "loss": 1.254,
+      "theoretical_loss": 3.4521743102820985,
+      "tokens_seen": 2047410176
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003833654309099663,
+      "loss": 1.1806,
+      "theoretical_loss": 3.4521561202841706,
+      "tokens_seen": 2047541248
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003833253089391751,
+      "loss": 1.26,
+      "theoretical_loss": 3.4521379317766403,
+      "tokens_seen": 2047672320
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003832851869683839,
+      "loss": 1.2856,
+      "theoretical_loss": 3.4521197447592904,
+      "tokens_seen": 2047803392
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003832450649975927,
+      "loss": 1.2789,
+      "theoretical_loss": 3.4521015592319024,
+      "tokens_seen": 2047934464
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003832049430268015,
+      "loss": 1.2422,
+      "theoretical_loss": 3.4520833751942597,
+      "tokens_seen": 2048065536
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003831648210560103,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4520651926461445,
+      "tokens_seen": 2048196608
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038312469908521906,
+      "loss": 1.284,
+      "theoretical_loss": 3.4520470115873403,
+      "tokens_seen": 2048327680
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038308457711442787,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4520288320176284,
+      "tokens_seen": 2048458752
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003830444551436367,
+      "loss": 1.2684,
+      "theoretical_loss": 3.4520106539367923,
+      "tokens_seen": 2048589824
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003830043331728454,
+      "loss": 1.2967,
+      "theoretical_loss": 3.451992477344615,
+      "tokens_seen": 2048720896
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003829642112020543,
+      "loss": 1.3953,
+      "theoretical_loss": 3.4519743022408798,
+      "tokens_seen": 2048851968
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038292408923126304,
+      "loss": 1.2687,
+      "theoretical_loss": 3.4519561286253686,
+      "tokens_seen": 2048983040
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038288396726047185,
+      "loss": 1.289,
+      "theoretical_loss": 3.451937956497865,
+      "tokens_seen": 2049114112
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038284384528968065,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4519197858581516,
+      "tokens_seen": 2049245184
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003828037233188894,
+      "loss": 1.3717,
+      "theoretical_loss": 3.451901616706012,
+      "tokens_seen": 2049376256
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003827636013480982,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4518834490412282,
+      "tokens_seen": 2049507328
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.000382723479377307,
+      "loss": 1.2113,
+      "theoretical_loss": 3.4518652828635847,
+      "tokens_seen": 2049638400
+    },
+    {
+      "epoch": 0.24,
+      "objective/train/docs_used": 689117,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1823278665542603,
+      "objective/train/theoretical_loss": 3.4518471181728634,
+      "objective/train/tokens_used": 420228576,
+      "theoretical_loss": 3.4518471181728634,
+      "tokens_seen": 2049769472
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003826833574065158,
+      "loss": 1.2106,
+      "theoretical_loss": 3.4518471181728634,
+      "tokens_seen": 2049769472
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003826432354357246,
+      "loss": 1.2458,
+      "theoretical_loss": 3.451828954968849,
+      "tokens_seen": 2049900544
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038260311346493344,
+      "loss": 1.3389,
+      "theoretical_loss": 3.4518107932513233,
+      "tokens_seen": 2050031616
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003825629914941422,
+      "loss": 1.2636,
+      "theoretical_loss": 3.4517926330200703,
+      "tokens_seen": 2050162688
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.000382522869523351,
+      "loss": 1.2477,
+      "theoretical_loss": 3.4517744742748735,
+      "tokens_seen": 2050293760
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003824827475525598,
+      "loss": 1.2708,
+      "theoretical_loss": 3.4517563170155157,
+      "tokens_seen": 2050424832
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038244262558176855,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4517381612417815,
+      "tokens_seen": 2050555904
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003824025036109774,
+      "loss": 1.3014,
+      "theoretical_loss": 3.451720006953453,
+      "tokens_seen": 2050686976
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038236238164018616,
+      "loss": 1.252,
+      "theoretical_loss": 3.451701854150315,
+      "tokens_seen": 2050818048
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038232225966939497,
+      "loss": 1.3311,
+      "theoretical_loss": 3.45168370283215,
+      "tokens_seen": 2050949120
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003822821376986038,
+      "loss": 1.2057,
+      "theoretical_loss": 3.451665552998742,
+      "tokens_seen": 2051080192
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038224201572781253,
+      "loss": 1.2415,
+      "theoretical_loss": 3.451647404649875,
+      "tokens_seen": 2051211264
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038220189375702134,
+      "loss": 1.2759,
+      "theoretical_loss": 3.4516292577853322,
+      "tokens_seen": 2051342336
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038216177178623014,
+      "loss": 1.2833,
+      "theoretical_loss": 3.4516111124048985,
+      "tokens_seen": 2051473408
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038212164981543895,
+      "loss": 1.3069,
+      "theoretical_loss": 3.451592968508356,
+      "tokens_seen": 2051604480
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003820815278446477,
+      "loss": 1.3005,
+      "theoretical_loss": 3.45157482609549,
+      "tokens_seen": 2051735552
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038204140587385656,
+      "loss": 1.2687,
+      "theoretical_loss": 3.451556685166084,
+      "tokens_seen": 2051866624
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003820012839030653,
+      "loss": 1.371,
+      "theoretical_loss": 3.4515385457199215,
+      "tokens_seen": 2051997696
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003819611619322742,
+      "loss": 1.2711,
+      "theoretical_loss": 3.451520407756787,
+      "tokens_seen": 2052128768
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003819210399614829,
+      "loss": 1.3074,
+      "theoretical_loss": 3.451502271276464,
+      "tokens_seen": 2052259840
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003818809179906917,
+      "loss": 1.2615,
+      "theoretical_loss": 3.4514841362787374,
+      "tokens_seen": 2052390912
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038184079601990054,
+      "loss": 1.381,
+      "theoretical_loss": 3.4514660027633908,
+      "tokens_seen": 2052521984
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003818006740491093,
+      "loss": 1.2062,
+      "theoretical_loss": 3.4514478707302083,
+      "tokens_seen": 2052653056
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003817605520783181,
+      "loss": 1.2046,
+      "theoretical_loss": 3.451429740178974,
+      "tokens_seen": 2052784128
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003817204301075269,
+      "loss": 1.3089,
+      "theoretical_loss": 3.451411611109473,
+      "tokens_seen": 2052915200
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003816803081367357,
+      "loss": 1.3029,
+      "theoretical_loss": 3.451393483521489,
+      "tokens_seen": 2053046272
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038164018616594446,
+      "loss": 1.2641,
+      "theoretical_loss": 3.451375357414806,
+      "tokens_seen": 2053177344
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038160006419515327,
+      "loss": 1.253,
+      "theoretical_loss": 3.451357232789209,
+      "tokens_seen": 2053308416
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003815599422243621,
+      "loss": 1.2477,
+      "theoretical_loss": 3.4513391096444823,
+      "tokens_seen": 2053439488
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003815198202535708,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4513209879804103,
+      "tokens_seen": 2053570560
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003814796982827797,
+      "loss": 1.4032,
+      "theoretical_loss": 3.4513028677967776,
+      "tokens_seen": 2053701632
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038143957631198844,
+      "loss": 1.2598,
+      "theoretical_loss": 3.451284749093369,
+      "tokens_seen": 2053832704
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038139945434119725,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4512666318699683,
+      "tokens_seen": 2053963776
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038135933237040605,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4512485161263613,
+      "tokens_seen": 2054094848
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003813192103996148,
+      "loss": 1.2476,
+      "theoretical_loss": 3.451230401862332,
+      "tokens_seen": 2054225920
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038127908842882366,
+      "loss": 1.3448,
+      "theoretical_loss": 3.4512122890776658,
+      "tokens_seen": 2054356992
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003812389664580324,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4511941777721464,
+      "tokens_seen": 2054488064
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003811988444872412,
+      "loss": 1.2751,
+      "theoretical_loss": 3.4511760679455596,
+      "tokens_seen": 2054619136
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038115872251645003,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4511579595976896,
+      "tokens_seen": 2054750208
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038111860054565884,
+      "loss": 1.2049,
+      "theoretical_loss": 3.451139852728322,
+      "tokens_seen": 2054881280
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003810784785748676,
+      "loss": 1.2679,
+      "theoretical_loss": 3.451121747337242,
+      "tokens_seen": 2055012352
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003810383566040764,
+      "loss": 1.3762,
+      "theoretical_loss": 3.4511036434242337,
+      "tokens_seen": 2055143424
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003809982346332852,
+      "loss": 1.3409,
+      "theoretical_loss": 3.451085540989083,
+      "tokens_seen": 2055274496
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038095811266249395,
+      "loss": 1.2602,
+      "theoretical_loss": 3.451067440031575,
+      "tokens_seen": 2055405568
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003809179906917028,
+      "loss": 1.259,
+      "theoretical_loss": 3.4510493405514935,
+      "tokens_seen": 2055536640
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038087786872091156,
+      "loss": 1.303,
+      "theoretical_loss": 3.451031242548625,
+      "tokens_seen": 2055667712
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038083774675012037,
+      "loss": 1.2317,
+      "theoretical_loss": 3.4510131460227553,
+      "tokens_seen": 2055798784
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003807976247793292,
+      "loss": 1.2399,
+      "theoretical_loss": 3.450995050973668,
+      "tokens_seen": 2055929856
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038075750280853793,
+      "loss": 1.2446,
+      "theoretical_loss": 3.45097695740115,
+      "tokens_seen": 2056060928
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003807173808377468,
+      "loss": 1.3403,
+      "theoretical_loss": 3.450958865304986,
+      "tokens_seen": 2056192000
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 691218,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3150357007980347,
+      "objective/train/theoretical_loss": 3.450940774684961,
+      "objective/train/tokens_used": 426782176,
+      "theoretical_loss": 3.450940774684961,
+      "tokens_seen": 2056323072
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038067725886695554,
+      "loss": 1.274,
+      "theoretical_loss": 3.450940774684961,
+      "tokens_seen": 2056323072
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038063713689616435,
+      "loss": 1.1861,
+      "theoretical_loss": 3.4509226855408617,
+      "tokens_seen": 2056454144
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038059701492537315,
+      "loss": 1.2292,
+      "theoretical_loss": 3.4509045978724724,
+      "tokens_seen": 2056585216
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038055689295458196,
+      "loss": 1.2396,
+      "theoretical_loss": 3.4508865116795793,
+      "tokens_seen": 2056716288
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003805167709837907,
+      "loss": 1.2636,
+      "theoretical_loss": 3.450868426961968,
+      "tokens_seen": 2056847360
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003804766490129996,
+      "loss": 1.3032,
+      "theoretical_loss": 3.450850343719424,
+      "tokens_seen": 2056978432
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003804365270422083,
+      "loss": 1.3117,
+      "theoretical_loss": 3.4508322619517333,
+      "tokens_seen": 2057109504
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003803964050714171,
+      "loss": 1.2991,
+      "theoretical_loss": 3.450814181658681,
+      "tokens_seen": 2057240576
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038035628310062594,
+      "loss": 1.3728,
+      "theoretical_loss": 3.4507961028400533,
+      "tokens_seen": 2057371648
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003803161611298347,
+      "loss": 1.2972,
+      "theoretical_loss": 3.4507780254956364,
+      "tokens_seen": 2057502720
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003802760391590435,
+      "loss": 1.298,
+      "theoretical_loss": 3.4507599496252155,
+      "tokens_seen": 2057633792
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003802359171882523,
+      "loss": 1.2414,
+      "theoretical_loss": 3.450741875228577,
+      "tokens_seen": 2057764864
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003801957952174611,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4507238023055065,
+      "tokens_seen": 2057895936
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038015567324666986,
+      "loss": 1.2566,
+      "theoretical_loss": 3.450705730855791,
+      "tokens_seen": 2058027008
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038011555127587867,
+      "loss": 1.2233,
+      "theoretical_loss": 3.450687660879215,
+      "tokens_seen": 2058158080
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003800754293050875,
+      "loss": 1.2501,
+      "theoretical_loss": 3.450669592375566,
+      "tokens_seen": 2058289152
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003800353073342963,
+      "loss": 1.3006,
+      "theoretical_loss": 3.4506515253446293,
+      "tokens_seen": 2058420224
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003799951853635051,
+      "loss": 1.1987,
+      "theoretical_loss": 3.4506334597861916,
+      "tokens_seen": 2058551296
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037995506339271384,
+      "loss": 1.1896,
+      "theoretical_loss": 3.450615395700039,
+      "tokens_seen": 2058682368
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003799149414219227,
+      "loss": 1.2343,
+      "theoretical_loss": 3.450597333085957,
+      "tokens_seen": 2058813440
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037987481945113145,
+      "loss": 1.2726,
+      "theoretical_loss": 3.4505792719437336,
+      "tokens_seen": 2058944512
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003798346974803402,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4505612122731537,
+      "tokens_seen": 2059075584
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037979457550954906,
+      "loss": 1.2628,
+      "theoretical_loss": 3.4505431540740044,
+      "tokens_seen": 2059206656
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003797544535387578,
+      "loss": 1.2196,
+      "theoretical_loss": 3.450525097346072,
+      "tokens_seen": 2059337728
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003797143315679666,
+      "loss": 1.2662,
+      "theoretical_loss": 3.450507042089143,
+      "tokens_seen": 2059468800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037967420959717543,
+      "loss": 1.2892,
+      "theoretical_loss": 3.450488988303004,
+      "tokens_seen": 2059599872
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037963408762638423,
+      "loss": 1.3215,
+      "theoretical_loss": 3.4504709359874415,
+      "tokens_seen": 2059730944
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000379593965655593,
+      "loss": 1.2654,
+      "theoretical_loss": 3.4504528851422425,
+      "tokens_seen": 2059862016
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003795538436848018,
+      "loss": 1.284,
+      "theoretical_loss": 3.450434835767193,
+      "tokens_seen": 2059993088
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003795137217140106,
+      "loss": 1.1707,
+      "theoretical_loss": 3.45041678786208,
+      "tokens_seen": 2060124160
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037947359974321935,
+      "loss": 1.2622,
+      "theoretical_loss": 3.450398741426691,
+      "tokens_seen": 2060255232
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003794334777724282,
+      "loss": 1.2485,
+      "theoretical_loss": 3.4503806964608117,
+      "tokens_seen": 2060386304
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037939335580163696,
+      "loss": 1.2289,
+      "theoretical_loss": 3.45036265296423,
+      "tokens_seen": 2060517376
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003793532338308458,
+      "loss": 1.2839,
+      "theoretical_loss": 3.450344610936732,
+      "tokens_seen": 2060648448
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003793131118600546,
+      "loss": 1.2984,
+      "theoretical_loss": 3.450326570378105,
+      "tokens_seen": 2060779520
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037927298988926333,
+      "loss": 1.2371,
+      "theoretical_loss": 3.4503085312881367,
+      "tokens_seen": 2060910592
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003792328679184722,
+      "loss": 1.3247,
+      "theoretical_loss": 3.4502904936666123,
+      "tokens_seen": 2061041664
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037919274594768094,
+      "loss": 1.2806,
+      "theoretical_loss": 3.4502724575133206,
+      "tokens_seen": 2061172736
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037915262397688975,
+      "loss": 1.3317,
+      "theoretical_loss": 3.4502544228280483,
+      "tokens_seen": 2061303808
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037911250200609855,
+      "loss": 1.2935,
+      "theoretical_loss": 3.4502363896105823,
+      "tokens_seen": 2061434880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037907238003530736,
+      "loss": 1.2551,
+      "theoretical_loss": 3.4502183578607095,
+      "tokens_seen": 2061565952
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003790322580645161,
+      "loss": 1.2589,
+      "theoretical_loss": 3.4502003275782185,
+      "tokens_seen": 2061697024
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003789921360937249,
+      "loss": 1.2344,
+      "theoretical_loss": 3.450182298762895,
+      "tokens_seen": 2061828096
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003789520141229337,
+      "loss": 1.2263,
+      "theoretical_loss": 3.4501642714145273,
+      "tokens_seen": 2061959168
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003789118921521425,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4501462455329026,
+      "tokens_seen": 2062090240
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037887177018135134,
+      "loss": 1.2811,
+      "theoretical_loss": 3.4501282211178084,
+      "tokens_seen": 2062221312
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003788316482105601,
+      "loss": 1.3112,
+      "theoretical_loss": 3.450110198169032,
+      "tokens_seen": 2062352384
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037879152623976895,
+      "loss": 1.1942,
+      "theoretical_loss": 3.4500921766863613,
+      "tokens_seen": 2062483456
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003787514042689777,
+      "loss": 1.3156,
+      "theoretical_loss": 3.450074156669583,
+      "tokens_seen": 2062614528
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003787112822981865,
+      "loss": 1.318,
+      "theoretical_loss": 3.4500561381184855,
+      "tokens_seen": 2062745600
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 693406,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3298795223236084,
+      "objective/train/theoretical_loss": 3.450038121032857,
+      "objective/train/tokens_used": 433335776,
+      "theoretical_loss": 3.450038121032857,
+      "tokens_seen": 2062876672
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003786711603273953,
+      "loss": 1.3291,
+      "theoretical_loss": 3.450038121032857,
+      "tokens_seen": 2062876672
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037863103835660407,
+      "loss": 1.3109,
+      "theoretical_loss": 3.450020105412484,
+      "tokens_seen": 2063007744
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003785909163858129,
+      "loss": 1.2442,
+      "theoretical_loss": 3.4500020912571547,
+      "tokens_seen": 2063138816
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003785507944150217,
+      "loss": 1.3179,
+      "theoretical_loss": 3.449984078566657,
+      "tokens_seen": 2063269888
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003785106724442305,
+      "loss": 1.2777,
+      "theoretical_loss": 3.449966067340779,
+      "tokens_seen": 2063400960
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037847055047343924,
+      "loss": 1.2136,
+      "theoretical_loss": 3.4499480575793084,
+      "tokens_seen": 2063532032
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003784304285026481,
+      "loss": 1.2345,
+      "theoretical_loss": 3.449930049282033,
+      "tokens_seen": 2063663104
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037839030653185685,
+      "loss": 1.3677,
+      "theoretical_loss": 3.44991204244874,
+      "tokens_seen": 2063794176
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003783501845610656,
+      "loss": 1.2723,
+      "theoretical_loss": 3.449894037079219,
+      "tokens_seen": 2063925248
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037831006259027446,
+      "loss": 1.2763,
+      "theoretical_loss": 3.4498760331732568,
+      "tokens_seen": 2064056320
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003782699406194832,
+      "loss": 1.3031,
+      "theoretical_loss": 3.4498580307306423,
+      "tokens_seen": 2064187392
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000378229818648692,
+      "loss": 1.2849,
+      "theoretical_loss": 3.4498400297511633,
+      "tokens_seen": 2064318464
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037818969667790083,
+      "loss": 1.2773,
+      "theoretical_loss": 3.449822030234608,
+      "tokens_seen": 2064449536
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037814957470710963,
+      "loss": 1.289,
+      "theoretical_loss": 3.449804032180765,
+      "tokens_seen": 2064580608
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037810945273631844,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4497860355894217,
+      "tokens_seen": 2064711680
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003780693307655272,
+      "loss": 1.2217,
+      "theoretical_loss": 3.4497680404603672,
+      "tokens_seen": 2064842752
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000378029208794736,
+      "loss": 1.2509,
+      "theoretical_loss": 3.44975004679339,
+      "tokens_seen": 2064973824
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003779890868239448,
+      "loss": 1.237,
+      "theoretical_loss": 3.449732054588278,
+      "tokens_seen": 2065104896
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003779489648531536,
+      "loss": 1.2885,
+      "theoretical_loss": 3.4497140638448194,
+      "tokens_seen": 2065235968
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037790884288236236,
+      "loss": 1.2472,
+      "theoretical_loss": 3.449696074562804,
+      "tokens_seen": 2065367040
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003778687209115712,
+      "loss": 1.2421,
+      "theoretical_loss": 3.449678086742019,
+      "tokens_seen": 2065498112
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037782859894078,
+      "loss": 1.3242,
+      "theoretical_loss": 3.4496601003822533,
+      "tokens_seen": 2065629184
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037778847696998873,
+      "loss": 1.3286,
+      "theoretical_loss": 3.4496421154832957,
+      "tokens_seen": 2065760256
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003777483549991976,
+      "loss": 1.3081,
+      "theoretical_loss": 3.449624132044935,
+      "tokens_seen": 2065891328
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037770823302840634,
+      "loss": 1.2526,
+      "theoretical_loss": 3.4496061500669604,
+      "tokens_seen": 2066022400
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037766811105761515,
+      "loss": 1.3294,
+      "theoretical_loss": 3.4495881695491595,
+      "tokens_seen": 2066153472
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037762798908682395,
+      "loss": 1.2963,
+      "theoretical_loss": 3.449570190491322,
+      "tokens_seen": 2066284544
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037758786711603276,
+      "loss": 1.3174,
+      "theoretical_loss": 3.449552212893236,
+      "tokens_seen": 2066415616
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003775477451452415,
+      "loss": 1.1944,
+      "theoretical_loss": 3.4495342367546913,
+      "tokens_seen": 2066546688
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003775076231744503,
+      "loss": 1.2906,
+      "theoretical_loss": 3.449516262075476,
+      "tokens_seen": 2066677760
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003774675012036591,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4494982888553802,
+      "tokens_seen": 2066808832
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037742737923286793,
+      "loss": 1.2249,
+      "theoretical_loss": 3.449480317094192,
+      "tokens_seen": 2066939904
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037738725726207674,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4494623467917003,
+      "tokens_seen": 2067070976
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003773471352912855,
+      "loss": 1.2647,
+      "theoretical_loss": 3.449444377947695,
+      "tokens_seen": 2067202048
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037730701332049435,
+      "loss": 1.218,
+      "theoretical_loss": 3.4494264105619648,
+      "tokens_seen": 2067333120
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003772668913497031,
+      "loss": 1.2924,
+      "theoretical_loss": 3.4494084446342983,
+      "tokens_seen": 2067464192
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003772267693789119,
+      "loss": 1.2775,
+      "theoretical_loss": 3.449390480164486,
+      "tokens_seen": 2067595264
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003771866474081207,
+      "loss": 1.2932,
+      "theoretical_loss": 3.449372517152317,
+      "tokens_seen": 2067726336
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037714652543732947,
+      "loss": 1.3169,
+      "theoretical_loss": 3.44935455559758,
+      "tokens_seen": 2067857408
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003771064034665383,
+      "loss": 1.2993,
+      "theoretical_loss": 3.4493365955000646,
+      "tokens_seen": 2067988480
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003770662814957471,
+      "loss": 1.2365,
+      "theoretical_loss": 3.4493186368595605,
+      "tokens_seen": 2068119552
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003770261595249559,
+      "loss": 1.2648,
+      "theoretical_loss": 3.4493006796758565,
+      "tokens_seen": 2068250624
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037698603755416464,
+      "loss": 1.2346,
+      "theoretical_loss": 3.449282723948743,
+      "tokens_seen": 2068381696
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003769459155833735,
+      "loss": 1.3223,
+      "theoretical_loss": 3.4492647696780088,
+      "tokens_seen": 2068512768
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037690579361258225,
+      "loss": 1.2695,
+      "theoretical_loss": 3.449246816863444,
+      "tokens_seen": 2068643840
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037686567164179106,
+      "loss": 1.2093,
+      "theoretical_loss": 3.4492288655048378,
+      "tokens_seen": 2068774912
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037682554967099986,
+      "loss": 1.3198,
+      "theoretical_loss": 3.4492109156019803,
+      "tokens_seen": 2068905984
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003767854277002086,
+      "loss": 1.2825,
+      "theoretical_loss": 3.4491929671546613,
+      "tokens_seen": 2069037056
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003767453057294175,
+      "loss": 1.2091,
+      "theoretical_loss": 3.44917502016267,
+      "tokens_seen": 2069168128
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037670518375862623,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4491570746257967,
+      "tokens_seen": 2069299200
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 695669,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2434518337249756,
+      "objective/train/theoretical_loss": 3.4491391305438315,
+      "objective/train/tokens_used": 439889376,
+      "theoretical_loss": 3.4491391305438315,
+      "tokens_seen": 2069430272
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037666506178783503,
+      "loss": 1.2309,
+      "theoretical_loss": 3.4491391305438315,
+      "tokens_seen": 2069430272
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037662493981704384,
+      "loss": 1.3162,
+      "theoretical_loss": 3.449121187916564,
+      "tokens_seen": 2069561344
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003765848178462526,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4491032467437837,
+      "tokens_seen": 2069692416
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003765446958754614,
+      "loss": 1.2734,
+      "theoretical_loss": 3.449085307025281,
+      "tokens_seen": 2069823488
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003765045739046702,
+      "loss": 1.2469,
+      "theoretical_loss": 3.449067368760846,
+      "tokens_seen": 2069954560
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000376464451933879,
+      "loss": 1.2963,
+      "theoretical_loss": 3.449049431950269,
+      "tokens_seen": 2070085632
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037642432996308776,
+      "loss": 1.257,
+      "theoretical_loss": 3.4490314965933395,
+      "tokens_seen": 2070216704
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003763842079922966,
+      "loss": 1.2486,
+      "theoretical_loss": 3.449013562689848,
+      "tokens_seen": 2070347776
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003763440860215054,
+      "loss": 1.2328,
+      "theoretical_loss": 3.448995630239585,
+      "tokens_seen": 2070478848
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037630396405071413,
+      "loss": 1.295,
+      "theoretical_loss": 3.44897769924234,
+      "tokens_seen": 2070609920
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000376263842079923,
+      "loss": 1.3685,
+      "theoretical_loss": 3.448959769697904,
+      "tokens_seen": 2070740992
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037622372010913174,
+      "loss": 1.2293,
+      "theoretical_loss": 3.448941841606067,
+      "tokens_seen": 2070872064
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003761835981383406,
+      "loss": 1.258,
+      "theoretical_loss": 3.4489239149666204,
+      "tokens_seen": 2071003136
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037614347616754935,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4489059897793526,
+      "tokens_seen": 2071134208
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037610335419675816,
+      "loss": 1.2647,
+      "theoretical_loss": 3.4488880660440557,
+      "tokens_seen": 2071265280
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037606323222596697,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4488701437605203,
+      "tokens_seen": 2071396352
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003760231102551757,
+      "loss": 1.2715,
+      "theoretical_loss": 3.4488522229285357,
+      "tokens_seen": 2071527424
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003759829882843845,
+      "loss": 1.2961,
+      "theoretical_loss": 3.448834303547893,
+      "tokens_seen": 2071658496
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037594286631359333,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4488163856183833,
+      "tokens_seen": 2071789568
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037590274434280214,
+      "loss": 1.2571,
+      "theoretical_loss": 3.4487984691397973,
+      "tokens_seen": 2071920640
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003758626223720109,
+      "loss": 1.2177,
+      "theoretical_loss": 3.4487805541119254,
+      "tokens_seen": 2072051712
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037582250040121975,
+      "loss": 1.3407,
+      "theoretical_loss": 3.448762640534558,
+      "tokens_seen": 2072182784
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003757823784304285,
+      "loss": 1.1549,
+      "theoretical_loss": 3.448744728407487,
+      "tokens_seen": 2072313856
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003757422564596373,
+      "loss": 1.323,
+      "theoretical_loss": 3.4487268177305017,
+      "tokens_seen": 2072444928
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003757021344888461,
+      "loss": 1.2779,
+      "theoretical_loss": 3.4487089085033946,
+      "tokens_seen": 2072576000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037566201251805487,
+      "loss": 1.2124,
+      "theoretical_loss": 3.448691000725956,
+      "tokens_seen": 2072707072
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003756218905472637,
+      "loss": 1.2625,
+      "theoretical_loss": 3.448673094397977,
+      "tokens_seen": 2072838144
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003755817685764725,
+      "loss": 1.2695,
+      "theoretical_loss": 3.448655189519248,
+      "tokens_seen": 2072969216
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003755416466056813,
+      "loss": 1.2683,
+      "theoretical_loss": 3.4486372860895607,
+      "tokens_seen": 2073100288
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003755015246348901,
+      "loss": 1.2828,
+      "theoretical_loss": 3.4486193841087065,
+      "tokens_seen": 2073231360
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003754614026640989,
+      "loss": 1.2288,
+      "theoretical_loss": 3.448601483576476,
+      "tokens_seen": 2073362432
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037542128069330765,
+      "loss": 1.2745,
+      "theoretical_loss": 3.4485835844926607,
+      "tokens_seen": 2073493504
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037538115872251646,
+      "loss": 1.2986,
+      "theoretical_loss": 3.4485656868570516,
+      "tokens_seen": 2073624576
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037534103675172526,
+      "loss": 1.2169,
+      "theoretical_loss": 3.4485477906694397,
+      "tokens_seen": 2073755648
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000375300914780934,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4485298959296173,
+      "tokens_seen": 2073886720
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003752607928101429,
+      "loss": 1.2168,
+      "theoretical_loss": 3.4485120026373757,
+      "tokens_seen": 2074017792
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037522067083935163,
+      "loss": 1.2803,
+      "theoretical_loss": 3.448494110792505,
+      "tokens_seen": 2074148864
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037518054886856043,
+      "loss": 1.2781,
+      "theoretical_loss": 3.4484762203947987,
+      "tokens_seen": 2074279936
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037514042689776924,
+      "loss": 1.2548,
+      "theoretical_loss": 3.448458331444046,
+      "tokens_seen": 2074411008
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000375100304926978,
+      "loss": 1.2377,
+      "theoretical_loss": 3.4484404439400405,
+      "tokens_seen": 2074542080
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003750601829561868,
+      "loss": 1.2561,
+      "theoretical_loss": 3.4484225578825725,
+      "tokens_seen": 2074673152
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003750200609853956,
+      "loss": 1.2777,
+      "theoretical_loss": 3.448404673271434,
+      "tokens_seen": 2074804224
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003749799390146044,
+      "loss": 1.2594,
+      "theoretical_loss": 3.4483867901064174,
+      "tokens_seen": 2074935296
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003749398170438132,
+      "loss": 1.3567,
+      "theoretical_loss": 3.4483689083873132,
+      "tokens_seen": 2075066368
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000374899695073022,
+      "loss": 1.2843,
+      "theoretical_loss": 3.448351028113914,
+      "tokens_seen": 2075197440
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003748595731022308,
+      "loss": 1.2335,
+      "theoretical_loss": 3.4483331492860114,
+      "tokens_seen": 2075328512
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003748194511314396,
+      "loss": 1.2877,
+      "theoretical_loss": 3.4483152719033976,
+      "tokens_seen": 2075459584
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003747793291606484,
+      "loss": 1.3088,
+      "theoretical_loss": 3.448297395965864,
+      "tokens_seen": 2075590656
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037473920718985714,
+      "loss": 1.2577,
+      "theoretical_loss": 3.448279521473202,
+      "tokens_seen": 2075721728
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000374699085219066,
+      "loss": 1.2371,
+      "theoretical_loss": 3.4482616484252047,
+      "tokens_seen": 2075852800
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 697911,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.220463752746582,
+      "objective/train/theoretical_loss": 3.4482437768216636,
+      "objective/train/tokens_used": 446442976,
+      "theoretical_loss": 3.4482437768216636,
+      "tokens_seen": 2075983872
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037465896324827475,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4482437768216636,
+      "tokens_seen": 2075983872
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037461884127748356,
+      "loss": 1.236,
+      "theoretical_loss": 3.4482259066623717,
+      "tokens_seen": 2076114944
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037457871930669237,
+      "loss": 1.3026,
+      "theoretical_loss": 3.4482080379471194,
+      "tokens_seen": 2076246016
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003745385973359011,
+      "loss": 1.3109,
+      "theoretical_loss": 3.4481901706757,
+      "tokens_seen": 2076377088
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003744984753651099,
+      "loss": 1.2181,
+      "theoretical_loss": 3.4481723048479056,
+      "tokens_seen": 2076508160
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037445835339431873,
+      "loss": 1.2734,
+      "theoretical_loss": 3.4481544404635285,
+      "tokens_seen": 2076639232
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037441823142352754,
+      "loss": 1.2435,
+      "theoretical_loss": 3.4481365775223605,
+      "tokens_seen": 2076770304
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003743781094527363,
+      "loss": 1.3142,
+      "theoretical_loss": 3.4481187160241946,
+      "tokens_seen": 2076901376
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037433798748194515,
+      "loss": 1.293,
+      "theoretical_loss": 3.448100855968822,
+      "tokens_seen": 2077032448
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003742978655111539,
+      "loss": 1.2873,
+      "theoretical_loss": 3.4480829973560367,
+      "tokens_seen": 2077163520
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003742577435403627,
+      "loss": 1.2699,
+      "theoretical_loss": 3.4480651401856304,
+      "tokens_seen": 2077294592
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003742176215695715,
+      "loss": 1.2831,
+      "theoretical_loss": 3.4480472844573953,
+      "tokens_seen": 2077425664
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037417749959878027,
+      "loss": 1.2584,
+      "theoretical_loss": 3.4480294301711245,
+      "tokens_seen": 2077556736
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003741373776279891,
+      "loss": 1.3119,
+      "theoretical_loss": 3.4480115773266102,
+      "tokens_seen": 2077687808
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003740972556571979,
+      "loss": 1.295,
+      "theoretical_loss": 3.447993725923645,
+      "tokens_seen": 2077818880
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003740571336864067,
+      "loss": 1.2819,
+      "theoretical_loss": 3.447975875962022,
+      "tokens_seen": 2077949952
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003740170117156155,
+      "loss": 1.2189,
+      "theoretical_loss": 3.447958027441534,
+      "tokens_seen": 2078081024
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003739768897448243,
+      "loss": 1.2727,
+      "theoretical_loss": 3.447940180361973,
+      "tokens_seen": 2078212096
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037393676777403305,
+      "loss": 1.2887,
+      "theoretical_loss": 3.447922334723132,
+      "tokens_seen": 2078343168
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037389664580324186,
+      "loss": 1.25,
+      "theoretical_loss": 3.4479044905248046,
+      "tokens_seen": 2078474240
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037385652383245066,
+      "loss": 1.24,
+      "theoretical_loss": 3.447886647766783,
+      "tokens_seen": 2078605312
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003738164018616594,
+      "loss": 1.2597,
+      "theoretical_loss": 3.4478688064488603,
+      "tokens_seen": 2078736384
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003737762798908683,
+      "loss": 1.2775,
+      "theoretical_loss": 3.4478509665708295,
+      "tokens_seen": 2078867456
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000373736157920077,
+      "loss": 1.194,
+      "theoretical_loss": 3.4478331281324834,
+      "tokens_seen": 2078998528
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003736960359492859,
+      "loss": 1.2793,
+      "theoretical_loss": 3.4478152911336157,
+      "tokens_seen": 2079129600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037365591397849464,
+      "loss": 1.2748,
+      "theoretical_loss": 3.447797455574019,
+      "tokens_seen": 2079260672
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003736157920077034,
+      "loss": 1.2551,
+      "theoretical_loss": 3.447779621453486,
+      "tokens_seen": 2079391744
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037357567003691225,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4477617887718113,
+      "tokens_seen": 2079522816
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000373535548066121,
+      "loss": 1.2383,
+      "theoretical_loss": 3.4477439575287865,
+      "tokens_seen": 2079653888
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003734954260953298,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4477261277242057,
+      "tokens_seen": 2079784960
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003734553041245386,
+      "loss": 1.2647,
+      "theoretical_loss": 3.4477082993578625,
+      "tokens_seen": 2079916032
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003734151821537474,
+      "loss": 1.2706,
+      "theoretical_loss": 3.4476904724295494,
+      "tokens_seen": 2080047104
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003733750601829562,
+      "loss": 1.2914,
+      "theoretical_loss": 3.447672646939061,
+      "tokens_seen": 2080178176
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000373334938212165,
+      "loss": 1.2646,
+      "theoretical_loss": 3.4476548228861894,
+      "tokens_seen": 2080309248
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003732948162413738,
+      "loss": 1.3158,
+      "theoretical_loss": 3.447637000270729,
+      "tokens_seen": 2080440320
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037325469427058254,
+      "loss": 1.1944,
+      "theoretical_loss": 3.447619179092473,
+      "tokens_seen": 2080571392
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003732145722997914,
+      "loss": 1.2217,
+      "theoretical_loss": 3.447601359351215,
+      "tokens_seen": 2080702464
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037317445032900015,
+      "loss": 1.2266,
+      "theoretical_loss": 3.447583541046749,
+      "tokens_seen": 2080833536
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037313432835820896,
+      "loss": 1.2351,
+      "theoretical_loss": 3.447565724178868,
+      "tokens_seen": 2080964608
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037309420638741777,
+      "loss": 1.3156,
+      "theoretical_loss": 3.4475479087473664,
+      "tokens_seen": 2081095680
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003730540844166265,
+      "loss": 1.2886,
+      "theoretical_loss": 3.447530094752037,
+      "tokens_seen": 2081226752
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003730139624458354,
+      "loss": 1.3365,
+      "theoretical_loss": 3.4475122821926742,
+      "tokens_seen": 2081357824
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037297384047504413,
+      "loss": 1.1976,
+      "theoretical_loss": 3.4474944710690725,
+      "tokens_seen": 2081488896
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037293371850425294,
+      "loss": 1.1991,
+      "theoretical_loss": 3.4474766613810246,
+      "tokens_seen": 2081619968
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037289359653346174,
+      "loss": 1.2606,
+      "theoretical_loss": 3.4474588531283246,
+      "tokens_seen": 2081751040
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037285347456267055,
+      "loss": 1.2234,
+      "theoretical_loss": 3.447441046310767,
+      "tokens_seen": 2081882112
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003728133525918793,
+      "loss": 1.2862,
+      "theoretical_loss": 3.4474232409281456,
+      "tokens_seen": 2082013184
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003727732306210881,
+      "loss": 1.2367,
+      "theoretical_loss": 3.4474054369802545,
+      "tokens_seen": 2082144256
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003727331086502969,
+      "loss": 1.3164,
+      "theoretical_loss": 3.4473876344668875,
+      "tokens_seen": 2082275328
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037269298667950567,
+      "loss": 1.2982,
+      "theoretical_loss": 3.447369833387839,
+      "tokens_seen": 2082406400
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 700019,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2040245532989502,
+      "objective/train/theoretical_loss": 3.447352033742903,
+      "objective/train/tokens_used": 452996576,
+      "theoretical_loss": 3.447352033742903,
+      "tokens_seen": 2082537472
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003726528647087145,
+      "loss": 1.2779,
+      "theoretical_loss": 3.447352033742903,
+      "tokens_seen": 2082537472
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003726127427379233,
+      "loss": 1.2438,
+      "theoretical_loss": 3.4473342355318737,
+      "tokens_seen": 2082668544
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003725726207671321,
+      "loss": 1.2366,
+      "theoretical_loss": 3.4473164387545454,
+      "tokens_seen": 2082799616
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003725324987963409,
+      "loss": 1.2835,
+      "theoretical_loss": 3.4472986434107122,
+      "tokens_seen": 2082930688
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003724923768255497,
+      "loss": 1.3018,
+      "theoretical_loss": 3.447280849500169,
+      "tokens_seen": 2083061760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003724522548547585,
+      "loss": 1.2636,
+      "theoretical_loss": 3.44726305702271,
+      "tokens_seen": 2083192832
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037241213288396726,
+      "loss": 1.3068,
+      "theoretical_loss": 3.4472452659781294,
+      "tokens_seen": 2083323904
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037237201091317606,
+      "loss": 1.2968,
+      "theoretical_loss": 3.447227476366222,
+      "tokens_seen": 2083454976
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037233188894238487,
+      "loss": 1.2419,
+      "theoretical_loss": 3.4472096881867813,
+      "tokens_seen": 2083586048
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003722917669715937,
+      "loss": 1.2811,
+      "theoretical_loss": 3.447191901439603,
+      "tokens_seen": 2083717120
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003722516450008024,
+      "loss": 1.2307,
+      "theoretical_loss": 3.447174116124481,
+      "tokens_seen": 2083848192
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003722115230300113,
+      "loss": 1.2271,
+      "theoretical_loss": 3.447156332241211,
+      "tokens_seen": 2083979264
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037217140105922004,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4471385497895866,
+      "tokens_seen": 2084110336
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003721312790884288,
+      "loss": 1.3279,
+      "theoretical_loss": 3.4471207687694028,
+      "tokens_seen": 2084241408
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037209115711763765,
+      "loss": 1.2781,
+      "theoretical_loss": 3.447102989180454,
+      "tokens_seen": 2084372480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003720510351468464,
+      "loss": 1.1959,
+      "theoretical_loss": 3.447085211022536,
+      "tokens_seen": 2084503552
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003720109131760552,
+      "loss": 1.2471,
+      "theoretical_loss": 3.447067434295443,
+      "tokens_seen": 2084634624
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000371970791205264,
+      "loss": 1.2377,
+      "theoretical_loss": 3.44704965899897,
+      "tokens_seen": 2084765696
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003719306692344728,
+      "loss": 1.3095,
+      "theoretical_loss": 3.447031885132911,
+      "tokens_seen": 2084896768
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003718905472636816,
+      "loss": 1.2265,
+      "theoretical_loss": 3.4470141126970626,
+      "tokens_seen": 2085027840
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003718504252928904,
+      "loss": 1.286,
+      "theoretical_loss": 3.446996341691219,
+      "tokens_seen": 2085158912
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003718103033220992,
+      "loss": 1.3023,
+      "theoretical_loss": 3.4469785721151753,
+      "tokens_seen": 2085289984
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000371770181351308,
+      "loss": 1.2979,
+      "theoretical_loss": 3.446960803968727,
+      "tokens_seen": 2085421056
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003717300593805168,
+      "loss": 1.2381,
+      "theoretical_loss": 3.4469430372516685,
+      "tokens_seen": 2085552128
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037168993740972555,
+      "loss": 1.2683,
+      "theoretical_loss": 3.4469252719637957,
+      "tokens_seen": 2085683200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003716498154389344,
+      "loss": 1.2994,
+      "theoretical_loss": 3.4469075081049034,
+      "tokens_seen": 2085814272
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037160969346814316,
+      "loss": 1.1948,
+      "theoretical_loss": 3.4468897456747865,
+      "tokens_seen": 2085945344
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003715695714973519,
+      "loss": 1.2646,
+      "theoretical_loss": 3.4468719846732414,
+      "tokens_seen": 2086076416
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003715294495265608,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4468542251000627,
+      "tokens_seen": 2086207488
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037148932755576953,
+      "loss": 1.2829,
+      "theoretical_loss": 3.4468364669550455,
+      "tokens_seen": 2086338560
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037144920558497834,
+      "loss": 1.2324,
+      "theoretical_loss": 3.446818710237986,
+      "tokens_seen": 2086469632
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037140908361418714,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4468009549486798,
+      "tokens_seen": 2086600704
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037136896164339595,
+      "loss": 1.2216,
+      "theoretical_loss": 3.4467832010869213,
+      "tokens_seen": 2086731776
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003713288396726047,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4467654486525072,
+      "tokens_seen": 2086862848
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003712887177018135,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4467476976452325,
+      "tokens_seen": 2086993920
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003712485957310223,
+      "loss": 1.2274,
+      "theoretical_loss": 3.446729948064893,
+      "tokens_seen": 2087124992
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037120847376023107,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4467121999112846,
+      "tokens_seen": 2087256064
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003711683517894399,
+      "loss": 1.1969,
+      "theoretical_loss": 3.446694453184203,
+      "tokens_seen": 2087387136
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003711282298186487,
+      "loss": 1.3432,
+      "theoretical_loss": 3.4466767078834435,
+      "tokens_seen": 2087518208
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037108810784785754,
+      "loss": 1.3218,
+      "theoretical_loss": 3.4466589640088023,
+      "tokens_seen": 2087649280
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003710479858770663,
+      "loss": 1.2996,
+      "theoretical_loss": 3.4466412215600752,
+      "tokens_seen": 2087780352
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003710078639062751,
+      "loss": 1.2164,
+      "theoretical_loss": 3.4466234805370584,
+      "tokens_seen": 2087911424
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003709677419354839,
+      "loss": 1.2781,
+      "theoretical_loss": 3.446605740939547,
+      "tokens_seen": 2088042496
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037092761996469266,
+      "loss": 1.2464,
+      "theoretical_loss": 3.4465880027673377,
+      "tokens_seen": 2088173568
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037088749799390146,
+      "loss": 1.2262,
+      "theoretical_loss": 3.4465702660202266,
+      "tokens_seen": 2088304640
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037084737602311027,
+      "loss": 1.1463,
+      "theoretical_loss": 3.4465525306980096,
+      "tokens_seen": 2088435712
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003708072540523191,
+      "loss": 1.2657,
+      "theoretical_loss": 3.4465347968004827,
+      "tokens_seen": 2088566784
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003707671320815278,
+      "loss": 1.2582,
+      "theoretical_loss": 3.446517064327442,
+      "tokens_seen": 2088697856
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003707270101107367,
+      "loss": 1.306,
+      "theoretical_loss": 3.446499333278684,
+      "tokens_seen": 2088828928
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037068688813994544,
+      "loss": 1.1993,
+      "theoretical_loss": 3.4464816036540045,
+      "tokens_seen": 2088960000
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 702251,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2310762405395508,
+      "objective/train/theoretical_loss": 3.4464638754532,
+      "objective/train/tokens_used": 459550176,
+      "theoretical_loss": 3.4464638754532,
+      "tokens_seen": 2089091072
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003706467661691542,
+      "loss": 1.3021,
+      "theoretical_loss": 3.4464638754532,
+      "tokens_seen": 2089091072
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037060664419836305,
+      "loss": 1.2195,
+      "theoretical_loss": 3.446446148676067,
+      "tokens_seen": 2089222144
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003705665222275718,
+      "loss": 1.2985,
+      "theoretical_loss": 3.446428423322402,
+      "tokens_seen": 2089353216
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037052640025678066,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4464106993920005,
+      "tokens_seen": 2089484288
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003704862782859894,
+      "loss": 1.2373,
+      "theoretical_loss": 3.44639297688466,
+      "tokens_seen": 2089615360
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003704461563151982,
+      "loss": 1.3218,
+      "theoretical_loss": 3.446375255800177,
+      "tokens_seen": 2089746432
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037040603434440703,
+      "loss": 1.2732,
+      "theoretical_loss": 3.446357536138347,
+      "tokens_seen": 2089877504
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003703659123736158,
+      "loss": 1.2884,
+      "theoretical_loss": 3.4463398178989673,
+      "tokens_seen": 2090008576
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003703257904028246,
+      "loss": 1.3478,
+      "theoretical_loss": 3.446322101081834,
+      "tokens_seen": 2090139648
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003702856684320334,
+      "loss": 1.297,
+      "theoretical_loss": 3.4463043856867444,
+      "tokens_seen": 2090270720
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003702455464612422,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4462866717134952,
+      "tokens_seen": 2090401792
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037020542449045095,
+      "loss": 1.2994,
+      "theoretical_loss": 3.4462689591618827,
+      "tokens_seen": 2090532864
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003701653025196598,
+      "loss": 1.231,
+      "theoretical_loss": 3.446251248031704,
+      "tokens_seen": 2090663936
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037012518054886856,
+      "loss": 1.3056,
+      "theoretical_loss": 3.4462335383227556,
+      "tokens_seen": 2090795008
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003700850585780773,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4462158300348347,
+      "tokens_seen": 2090926080
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003700449366072862,
+      "loss": 1.2883,
+      "theoretical_loss": 3.446198123167738,
+      "tokens_seen": 2091057152
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037000481463649493,
+      "loss": 1.2385,
+      "theoretical_loss": 3.4461804177212625,
+      "tokens_seen": 2091188224
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036996469266570374,
+      "loss": 1.2565,
+      "theoretical_loss": 3.4461627136952053,
+      "tokens_seen": 2091319296
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036992457069491254,
+      "loss": 1.2943,
+      "theoretical_loss": 3.446145011089363,
+      "tokens_seen": 2091450368
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036988444872412135,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4461273099035337,
+      "tokens_seen": 2091581440
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036984432675333015,
+      "loss": 1.1645,
+      "theoretical_loss": 3.4461096101375137,
+      "tokens_seen": 2091712512
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003698042047825389,
+      "loss": 1.2435,
+      "theoretical_loss": 3.4460919117911,
+      "tokens_seen": 2091843584
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003697640828117477,
+      "loss": 1.2144,
+      "theoretical_loss": 3.4460742148640904,
+      "tokens_seen": 2091974656
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003697239608409565,
+      "loss": 1.1974,
+      "theoretical_loss": 3.446056519356281,
+      "tokens_seen": 2092105728
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003696838388701653,
+      "loss": 1.1907,
+      "theoretical_loss": 3.446038825267471,
+      "tokens_seen": 2092236800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003696437168993741,
+      "loss": 1.3255,
+      "theoretical_loss": 3.446021132597456,
+      "tokens_seen": 2092367872
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036960359492858294,
+      "loss": 1.2819,
+      "theoretical_loss": 3.4460034413460345,
+      "tokens_seen": 2092498944
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003695634729577917,
+      "loss": 1.2211,
+      "theoretical_loss": 3.4459857515130032,
+      "tokens_seen": 2092630016
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036952335098700044,
+      "loss": 1.318,
+      "theoretical_loss": 3.4459680630981597,
+      "tokens_seen": 2092761088
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003694832290162093,
+      "loss": 1.2268,
+      "theoretical_loss": 3.445950376101302,
+      "tokens_seen": 2092892160
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036944310704541805,
+      "loss": 1.3172,
+      "theoretical_loss": 3.4459326905222265,
+      "tokens_seen": 2093023232
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036940298507462686,
+      "loss": 1.2603,
+      "theoretical_loss": 3.445915006360732,
+      "tokens_seen": 2093154304
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036936286310383567,
+      "loss": 1.3254,
+      "theoretical_loss": 3.4458973236166153,
+      "tokens_seen": 2093285376
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003693227411330445,
+      "loss": 1.2965,
+      "theoretical_loss": 3.4458796422896745,
+      "tokens_seen": 2093416448
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003692826191622532,
+      "loss": 1.3381,
+      "theoretical_loss": 3.4458619623797073,
+      "tokens_seen": 2093547520
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003692424971914621,
+      "loss": 1.2862,
+      "theoretical_loss": 3.445844283886511,
+      "tokens_seen": 2093678592
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036920237522067084,
+      "loss": 1.2351,
+      "theoretical_loss": 3.445826606809884,
+      "tokens_seen": 2093809664
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036916225324987964,
+      "loss": 1.2727,
+      "theoretical_loss": 3.4458089311496236,
+      "tokens_seen": 2093940736
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036912213127908845,
+      "loss": 1.243,
+      "theoretical_loss": 3.445791256905528,
+      "tokens_seen": 2094071808
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003690820093082972,
+      "loss": 1.3047,
+      "theoretical_loss": 3.445773584077395,
+      "tokens_seen": 2094202880
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036904188733750606,
+      "loss": 1.2696,
+      "theoretical_loss": 3.4457559126650223,
+      "tokens_seen": 2094333952
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003690017653667148,
+      "loss": 1.2513,
+      "theoretical_loss": 3.4457382426682086,
+      "tokens_seen": 2094465024
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003689616433959236,
+      "loss": 1.3068,
+      "theoretical_loss": 3.445720574086751,
+      "tokens_seen": 2094596096
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036892152142513243,
+      "loss": 1.2216,
+      "theoretical_loss": 3.445702906920448,
+      "tokens_seen": 2094727168
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003688813994543412,
+      "loss": 1.2417,
+      "theoretical_loss": 3.445685241169098,
+      "tokens_seen": 2094858240
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036884127748355,
+      "loss": 1.2698,
+      "theoretical_loss": 3.445667576832499,
+      "tokens_seen": 2094989312
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003688011555127588,
+      "loss": 1.3302,
+      "theoretical_loss": 3.445649913910449,
+      "tokens_seen": 2095120384
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003687610335419676,
+      "loss": 1.3053,
+      "theoretical_loss": 3.445632252402746,
+      "tokens_seen": 2095251456
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036872091157117635,
+      "loss": 1.2542,
+      "theoretical_loss": 3.445614592309189,
+      "tokens_seen": 2095382528
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003686807896003852,
+      "loss": 1.3044,
+      "theoretical_loss": 3.4455969336295755,
+      "tokens_seen": 2095513600
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 704471,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3082102537155151,
+      "objective/train/theoretical_loss": 3.4455792763637048,
+      "objective/train/tokens_used": 466103776,
+      "theoretical_loss": 3.4455792763637048,
+      "tokens_seen": 2095644672
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036864066762959396,
+      "loss": 1.2831,
+      "theoretical_loss": 3.4455792763637048,
+      "tokens_seen": 2095644672
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036860054565880277,
+      "loss": 1.2854,
+      "theoretical_loss": 3.445561620511375,
+      "tokens_seen": 2095775744
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003685604236880116,
+      "loss": 1.2993,
+      "theoretical_loss": 3.4455439660723837,
+      "tokens_seen": 2095906816
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036852030171722033,
+      "loss": 1.2296,
+      "theoretical_loss": 3.44552631304653,
+      "tokens_seen": 2096037888
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003684801797464292,
+      "loss": 1.2627,
+      "theoretical_loss": 3.445508661433613,
+      "tokens_seen": 2096168960
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036844005777563794,
+      "loss": 1.3165,
+      "theoretical_loss": 3.4454910112334307,
+      "tokens_seen": 2096300032
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036839993580484675,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4454733624457816,
+      "tokens_seen": 2096431104
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036835981383405555,
+      "loss": 1.3547,
+      "theoretical_loss": 3.445455715070465,
+      "tokens_seen": 2096562176
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003683196918632643,
+      "loss": 1.2747,
+      "theoretical_loss": 3.4454380691072783,
+      "tokens_seen": 2096693248
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003682795698924731,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4454204245560214,
+      "tokens_seen": 2096824320
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003682394479216819,
+      "loss": 1.2171,
+      "theoretical_loss": 3.445402781416493,
+      "tokens_seen": 2096955392
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003681993259508907,
+      "loss": 1.2629,
+      "theoretical_loss": 3.445385139688491,
+      "tokens_seen": 2097086464
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003681592039800995,
+      "loss": 1.2692,
+      "theoretical_loss": 3.445367499371815,
+      "tokens_seen": 2097217536
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036811908200930834,
+      "loss": 1.2968,
+      "theoretical_loss": 3.4453498604662642,
+      "tokens_seen": 2097348608
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003680789600385171,
+      "loss": 1.279,
+      "theoretical_loss": 3.445332222971637,
+      "tokens_seen": 2097479680
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036803883806772584,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4453145868877324,
+      "tokens_seen": 2097610752
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003679987160969347,
+      "loss": 1.2839,
+      "theoretical_loss": 3.44529695221435,
+      "tokens_seen": 2097741824
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036795859412614345,
+      "loss": 1.2518,
+      "theoretical_loss": 3.445279318951288,
+      "tokens_seen": 2097872896
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003679184721553523,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4452616870983457,
+      "tokens_seen": 2098003968
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036787835018456107,
+      "loss": 1.253,
+      "theoretical_loss": 3.445244056655323,
+      "tokens_seen": 2098135040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003678382282137699,
+      "loss": 1.3108,
+      "theoretical_loss": 3.4452264276220177,
+      "tokens_seen": 2098266112
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003677981062429787,
+      "loss": 1.2336,
+      "theoretical_loss": 3.4452087999982304,
+      "tokens_seen": 2098397184
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003677579842721875,
+      "loss": 1.28,
+      "theoretical_loss": 3.44519117378376,
+      "tokens_seen": 2098528256
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036771786230139624,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4451735489784054,
+      "tokens_seen": 2098659328
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036767774033060504,
+      "loss": 1.2715,
+      "theoretical_loss": 3.4451559255819664,
+      "tokens_seen": 2098790400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036763761835981385,
+      "loss": 1.1869,
+      "theoretical_loss": 3.4451383035942422,
+      "tokens_seen": 2098921472
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003675974963890226,
+      "loss": 1.2467,
+      "theoretical_loss": 3.4451206830150323,
+      "tokens_seen": 2099052544
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036755737441823146,
+      "loss": 1.3356,
+      "theoretical_loss": 3.445103063844136,
+      "tokens_seen": 2099183616
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003675172524474402,
+      "loss": 1.3128,
+      "theoretical_loss": 3.445085446081353,
+      "tokens_seen": 2099314688
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000367477130476649,
+      "loss": 1.3441,
+      "theoretical_loss": 3.4450678297264825,
+      "tokens_seen": 2099445760
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036743700850585783,
+      "loss": 1.2776,
+      "theoretical_loss": 3.4450502147793243,
+      "tokens_seen": 2099576832
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003673968865350666,
+      "loss": 1.212,
+      "theoretical_loss": 3.445032601239679,
+      "tokens_seen": 2099707904
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036735676456427544,
+      "loss": 1.2656,
+      "theoretical_loss": 3.4450149891073445,
+      "tokens_seen": 2099838976
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003673166425934842,
+      "loss": 1.3558,
+      "theoretical_loss": 3.444997378382122,
+      "tokens_seen": 2099970048
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000367276520622693,
+      "loss": 1.2799,
+      "theoretical_loss": 3.4449797690638104,
+      "tokens_seen": 2100101120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003672363986519018,
+      "loss": 1.2276,
+      "theoretical_loss": 3.44496216115221,
+      "tokens_seen": 2100232192
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003671962766811106,
+      "loss": 1.28,
+      "theoretical_loss": 3.4449445546471207,
+      "tokens_seen": 2100363264
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036715615471031936,
+      "loss": 1.2168,
+      "theoretical_loss": 3.4449269495483414,
+      "tokens_seen": 2100494336
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036711603273952817,
+      "loss": 1.2751,
+      "theoretical_loss": 3.4449093458556734,
+      "tokens_seen": 2100625408
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000367075910768737,
+      "loss": 1.2994,
+      "theoretical_loss": 3.444891743568916,
+      "tokens_seen": 2100756480
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036703578879794573,
+      "loss": 1.4633,
+      "theoretical_loss": 3.444874142687869,
+      "tokens_seen": 2100887552
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003669956668271546,
+      "loss": 1.3278,
+      "theoretical_loss": 3.4448565432123326,
+      "tokens_seen": 2101018624
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036695554485636334,
+      "loss": 1.3109,
+      "theoretical_loss": 3.4448389451421075,
+      "tokens_seen": 2101149696
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036691542288557215,
+      "loss": 1.4597,
+      "theoretical_loss": 3.4448213484769927,
+      "tokens_seen": 2101280768
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036687530091478095,
+      "loss": 1.3016,
+      "theoretical_loss": 3.4448037532167897,
+      "tokens_seen": 2101411840
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003668351789439897,
+      "loss": 1.3968,
+      "theoretical_loss": 3.444786159361297,
+      "tokens_seen": 2101542912
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003667950569731985,
+      "loss": 1.3186,
+      "theoretical_loss": 3.4447685669103167,
+      "tokens_seen": 2101673984
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003667549350024073,
+      "loss": 1.3323,
+      "theoretical_loss": 3.4447509758636485,
+      "tokens_seen": 2101805056
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003667148130316161,
+      "loss": 1.2693,
+      "theoretical_loss": 3.444733386221092,
+      "tokens_seen": 2101936128
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036667469106082493,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4447157979824476,
+      "tokens_seen": 2102067200
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 706780,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2601630687713623,
+      "objective/train/theoretical_loss": 3.444698211147517,
+      "objective/train/tokens_used": 472657376,
+      "theoretical_loss": 3.444698211147517,
+      "tokens_seen": 2102198272
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036663456909003374,
+      "loss": 1.2839,
+      "theoretical_loss": 3.444698211147517,
+      "tokens_seen": 2102198272
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003665944471192425,
+      "loss": 1.3206,
+      "theoretical_loss": 3.4446806257160993,
+      "tokens_seen": 2102329344
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003665543251484513,
+      "loss": 1.3427,
+      "theoretical_loss": 3.444663041687996,
+      "tokens_seen": 2102460416
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003665142031776601,
+      "loss": 1.2852,
+      "theoretical_loss": 3.4446454590630067,
+      "tokens_seen": 2102591488
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036647408120686885,
+      "loss": 1.3512,
+      "theoretical_loss": 3.444627877840933,
+      "tokens_seen": 2102722560
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003664339592360777,
+      "loss": 1.277,
+      "theoretical_loss": 3.4446102980215745,
+      "tokens_seen": 2102853632
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036639383726528647,
+      "loss": 1.3702,
+      "theoretical_loss": 3.444592719604733,
+      "tokens_seen": 2102984704
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003663537152944953,
+      "loss": 1.2568,
+      "theoretical_loss": 3.444575142590208,
+      "tokens_seen": 2103115776
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003663135933237041,
+      "loss": 1.2492,
+      "theoretical_loss": 3.4445575669778012,
+      "tokens_seen": 2103246848
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003662734713529129,
+      "loss": 1.263,
+      "theoretical_loss": 3.4445399927673126,
+      "tokens_seen": 2103377920
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036623334938212164,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4445224199585436,
+      "tokens_seen": 2103508992
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036619322741133044,
+      "loss": 1.2625,
+      "theoretical_loss": 3.444504848551295,
+      "tokens_seen": 2103640064
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036615310544053925,
+      "loss": 1.2856,
+      "theoretical_loss": 3.4444872785453673,
+      "tokens_seen": 2103771136
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000366112983469748,
+      "loss": 1.3447,
+      "theoretical_loss": 3.444469709940562,
+      "tokens_seen": 2103902208
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036607286149895686,
+      "loss": 1.174,
+      "theoretical_loss": 3.4444521427366794,
+      "tokens_seen": 2104033280
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003660327395281656,
+      "loss": 1.2926,
+      "theoretical_loss": 3.4444345769335216,
+      "tokens_seen": 2104164352
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003659926175573745,
+      "loss": 1.256,
+      "theoretical_loss": 3.444417012530889,
+      "tokens_seen": 2104295424
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036595249558658323,
+      "loss": 1.3112,
+      "theoretical_loss": 3.444399449528582,
+      "tokens_seen": 2104426496
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000365912373615792,
+      "loss": 1.2265,
+      "theoretical_loss": 3.444381887926403,
+      "tokens_seen": 2104557568
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036587225164500084,
+      "loss": 1.2456,
+      "theoretical_loss": 3.4443643277241525,
+      "tokens_seen": 2104688640
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003658321296742096,
+      "loss": 1.3088,
+      "theoretical_loss": 3.444346768921632,
+      "tokens_seen": 2104819712
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003657920077034184,
+      "loss": 1.3002,
+      "theoretical_loss": 3.444329211518643,
+      "tokens_seen": 2104950784
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003657518857326272,
+      "loss": 1.3139,
+      "theoretical_loss": 3.444311655514986,
+      "tokens_seen": 2105081856
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000365711763761836,
+      "loss": 1.318,
+      "theoretical_loss": 3.4442941009104633,
+      "tokens_seen": 2105212928
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036567164179104476,
+      "loss": 1.2146,
+      "theoretical_loss": 3.4442765477048756,
+      "tokens_seen": 2105344000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036563151982025357,
+      "loss": 1.3302,
+      "theoretical_loss": 3.444258995898024,
+      "tokens_seen": 2105475072
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003655913978494624,
+      "loss": 1.2668,
+      "theoretical_loss": 3.4442414454897117,
+      "tokens_seen": 2105606144
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036555127587867113,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4442238964797385,
+      "tokens_seen": 2105737216
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036551115390788,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4442063488679064,
+      "tokens_seen": 2105868288
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036547103193708874,
+      "loss": 1.2563,
+      "theoretical_loss": 3.4441888026540175,
+      "tokens_seen": 2105999360
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003654309099662976,
+      "loss": 1.2203,
+      "theoretical_loss": 3.4441712578378727,
+      "tokens_seen": 2106130432
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036539078799550635,
+      "loss": 1.2573,
+      "theoretical_loss": 3.444153714419274,
+      "tokens_seen": 2106261504
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003653506660247151,
+      "loss": 1.2632,
+      "theoretical_loss": 3.4441361723980233,
+      "tokens_seen": 2106392576
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036531054405392397,
+      "loss": 1.2445,
+      "theoretical_loss": 3.4441186317739225,
+      "tokens_seen": 2106523648
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003652704220831327,
+      "loss": 1.3339,
+      "theoretical_loss": 3.4441010925467728,
+      "tokens_seen": 2106654720
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003652303001123415,
+      "loss": 1.3073,
+      "theoretical_loss": 3.444083554716377,
+      "tokens_seen": 2106785792
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036519017814155033,
+      "loss": 1.287,
+      "theoretical_loss": 3.444066018282536,
+      "tokens_seen": 2106916864
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036515005617075914,
+      "loss": 1.3118,
+      "theoretical_loss": 3.4440484832450515,
+      "tokens_seen": 2107047936
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003651099341999679,
+      "loss": 1.3572,
+      "theoretical_loss": 3.444030949603726,
+      "tokens_seen": 2107179008
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003650698122291767,
+      "loss": 1.2761,
+      "theoretical_loss": 3.444013417358362,
+      "tokens_seen": 2107310080
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003650296902583855,
+      "loss": 1.2581,
+      "theoretical_loss": 3.443995886508761,
+      "tokens_seen": 2107441152
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036498956828759425,
+      "loss": 1.3114,
+      "theoretical_loss": 3.4439783570547253,
+      "tokens_seen": 2107572224
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003649494463168031,
+      "loss": 1.1815,
+      "theoretical_loss": 3.443960828996057,
+      "tokens_seen": 2107703296
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036490932434601187,
+      "loss": 1.258,
+      "theoretical_loss": 3.4439433023325576,
+      "tokens_seen": 2107834368
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036486920237522067,
+      "loss": 1.2679,
+      "theoretical_loss": 3.44392577706403,
+      "tokens_seen": 2107965440
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003648290804044295,
+      "loss": 1.2695,
+      "theoretical_loss": 3.4439082531902767,
+      "tokens_seen": 2108096512
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036478895843363823,
+      "loss": 1.2221,
+      "theoretical_loss": 3.4438907307110993,
+      "tokens_seen": 2108227584
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003647488364628471,
+      "loss": 1.2551,
+      "theoretical_loss": 3.4438732096263003,
+      "tokens_seen": 2108358656
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036470871449205584,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4438556899356825,
+      "tokens_seen": 2108489728
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036466859252126465,
+      "loss": 1.3738,
+      "theoretical_loss": 3.4438381716390474,
+      "tokens_seen": 2108620800
+    },
+    {
+      "epoch": 0.28,
+      "objective/train/docs_used": 708880,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.30996572971344,
+      "objective/train/theoretical_loss": 3.4438206547361983,
+      "objective/train/tokens_used": 479210976,
+      "theoretical_loss": 3.4438206547361983,
+      "tokens_seen": 2108751872
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036462847055047346,
+      "loss": 1.2585,
+      "theoretical_loss": 3.4438206547361983,
+      "tokens_seen": 2108751872
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036458834857968226,
+      "loss": 1.3179,
+      "theoretical_loss": 3.4438031392269375,
+      "tokens_seen": 2108882944
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000364548226608891,
+      "loss": 1.2702,
+      "theoretical_loss": 3.4437856251110674,
+      "tokens_seen": 2109014016
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003645081046380999,
+      "loss": 1.354,
+      "theoretical_loss": 3.4437681123883905,
+      "tokens_seen": 2109145088
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036446798266730863,
+      "loss": 1.2644,
+      "theoretical_loss": 3.44375060105871,
+      "tokens_seen": 2109276160
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003644278606965174,
+      "loss": 1.2607,
+      "theoretical_loss": 3.4437330911218273,
+      "tokens_seen": 2109407232
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036438773872572624,
+      "loss": 1.3385,
+      "theoretical_loss": 3.4437155825775463,
+      "tokens_seen": 2109538304
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000364347616754935,
+      "loss": 1.1981,
+      "theoretical_loss": 3.4436980754256687,
+      "tokens_seen": 2109669376
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003643074947841438,
+      "loss": 1.254,
+      "theoretical_loss": 3.4436805696659984,
+      "tokens_seen": 2109800448
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003642673728133526,
+      "loss": 1.3018,
+      "theoretical_loss": 3.4436630652983373,
+      "tokens_seen": 2109931520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003642272508425614,
+      "loss": 1.2445,
+      "theoretical_loss": 3.443645562322489,
+      "tokens_seen": 2110062592
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036418712887177016,
+      "loss": 1.2045,
+      "theoretical_loss": 3.4436280607382557,
+      "tokens_seen": 2110193664
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036414700690097897,
+      "loss": 1.2984,
+      "theoretical_loss": 3.443610560545441,
+      "tokens_seen": 2110324736
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003641068849301878,
+      "loss": 1.2719,
+      "theoretical_loss": 3.443593061743847,
+      "tokens_seen": 2110455808
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003640667629593966,
+      "loss": 1.2466,
+      "theoretical_loss": 3.443575564333277,
+      "tokens_seen": 2110586880
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003640266409886054,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4435580683135347,
+      "tokens_seen": 2110717952
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036398651901781414,
+      "loss": 1.2517,
+      "theoretical_loss": 3.4435405736844222,
+      "tokens_seen": 2110849024
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000363946397047023,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4435230804457437,
+      "tokens_seen": 2110980096
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036390627507623175,
+      "loss": 1.2502,
+      "theoretical_loss": 3.4435055885973016,
+      "tokens_seen": 2111111168
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003638661531054405,
+      "loss": 1.3105,
+      "theoretical_loss": 3.443488098138899,
+      "tokens_seen": 2111242240
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036382603113464937,
+      "loss": 1.289,
+      "theoretical_loss": 3.44347060907034,
+      "tokens_seen": 2111373312
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003637859091638581,
+      "loss": 1.2364,
+      "theoretical_loss": 3.4434531213914275,
+      "tokens_seen": 2111504384
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003637457871930669,
+      "loss": 1.2456,
+      "theoretical_loss": 3.443435635101964,
+      "tokens_seen": 2111635456
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036370566522227573,
+      "loss": 1.2152,
+      "theoretical_loss": 3.4434181502017536,
+      "tokens_seen": 2111766528
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036366554325148454,
+      "loss": 1.2905,
+      "theoretical_loss": 3.4434006666906,
+      "tokens_seen": 2111897600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003636254212806933,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4433831845683063,
+      "tokens_seen": 2112028672
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003635852993099021,
+      "loss": 1.2383,
+      "theoretical_loss": 3.443365703834676,
+      "tokens_seen": 2112159744
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003635451773391109,
+      "loss": 1.3259,
+      "theoretical_loss": 3.4433482244895126,
+      "tokens_seen": 2112290816
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003635050553683197,
+      "loss": 1.2868,
+      "theoretical_loss": 3.44333074653262,
+      "tokens_seen": 2112421888
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003634649333975285,
+      "loss": 1.283,
+      "theoretical_loss": 3.4433132699638014,
+      "tokens_seen": 2112552960
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036342481142673727,
+      "loss": 1.319,
+      "theoretical_loss": 3.44329579478286,
+      "tokens_seen": 2112684032
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003633846894559461,
+      "loss": 1.2314,
+      "theoretical_loss": 3.4432783209896005,
+      "tokens_seen": 2112815104
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003633445674851549,
+      "loss": 1.2491,
+      "theoretical_loss": 3.443260848583826,
+      "tokens_seen": 2112946176
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036330444551436363,
+      "loss": 1.3446,
+      "theoretical_loss": 3.4432433775653406,
+      "tokens_seen": 2113077248
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003632643235435725,
+      "loss": 1.2875,
+      "theoretical_loss": 3.443225907933948,
+      "tokens_seen": 2113208320
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036322420157278124,
+      "loss": 1.2108,
+      "theoretical_loss": 3.4432084396894522,
+      "tokens_seen": 2113339392
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036318407960199005,
+      "loss": 1.2899,
+      "theoretical_loss": 3.4431909728316565,
+      "tokens_seen": 2113470464
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036314395763119886,
+      "loss": 1.3049,
+      "theoretical_loss": 3.443173507360365,
+      "tokens_seen": 2113601536
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036310383566040766,
+      "loss": 1.3381,
+      "theoretical_loss": 3.4431560432753825,
+      "tokens_seen": 2113732608
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003630637136896164,
+      "loss": 1.3025,
+      "theoretical_loss": 3.4431385805765125,
+      "tokens_seen": 2113863680
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003630235917188253,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4431211192635582,
+      "tokens_seen": 2113994752
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000362983469748034,
+      "loss": 1.3669,
+      "theoretical_loss": 3.443103659336325,
+      "tokens_seen": 2114125824
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003629433477772428,
+      "loss": 1.2767,
+      "theoretical_loss": 3.4430862007946166,
+      "tokens_seen": 2114256896
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036290322580645164,
+      "loss": 1.2799,
+      "theoretical_loss": 3.4430687436382366,
+      "tokens_seen": 2114387968
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003628631038356604,
+      "loss": 1.2937,
+      "theoretical_loss": 3.4430512878669894,
+      "tokens_seen": 2114519040
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036282298186486925,
+      "loss": 1.3374,
+      "theoretical_loss": 3.4430338334806803,
+      "tokens_seen": 2114650112
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000362782859894078,
+      "loss": 1.2928,
+      "theoretical_loss": 3.4430163804791123,
+      "tokens_seen": 2114781184
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003627427379232868,
+      "loss": 1.3293,
+      "theoretical_loss": 3.4429989288620906,
+      "tokens_seen": 2114912256
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003627026159524956,
+      "loss": 1.2746,
+      "theoretical_loss": 3.4429814786294184,
+      "tokens_seen": 2115043328
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036266249398170437,
+      "loss": 1.2435,
+      "theoretical_loss": 3.442964029780901,
+      "tokens_seen": 2115174400
+    },
+    {
+      "epoch": 0.28,
+      "objective/train/docs_used": 711117,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.289231538772583,
+      "objective/train/theoretical_loss": 3.4429465823163437,
+      "objective/train/tokens_used": 485764576,
+      "theoretical_loss": 3.4429465823163437,
+      "tokens_seen": 2115305472
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003626223720109132,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4429465823163437,
+      "tokens_seen": 2115305472
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000362582250040122,
+      "loss": 1.3216,
+      "theoretical_loss": 3.4429291362355494,
+      "tokens_seen": 2115436544
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003625421280693308,
+      "loss": 1.2695,
+      "theoretical_loss": 3.442911691538323,
+      "tokens_seen": 2115567616
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036250200609853954,
+      "loss": 1.2446,
+      "theoretical_loss": 3.4428942482244693,
+      "tokens_seen": 2115698688
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003624618841277484,
+      "loss": 1.274,
+      "theoretical_loss": 3.4428768062937936,
+      "tokens_seen": 2115829760
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036242176215695715,
+      "loss": 1.2277,
+      "theoretical_loss": 3.4428593657460986,
+      "tokens_seen": 2115960832
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003623816401861659,
+      "loss": 1.3712,
+      "theoretical_loss": 3.4428419265811914,
+      "tokens_seen": 2116091904
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036234151821537477,
+      "loss": 1.3128,
+      "theoretical_loss": 3.442824488798875,
+      "tokens_seen": 2116222976
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003623013962445835,
+      "loss": 1.2109,
+      "theoretical_loss": 3.442807052398955,
+      "tokens_seen": 2116354048
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003622612742737924,
+      "loss": 1.2376,
+      "theoretical_loss": 3.442789617381236,
+      "tokens_seen": 2116485120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036222115230300113,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4427721837455225,
+      "tokens_seen": 2116616192
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036218103033220994,
+      "loss": 1.3205,
+      "theoretical_loss": 3.44275475149162,
+      "tokens_seen": 2116747264
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036214090836141874,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4427373206193326,
+      "tokens_seen": 2116878336
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003621007863906275,
+      "loss": 1.2891,
+      "theoretical_loss": 3.442719891128466,
+      "tokens_seen": 2117009408
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003620606644198363,
+      "loss": 1.3568,
+      "theoretical_loss": 3.4427024630188248,
+      "tokens_seen": 2117140480
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003620205424490451,
+      "loss": 1.2437,
+      "theoretical_loss": 3.4426850362902144,
+      "tokens_seen": 2117271552
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003619804204782539,
+      "loss": 1.3345,
+      "theoretical_loss": 3.4426676109424394,
+      "tokens_seen": 2117402624
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036194029850746267,
+      "loss": 1.2233,
+      "theoretical_loss": 3.4426501869753054,
+      "tokens_seen": 2117533696
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003619001765366715,
+      "loss": 1.3624,
+      "theoretical_loss": 3.442632764388617,
+      "tokens_seen": 2117664768
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003618600545658803,
+      "loss": 1.2948,
+      "theoretical_loss": 3.4426153431821804,
+      "tokens_seen": 2117795840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036181993259508903,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4425979233558,
+      "tokens_seen": 2117926912
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003617798106242979,
+      "loss": 1.1979,
+      "theoretical_loss": 3.442580504909281,
+      "tokens_seen": 2118057984
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036173968865350664,
+      "loss": 1.2776,
+      "theoretical_loss": 3.442563087842429,
+      "tokens_seen": 2118189056
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036169956668271545,
+      "loss": 1.3257,
+      "theoretical_loss": 3.442545672155049,
+      "tokens_seen": 2118320128
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036165944471192426,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4425282578469476,
+      "tokens_seen": 2118451200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036161932274113306,
+      "loss": 1.276,
+      "theoretical_loss": 3.442510844917929,
+      "tokens_seen": 2118582272
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036157920077034187,
+      "loss": 1.2755,
+      "theoretical_loss": 3.4424934333677983,
+      "tokens_seen": 2118713344
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003615390787995507,
+      "loss": 1.2983,
+      "theoretical_loss": 3.4424760231963627,
+      "tokens_seen": 2118844416
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003614989568287594,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4424586144034266,
+      "tokens_seen": 2118975488
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036145883485796823,
+      "loss": 1.274,
+      "theoretical_loss": 3.4424412069887955,
+      "tokens_seen": 2119106560
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036141871288717704,
+      "loss": 1.3307,
+      "theoretical_loss": 3.442423800952276,
+      "tokens_seen": 2119237632
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003613785909163858,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4424063962936726,
+      "tokens_seen": 2119368704
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036133846894559465,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4423889930127913,
+      "tokens_seen": 2119499776
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003612983469748034,
+      "loss": 1.3591,
+      "theoretical_loss": 3.442371591109439,
+      "tokens_seen": 2119630848
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003612582250040122,
+      "loss": 1.2584,
+      "theoretical_loss": 3.44235419058342,
+      "tokens_seen": 2119761920
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000361218103033221,
+      "loss": 1.2807,
+      "theoretical_loss": 3.44233679143454,
+      "tokens_seen": 2119892992
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036117798106242977,
+      "loss": 1.2961,
+      "theoretical_loss": 3.442319393662607,
+      "tokens_seen": 2120024064
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003611378590916386,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4423019972674247,
+      "tokens_seen": 2120155136
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003610977371208474,
+      "loss": 1.2759,
+      "theoretical_loss": 3.4422846022488,
+      "tokens_seen": 2120286208
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003610576151500562,
+      "loss": 1.2892,
+      "theoretical_loss": 3.4422672086065385,
+      "tokens_seen": 2120417280
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036101749317926494,
+      "loss": 1.3195,
+      "theoretical_loss": 3.4422498163404462,
+      "tokens_seen": 2120548352
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003609773712084738,
+      "loss": 1.2454,
+      "theoretical_loss": 3.44223242545033,
+      "tokens_seen": 2120679424
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036093724923768255,
+      "loss": 1.3569,
+      "theoretical_loss": 3.4422150359359955,
+      "tokens_seen": 2120810496
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036089712726689136,
+      "loss": 1.1945,
+      "theoretical_loss": 3.4421976477972485,
+      "tokens_seen": 2120941568
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036085700529610017,
+      "loss": 1.3239,
+      "theoretical_loss": 3.4421802610338954,
+      "tokens_seen": 2121072640
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003608168833253089,
+      "loss": 1.3363,
+      "theoretical_loss": 3.4421628756457423,
+      "tokens_seen": 2121203712
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003607767613545178,
+      "loss": 1.2838,
+      "theoretical_loss": 3.442145491632596,
+      "tokens_seen": 2121334784
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036073663938372653,
+      "loss": 1.2433,
+      "theoretical_loss": 3.442128108994263,
+      "tokens_seen": 2121465856
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036069651741293534,
+      "loss": 1.2817,
+      "theoretical_loss": 3.4421107277305483,
+      "tokens_seen": 2121596928
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036065639544214414,
+      "loss": 1.3137,
+      "theoretical_loss": 3.4420933478412588,
+      "tokens_seen": 2121728000
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 713301,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3012292385101318,
+      "objective/train/theoretical_loss": 3.442075969326202,
+      "objective/train/tokens_used": 492318176,
+      "theoretical_loss": 3.442075969326202,
+      "tokens_seen": 2121859072
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003606162734713529,
+      "loss": 1.2795,
+      "theoretical_loss": 3.442075969326202,
+      "tokens_seen": 2121859072
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003605761515005617,
+      "loss": 1.3896,
+      "theoretical_loss": 3.442058592185183,
+      "tokens_seen": 2121990144
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003605360295297705,
+      "loss": 1.3243,
+      "theoretical_loss": 3.4420412164180094,
+      "tokens_seen": 2122121216
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003604959075589793,
+      "loss": 1.2634,
+      "theoretical_loss": 3.4420238420244864,
+      "tokens_seen": 2122252288
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036045578558818807,
+      "loss": 1.2636,
+      "theoretical_loss": 3.442006469004422,
+      "tokens_seen": 2122383360
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003604156636173969,
+      "loss": 1.3013,
+      "theoretical_loss": 3.441989097357622,
+      "tokens_seen": 2122514432
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003603755416466057,
+      "loss": 1.2203,
+      "theoretical_loss": 3.441971727083893,
+      "tokens_seen": 2122645504
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003603354196758145,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4419543581830423,
+      "tokens_seen": 2122776576
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003602952977050233,
+      "loss": 1.2802,
+      "theoretical_loss": 3.4419369906548765,
+      "tokens_seen": 2122907648
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036025517573423204,
+      "loss": 1.3124,
+      "theoretical_loss": 3.441919624499201,
+      "tokens_seen": 2123038720
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003602150537634409,
+      "loss": 1.2915,
+      "theoretical_loss": 3.441902259715825,
+      "tokens_seen": 2123169792
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036017493179264966,
+      "loss": 1.2798,
+      "theoretical_loss": 3.441884896304554,
+      "tokens_seen": 2123300864
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036013480982185846,
+      "loss": 1.2283,
+      "theoretical_loss": 3.441867534265194,
+      "tokens_seen": 2123431936
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036009468785106727,
+      "loss": 1.2962,
+      "theoretical_loss": 3.4418501735975537,
+      "tokens_seen": 2123563008
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000360054565880276,
+      "loss": 1.2721,
+      "theoretical_loss": 3.4418328143014394,
+      "tokens_seen": 2123694080
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003600144439094848,
+      "loss": 1.2138,
+      "theoretical_loss": 3.441815456376658,
+      "tokens_seen": 2123825152
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035997432193869363,
+      "loss": 1.2754,
+      "theoretical_loss": 3.441798099823016,
+      "tokens_seen": 2123956224
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035993419996790244,
+      "loss": 1.2382,
+      "theoretical_loss": 3.4417807446403215,
+      "tokens_seen": 2124087296
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003598940779971112,
+      "loss": 1.2626,
+      "theoretical_loss": 3.441763390828381,
+      "tokens_seen": 2124218368
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035985395602632005,
+      "loss": 1.28,
+      "theoretical_loss": 3.4417460383870018,
+      "tokens_seen": 2124349440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003598138340555288,
+      "loss": 1.3265,
+      "theoretical_loss": 3.441728687315991,
+      "tokens_seen": 2124480512
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003597737120847376,
+      "loss": 1.2573,
+      "theoretical_loss": 3.441711337615156,
+      "tokens_seen": 2124611584
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003597335901139464,
+      "loss": 1.2896,
+      "theoretical_loss": 3.4416939892843046,
+      "tokens_seen": 2124742656
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035969346814315517,
+      "loss": 1.251,
+      "theoretical_loss": 3.441676642323243,
+      "tokens_seen": 2124873728
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035965334617236403,
+      "loss": 1.2728,
+      "theoretical_loss": 3.441659296731779,
+      "tokens_seen": 2125004800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003596132242015728,
+      "loss": 1.369,
+      "theoretical_loss": 3.4416419525097206,
+      "tokens_seen": 2125135872
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003595731022307816,
+      "loss": 1.3294,
+      "theoretical_loss": 3.4416246096568752,
+      "tokens_seen": 2125266944
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003595329802599904,
+      "loss": 1.3291,
+      "theoretical_loss": 3.441607268173049,
+      "tokens_seen": 2125398016
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003594928582891992,
+      "loss": 1.3449,
+      "theoretical_loss": 3.4415899280580504,
+      "tokens_seen": 2125529088
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035945273631840795,
+      "loss": 1.2601,
+      "theoretical_loss": 3.4415725893116873,
+      "tokens_seen": 2125660160
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035941261434761676,
+      "loss": 1.3068,
+      "theoretical_loss": 3.441555251933767,
+      "tokens_seen": 2125791232
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035937249237682556,
+      "loss": 1.2805,
+      "theoretical_loss": 3.4415379159240964,
+      "tokens_seen": 2125922304
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003593323704060343,
+      "loss": 1.2664,
+      "theoretical_loss": 3.441520581282484,
+      "tokens_seen": 2126053376
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003592922484352432,
+      "loss": 1.3338,
+      "theoretical_loss": 3.441503248008738,
+      "tokens_seen": 2126184448
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035925212646445193,
+      "loss": 1.268,
+      "theoretical_loss": 3.441485916102665,
+      "tokens_seen": 2126315520
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035921200449366074,
+      "loss": 1.277,
+      "theoretical_loss": 3.4414685855640728,
+      "tokens_seen": 2126446592
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035917188252286954,
+      "loss": 1.3074,
+      "theoretical_loss": 3.44145125639277,
+      "tokens_seen": 2126577664
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003591317605520783,
+      "loss": 1.2962,
+      "theoretical_loss": 3.4414339285885642,
+      "tokens_seen": 2126708736
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035909163858128715,
+      "loss": 1.2887,
+      "theoretical_loss": 3.4414166021512633,
+      "tokens_seen": 2126839808
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003590515166104959,
+      "loss": 1.3125,
+      "theoretical_loss": 3.4413992770806754,
+      "tokens_seen": 2126970880
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003590113946397047,
+      "loss": 1.3692,
+      "theoretical_loss": 3.4413819533766077,
+      "tokens_seen": 2127101952
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003589712726689135,
+      "loss": 1.207,
+      "theoretical_loss": 3.4413646310388692,
+      "tokens_seen": 2127233024
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003589311506981223,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4413473100672674,
+      "tokens_seen": 2127364096
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003588910287273311,
+      "loss": 1.2479,
+      "theoretical_loss": 3.441329990461611,
+      "tokens_seen": 2127495168
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003588509067565399,
+      "loss": 1.2913,
+      "theoretical_loss": 3.441312672221707,
+      "tokens_seen": 2127626240
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003588107847857487,
+      "loss": 1.2834,
+      "theoretical_loss": 3.4412953553473646,
+      "tokens_seen": 2127757312
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035877066281495744,
+      "loss": 1.3472,
+      "theoretical_loss": 3.4412780398383918,
+      "tokens_seen": 2127888384
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003587305408441663,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4412607256945966,
+      "tokens_seen": 2128019456
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035869041887337506,
+      "loss": 1.216,
+      "theoretical_loss": 3.441243412915787,
+      "tokens_seen": 2128150528
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035865029690258386,
+      "loss": 1.381,
+      "theoretical_loss": 3.441226101501772,
+      "tokens_seen": 2128281600
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 715451,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2296104431152344,
+      "objective/train/theoretical_loss": 3.44120879145236,
+      "objective/train/tokens_used": 498871776,
+      "theoretical_loss": 3.44120879145236,
+      "tokens_seen": 2128412672
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035861017493179267,
+      "loss": 1.2176,
+      "theoretical_loss": 3.44120879145236,
+      "tokens_seen": 2128412672
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003585700529610014,
+      "loss": 1.2439,
+      "theoretical_loss": 3.4411914827673593,
+      "tokens_seen": 2128543744
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003585299309902102,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4411741754465774,
+      "tokens_seen": 2128674816
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035848980901941903,
+      "loss": 1.2646,
+      "theoretical_loss": 3.441156869489824,
+      "tokens_seen": 2128805888
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035844968704862784,
+      "loss": 1.2247,
+      "theoretical_loss": 3.441139564896907,
+      "tokens_seen": 2128936960
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035840956507783664,
+      "loss": 1.2397,
+      "theoretical_loss": 3.4411222616676356,
+      "tokens_seen": 2129068032
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035836944310704545,
+      "loss": 1.2926,
+      "theoretical_loss": 3.4411049598018177,
+      "tokens_seen": 2129199104
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003583293211362542,
+      "loss": 1.2134,
+      "theoretical_loss": 3.441087659299262,
+      "tokens_seen": 2129330176
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035828919916546306,
+      "loss": 1.2492,
+      "theoretical_loss": 3.4410703601597774,
+      "tokens_seen": 2129461248
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003582490771946718,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4410530623831725,
+      "tokens_seen": 2129592320
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035820895522388057,
+      "loss": 1.3286,
+      "theoretical_loss": 3.4410357659692563,
+      "tokens_seen": 2129723392
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035816883325308943,
+      "loss": 1.2752,
+      "theoretical_loss": 3.4410184709178377,
+      "tokens_seen": 2129854464
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003581287112822982,
+      "loss": 1.3049,
+      "theoretical_loss": 3.441001177228725,
+      "tokens_seen": 2129985536
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000358088589311507,
+      "loss": 1.269,
+      "theoretical_loss": 3.440983884901727,
+      "tokens_seen": 2130116608
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003580484673407158,
+      "loss": 1.2321,
+      "theoretical_loss": 3.440966593936653,
+      "tokens_seen": 2130247680
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003580083453699246,
+      "loss": 1.2811,
+      "theoretical_loss": 3.440949304333313,
+      "tokens_seen": 2130378752
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035796822339913335,
+      "loss": 1.2438,
+      "theoretical_loss": 3.440932016091514,
+      "tokens_seen": 2130509824
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035792810142834216,
+      "loss": 1.2702,
+      "theoretical_loss": 3.4409147292110656,
+      "tokens_seen": 2130640896
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035788797945755096,
+      "loss": 1.2069,
+      "theoretical_loss": 3.4408974436917776,
+      "tokens_seen": 2130771968
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003578478574867597,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4408801595334584,
+      "tokens_seen": 2130903040
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003578077355159686,
+      "loss": 1.2793,
+      "theoretical_loss": 3.4408628767359177,
+      "tokens_seen": 2131034112
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035776761354517733,
+      "loss": 1.296,
+      "theoretical_loss": 3.4408455952989643,
+      "tokens_seen": 2131165184
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003577274915743862,
+      "loss": 1.2821,
+      "theoretical_loss": 3.440828315222407,
+      "tokens_seen": 2131296256
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035768736960359494,
+      "loss": 1.2614,
+      "theoretical_loss": 3.4408110365060565,
+      "tokens_seen": 2131427328
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003576472476328037,
+      "loss": 1.2429,
+      "theoretical_loss": 3.4407937591497206,
+      "tokens_seen": 2131558400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035760712566201255,
+      "loss": 1.2216,
+      "theoretical_loss": 3.4407764831532095,
+      "tokens_seen": 2131689472
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003575670036912213,
+      "loss": 1.2782,
+      "theoretical_loss": 3.440759208516332,
+      "tokens_seen": 2131820544
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003575268817204301,
+      "loss": 1.2687,
+      "theoretical_loss": 3.440741935238898,
+      "tokens_seen": 2131951616
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003574867597496389,
+      "loss": 1.2709,
+      "theoretical_loss": 3.440724663320716,
+      "tokens_seen": 2132082688
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003574466377788477,
+      "loss": 1.2617,
+      "theoretical_loss": 3.440707392761597,
+      "tokens_seen": 2132213760
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003574065158080565,
+      "loss": 1.3367,
+      "theoretical_loss": 3.4406901235613496,
+      "tokens_seen": 2132344832
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003573663938372653,
+      "loss": 1.3159,
+      "theoretical_loss": 3.440672855719783,
+      "tokens_seen": 2132475904
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003573262718664741,
+      "loss": 1.2282,
+      "theoretical_loss": 3.440655589236708,
+      "tokens_seen": 2132606976
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035728614989568284,
+      "loss": 1.2756,
+      "theoretical_loss": 3.440638324111933,
+      "tokens_seen": 2132738048
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003572460279248917,
+      "loss": 1.308,
+      "theoretical_loss": 3.440621060345269,
+      "tokens_seen": 2132869120
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035720590595410045,
+      "loss": 1.3534,
+      "theoretical_loss": 3.4406037979365243,
+      "tokens_seen": 2133000192
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003571657839833093,
+      "loss": 1.2558,
+      "theoretical_loss": 3.440586536885509,
+      "tokens_seen": 2133131264
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035712566201251807,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4405692771920338,
+      "tokens_seen": 2133262336
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003570855400417268,
+      "loss": 1.2318,
+      "theoretical_loss": 3.4405520188559082,
+      "tokens_seen": 2133393408
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003570454180709357,
+      "loss": 1.2654,
+      "theoretical_loss": 3.440534761876941,
+      "tokens_seen": 2133524480
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035700529610014443,
+      "loss": 1.3162,
+      "theoretical_loss": 3.4405175062549436,
+      "tokens_seen": 2133655552
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035696517412935324,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4405002519897243,
+      "tokens_seen": 2133786624
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035692505215856204,
+      "loss": 1.3088,
+      "theoretical_loss": 3.440482999081095,
+      "tokens_seen": 2133917696
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035688493018777085,
+      "loss": 1.2957,
+      "theoretical_loss": 3.4404657475288642,
+      "tokens_seen": 2134048768
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003568448082169796,
+      "loss": 1.3535,
+      "theoretical_loss": 3.440448497332843,
+      "tokens_seen": 2134179840
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003568046862461884,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4404312484928408,
+      "tokens_seen": 2134310912
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003567645642753972,
+      "loss": 1.2661,
+      "theoretical_loss": 3.440414001008668,
+      "tokens_seen": 2134441984
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035672444230460597,
+      "loss": 1.2732,
+      "theoretical_loss": 3.440396754880134,
+      "tokens_seen": 2134573056
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035668432033381483,
+      "loss": 1.3129,
+      "theoretical_loss": 3.440379510107051,
+      "tokens_seen": 2134704128
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003566441983630236,
+      "loss": 1.3314,
+      "theoretical_loss": 3.4403622666892275,
+      "tokens_seen": 2134835200
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 717705,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2744487524032593,
+      "objective/train/theoretical_loss": 3.440345024626474,
+      "objective/train/tokens_used": 505425376,
+      "theoretical_loss": 3.440345024626474,
+      "tokens_seen": 2134966272
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003566040763922324,
+      "loss": 1.2766,
+      "theoretical_loss": 3.440345024626474,
+      "tokens_seen": 2134966272
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003565639544214412,
+      "loss": 1.3119,
+      "theoretical_loss": 3.4403277839186015,
+      "tokens_seen": 2135097344
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035652383245065,
+      "loss": 1.2104,
+      "theoretical_loss": 3.4403105445654196,
+      "tokens_seen": 2135228416
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003564837104798588,
+      "loss": 1.3406,
+      "theoretical_loss": 3.4402933065667396,
+      "tokens_seen": 2135359488
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035644358850906756,
+      "loss": 1.242,
+      "theoretical_loss": 3.4402760699223713,
+      "tokens_seen": 2135490560
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035640346653827636,
+      "loss": 1.3101,
+      "theoretical_loss": 3.4402588346321252,
+      "tokens_seen": 2135621632
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035636334456748517,
+      "loss": 1.3194,
+      "theoretical_loss": 3.440241600695812,
+      "tokens_seen": 2135752704
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000356323222596694,
+      "loss": 1.255,
+      "theoretical_loss": 3.4402243681132427,
+      "tokens_seen": 2135883776
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035628310062590273,
+      "loss": 1.2777,
+      "theoretical_loss": 3.4402071368842266,
+      "tokens_seen": 2136014848
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003562429786551116,
+      "loss": 1.3023,
+      "theoretical_loss": 3.4401899070085755,
+      "tokens_seen": 2136145920
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035620285668432034,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4401726784860998,
+      "tokens_seen": 2136276992
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003561627347135291,
+      "loss": 1.2647,
+      "theoretical_loss": 3.44015545131661,
+      "tokens_seen": 2136408064
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035612261274273795,
+      "loss": 1.2628,
+      "theoretical_loss": 3.4401382254999175,
+      "tokens_seen": 2136539136
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003560824907719467,
+      "loss": 1.2576,
+      "theoretical_loss": 3.440121001035832,
+      "tokens_seen": 2136670208
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003560423688011555,
+      "loss": 1.2907,
+      "theoretical_loss": 3.4401037779241648,
+      "tokens_seen": 2136801280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003560022468303643,
+      "loss": 1.2752,
+      "theoretical_loss": 3.440086556164727,
+      "tokens_seen": 2136932352
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003559621248595731,
+      "loss": 1.2843,
+      "theoretical_loss": 3.4400693357573298,
+      "tokens_seen": 2137063424
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003559220028887819,
+      "loss": 1.2928,
+      "theoretical_loss": 3.440052116701783,
+      "tokens_seen": 2137194496
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003558818809179907,
+      "loss": 1.1641,
+      "theoretical_loss": 3.440034898997899,
+      "tokens_seen": 2137325568
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003558417589471995,
+      "loss": 1.3476,
+      "theoretical_loss": 3.440017682645488,
+      "tokens_seen": 2137456640
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003558016369764083,
+      "loss": 1.2415,
+      "theoretical_loss": 3.44000046764436,
+      "tokens_seen": 2137587712
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003557615150056171,
+      "loss": 1.2899,
+      "theoretical_loss": 3.4399832539943285,
+      "tokens_seen": 2137718784
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035572139303482585,
+      "loss": 1.2801,
+      "theoretical_loss": 3.439966041695203,
+      "tokens_seen": 2137849856
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003556812710640347,
+      "loss": 1.2015,
+      "theoretical_loss": 3.439948830746795,
+      "tokens_seen": 2137980928
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035564114909324347,
+      "loss": 1.2752,
+      "theoretical_loss": 3.439931621148916,
+      "tokens_seen": 2138112000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003556010271224522,
+      "loss": 1.2861,
+      "theoretical_loss": 3.4399144129013766,
+      "tokens_seen": 2138243072
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003555609051516611,
+      "loss": 1.2922,
+      "theoretical_loss": 3.4398972060039887,
+      "tokens_seen": 2138374144
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035552078318086983,
+      "loss": 1.24,
+      "theoretical_loss": 3.4398800004565633,
+      "tokens_seen": 2138505216
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035548066121007864,
+      "loss": 1.2741,
+      "theoretical_loss": 3.439862796258912,
+      "tokens_seen": 2138636288
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035544053923928744,
+      "loss": 1.267,
+      "theoretical_loss": 3.439845593410846,
+      "tokens_seen": 2138767360
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035540041726849625,
+      "loss": 1.2648,
+      "theoretical_loss": 3.439828391912177,
+      "tokens_seen": 2138898432
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000355360295297705,
+      "loss": 1.1986,
+      "theoretical_loss": 3.4398111917627157,
+      "tokens_seen": 2139029504
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003553201733269138,
+      "loss": 1.2852,
+      "theoretical_loss": 3.439793992962274,
+      "tokens_seen": 2139160576
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003552800513561226,
+      "loss": 1.2443,
+      "theoretical_loss": 3.439776795510664,
+      "tokens_seen": 2139291648
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003552399293853314,
+      "loss": 1.2683,
+      "theoretical_loss": 3.4397595994076973,
+      "tokens_seen": 2139422720
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035519980741454023,
+      "loss": 1.2471,
+      "theoretical_loss": 3.4397424046531846,
+      "tokens_seen": 2139553792
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000355159685443749,
+      "loss": 1.2335,
+      "theoretical_loss": 3.4397252112469383,
+      "tokens_seen": 2139684864
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035511956347295784,
+      "loss": 1.2423,
+      "theoretical_loss": 3.4397080191887692,
+      "tokens_seen": 2139815936
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003550794415021666,
+      "loss": 1.3182,
+      "theoretical_loss": 3.439690828478491,
+      "tokens_seen": 2139947008
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003550393195313754,
+      "loss": 1.2701,
+      "theoretical_loss": 3.439673639115913,
+      "tokens_seen": 2140078080
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003549991975605842,
+      "loss": 1.2814,
+      "theoretical_loss": 3.4396564511008485,
+      "tokens_seen": 2140209152
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035495907558979296,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4396392644331097,
+      "tokens_seen": 2140340224
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035491895361900176,
+      "loss": 1.3272,
+      "theoretical_loss": 3.4396220791125067,
+      "tokens_seen": 2140471296
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035487883164821057,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4396048951388534,
+      "tokens_seen": 2140602368
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003548387096774194,
+      "loss": 1.3227,
+      "theoretical_loss": 3.439587712511961,
+      "tokens_seen": 2140733440
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035479858770662813,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4395705312316407,
+      "tokens_seen": 2140864512
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000354758465735837,
+      "loss": 1.2832,
+      "theoretical_loss": 3.439553351297706,
+      "tokens_seen": 2140995584
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035471834376504574,
+      "loss": 1.3532,
+      "theoretical_loss": 3.439536172709968,
+      "tokens_seen": 2141126656
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003546782217942545,
+      "loss": 1.2401,
+      "theoretical_loss": 3.439518995468239,
+      "tokens_seen": 2141257728
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035463809982346335,
+      "loss": 1.2189,
+      "theoretical_loss": 3.4395018195723313,
+      "tokens_seen": 2141388800
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 719922,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3278145790100098,
+      "objective/train/theoretical_loss": 3.4394846450220564,
+      "objective/train/tokens_used": 511978976,
+      "theoretical_loss": 3.4394846450220564,
+      "tokens_seen": 2141519872
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003545979778526721,
+      "loss": 1.293,
+      "theoretical_loss": 3.4394846450220564,
+      "tokens_seen": 2141519872
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035455785588188097,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4394674718172276,
+      "tokens_seen": 2141650944
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003545177339110897,
+      "loss": 1.2982,
+      "theoretical_loss": 3.439450299957657,
+      "tokens_seen": 2141782016
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003544776119402985,
+      "loss": 1.3623,
+      "theoretical_loss": 3.4394331294431564,
+      "tokens_seen": 2141913088
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035443748996950733,
+      "loss": 1.3251,
+      "theoretical_loss": 3.4394159602735384,
+      "tokens_seen": 2142044160
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003543973679987161,
+      "loss": 1.2357,
+      "theoretical_loss": 3.439398792448615,
+      "tokens_seen": 2142175232
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003543572460279249,
+      "loss": 1.2952,
+      "theoretical_loss": 3.439381625968199,
+      "tokens_seen": 2142306304
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003543171240571337,
+      "loss": 1.2487,
+      "theoretical_loss": 3.4393644608321026,
+      "tokens_seen": 2142437376
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003542770020863425,
+      "loss": 1.1828,
+      "theoretical_loss": 3.439347297040139,
+      "tokens_seen": 2142568448
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035423688011555125,
+      "loss": 1.2672,
+      "theoretical_loss": 3.4393301345921197,
+      "tokens_seen": 2142699520
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003541967581447601,
+      "loss": 1.2909,
+      "theoretical_loss": 3.4393129734878576,
+      "tokens_seen": 2142830592
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035415663617396887,
+      "loss": 1.3088,
+      "theoretical_loss": 3.439295813727166,
+      "tokens_seen": 2142961664
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003541165142031776,
+      "loss": 1.2976,
+      "theoretical_loss": 3.439278655309857,
+      "tokens_seen": 2143092736
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003540763922323865,
+      "loss": 1.2771,
+      "theoretical_loss": 3.439261498235743,
+      "tokens_seen": 2143223808
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035403627026159523,
+      "loss": 1.3076,
+      "theoretical_loss": 3.439244342504637,
+      "tokens_seen": 2143354880
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003539961482908041,
+      "loss": 1.2861,
+      "theoretical_loss": 3.4392271881163516,
+      "tokens_seen": 2143485952
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035395602632001284,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4392100350707002,
+      "tokens_seen": 2143617024
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035391590434922165,
+      "loss": 1.2274,
+      "theoretical_loss": 3.439192883367495,
+      "tokens_seen": 2143748096
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035387578237843046,
+      "loss": 1.29,
+      "theoretical_loss": 3.4391757330065493,
+      "tokens_seen": 2143879168
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003538356604076392,
+      "loss": 1.3252,
+      "theoretical_loss": 3.439158583987675,
+      "tokens_seen": 2144010240
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000353795538436848,
+      "loss": 1.258,
+      "theoretical_loss": 3.4391414363106865,
+      "tokens_seen": 2144141312
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003537554164660568,
+      "loss": 1.2261,
+      "theoretical_loss": 3.439124289975396,
+      "tokens_seen": 2144272384
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035371529449526563,
+      "loss": 1.3187,
+      "theoretical_loss": 3.4391071449816164,
+      "tokens_seen": 2144403456
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003536751725244744,
+      "loss": 1.2694,
+      "theoretical_loss": 3.439090001329161,
+      "tokens_seen": 2144534528
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035363505055368324,
+      "loss": 1.263,
+      "theoretical_loss": 3.4390728590178425,
+      "tokens_seen": 2144665600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000353594928582892,
+      "loss": 1.2428,
+      "theoretical_loss": 3.4390557180474746,
+      "tokens_seen": 2144796672
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003535548066121008,
+      "loss": 1.1982,
+      "theoretical_loss": 3.4390385784178705,
+      "tokens_seen": 2144927744
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003535146846413096,
+      "loss": 1.283,
+      "theoretical_loss": 3.439021440128843,
+      "tokens_seen": 2145058816
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035347456267051836,
+      "loss": 1.2615,
+      "theoretical_loss": 3.439004303180205,
+      "tokens_seen": 2145189888
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035343444069972716,
+      "loss": 1.2981,
+      "theoretical_loss": 3.4389871675717707,
+      "tokens_seen": 2145320960
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035339431872893597,
+      "loss": 1.2776,
+      "theoretical_loss": 3.4389700333033533,
+      "tokens_seen": 2145452032
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003533541967581448,
+      "loss": 1.2793,
+      "theoretical_loss": 3.4389529003747654,
+      "tokens_seen": 2145583104
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003533140747873536,
+      "loss": 1.3495,
+      "theoretical_loss": 3.4389357687858206,
+      "tokens_seen": 2145714176
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003532739528165624,
+      "loss": 1.2548,
+      "theoretical_loss": 3.4389186385363333,
+      "tokens_seen": 2145845248
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035323383084577114,
+      "loss": 1.2424,
+      "theoretical_loss": 3.438901509626115,
+      "tokens_seen": 2145976320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035319370887497995,
+      "loss": 1.2527,
+      "theoretical_loss": 3.4388843820549813,
+      "tokens_seen": 2146107392
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035315358690418875,
+      "loss": 1.2786,
+      "theoretical_loss": 3.438867255822745,
+      "tokens_seen": 2146238464
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003531134649333975,
+      "loss": 1.223,
+      "theoretical_loss": 3.438850130929219,
+      "tokens_seen": 2146369536
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035307334296260637,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4388330073742175,
+      "tokens_seen": 2146500608
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003530332209918151,
+      "loss": 1.3296,
+      "theoretical_loss": 3.4388158851575543,
+      "tokens_seen": 2146631680
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003529930990210239,
+      "loss": 1.2642,
+      "theoretical_loss": 3.438798764279043,
+      "tokens_seen": 2146762752
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035295297705023273,
+      "loss": 1.2435,
+      "theoretical_loss": 3.438781644738497,
+      "tokens_seen": 2146893824
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003529128550794415,
+      "loss": 1.2858,
+      "theoretical_loss": 3.4387645265357305,
+      "tokens_seen": 2147024896
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003528727331086503,
+      "loss": 1.2829,
+      "theoretical_loss": 3.438747409670557,
+      "tokens_seen": 2147155968
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003528326111378591,
+      "loss": 1.3171,
+      "theoretical_loss": 3.4387302941427897,
+      "tokens_seen": 2147287040
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003527924891670679,
+      "loss": 1.2366,
+      "theoretical_loss": 3.438713179952244,
+      "tokens_seen": 2147418112
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035275236719627665,
+      "loss": 1.3092,
+      "theoretical_loss": 3.438696067098733,
+      "tokens_seen": 2147549184
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003527122452254855,
+      "loss": 1.3124,
+      "theoretical_loss": 3.4386789555820707,
+      "tokens_seen": 2147680256
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035267212325469427,
+      "loss": 1.2731,
+      "theoretical_loss": 3.4386618454020708,
+      "tokens_seen": 2147811328
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035263200128390307,
+      "loss": 1.3107,
+      "theoretical_loss": 3.4386447365585475,
+      "tokens_seen": 2147942400
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 721999,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3769314289093018,
+      "objective/train/theoretical_loss": 3.4386276290513154,
+      "objective/train/tokens_used": 518532576,
+      "theoretical_loss": 3.4386276290513154,
+      "tokens_seen": 2148073472
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003525918793131119,
+      "loss": 1.3277,
+      "theoretical_loss": 3.4386276290513154,
+      "tokens_seen": 2148073472
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035255175734232063,
+      "loss": 1.1745,
+      "theoretical_loss": 3.4386105228801878,
+      "tokens_seen": 2148204544
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003525116353715295,
+      "loss": 1.1551,
+      "theoretical_loss": 3.438593418044979,
+      "tokens_seen": 2148335616
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035247151340073824,
+      "loss": 1.2743,
+      "theoretical_loss": 3.4385763145455037,
+      "tokens_seen": 2148466688
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035243139142994705,
+      "loss": 1.2746,
+      "theoretical_loss": 3.438559212381575,
+      "tokens_seen": 2148597760
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035239126945915586,
+      "loss": 1.2348,
+      "theoretical_loss": 3.4385421115530086,
+      "tokens_seen": 2148728832
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003523511474883646,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4385250120596185,
+      "tokens_seen": 2148859904
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003523110255175734,
+      "loss": 1.2969,
+      "theoretical_loss": 3.4385079139012182,
+      "tokens_seen": 2148990976
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003522709035467822,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4384908170776223,
+      "tokens_seen": 2149122048
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035223078157599103,
+      "loss": 1.237,
+      "theoretical_loss": 3.4384737215886454,
+      "tokens_seen": 2149253120
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003521906596051998,
+      "loss": 1.3236,
+      "theoretical_loss": 3.438456627434102,
+      "tokens_seen": 2149384192
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035215053763440864,
+      "loss": 1.2312,
+      "theoretical_loss": 3.4384395346138064,
+      "tokens_seen": 2149515264
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003521104156636174,
+      "loss": 1.1877,
+      "theoretical_loss": 3.4384224431275734,
+      "tokens_seen": 2149646336
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003520702936928262,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4384053529752174,
+      "tokens_seen": 2149777408
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000352030171722035,
+      "loss": 1.2868,
+      "theoretical_loss": 3.4383882641565524,
+      "tokens_seen": 2149908480
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035199004975124376,
+      "loss": 1.2349,
+      "theoretical_loss": 3.438371176671394,
+      "tokens_seen": 2150039552
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003519499277804526,
+      "loss": 1.2884,
+      "theoretical_loss": 3.438354090519556,
+      "tokens_seen": 2150170624
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035190980580966137,
+      "loss": 1.2139,
+      "theoretical_loss": 3.4383370057008538,
+      "tokens_seen": 2150301696
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003518696838388702,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4383199222151015,
+      "tokens_seen": 2150432768
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000351829561868079,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4383028400621147,
+      "tokens_seen": 2150563840
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003517894398972878,
+      "loss": 1.2953,
+      "theoretical_loss": 3.4382857592417073,
+      "tokens_seen": 2150694912
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035174931792649654,
+      "loss": 1.3031,
+      "theoretical_loss": 3.438268679753695,
+      "tokens_seen": 2150825984
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035170919595570535,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4382516015978917,
+      "tokens_seen": 2150957056
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035166907398491415,
+      "loss": 1.2446,
+      "theoretical_loss": 3.4382345247741135,
+      "tokens_seen": 2151088128
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003516289520141229,
+      "loss": 1.2701,
+      "theoretical_loss": 3.438217449282174,
+      "tokens_seen": 2151219200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035158883004333177,
+      "loss": 1.2187,
+      "theoretical_loss": 3.4382003751218893,
+      "tokens_seen": 2151350272
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003515487080725405,
+      "loss": 1.2484,
+      "theoretical_loss": 3.4381833022930737,
+      "tokens_seen": 2151481344
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003515085861017493,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4381662307955425,
+      "tokens_seen": 2151612416
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035146846413095813,
+      "loss": 1.2261,
+      "theoretical_loss": 3.438149160629111,
+      "tokens_seen": 2151743488
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003514283421601669,
+      "loss": 1.2622,
+      "theoretical_loss": 3.4381320917935945,
+      "tokens_seen": 2151874560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035138822018937574,
+      "loss": 1.2288,
+      "theoretical_loss": 3.4381150242888077,
+      "tokens_seen": 2152005632
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003513480982185845,
+      "loss": 1.2093,
+      "theoretical_loss": 3.438097958114566,
+      "tokens_seen": 2152136704
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003513079762477933,
+      "loss": 1.3275,
+      "theoretical_loss": 3.4380808932706843,
+      "tokens_seen": 2152267776
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003512678542770021,
+      "loss": 1.234,
+      "theoretical_loss": 3.4380638297569783,
+      "tokens_seen": 2152398848
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003512277323062109,
+      "loss": 1.2481,
+      "theoretical_loss": 3.4380467675732636,
+      "tokens_seen": 2152529920
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035118761033541967,
+      "loss": 1.2419,
+      "theoretical_loss": 3.4380297067193553,
+      "tokens_seen": 2152660992
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035114748836462847,
+      "loss": 1.2735,
+      "theoretical_loss": 3.4380126471950683,
+      "tokens_seen": 2152792064
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003511073663938373,
+      "loss": 1.3152,
+      "theoretical_loss": 3.4379955890002183,
+      "tokens_seen": 2152923136
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035106724442304603,
+      "loss": 1.2746,
+      "theoretical_loss": 3.4379785321346215,
+      "tokens_seen": 2153054208
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003510271224522549,
+      "loss": 1.2905,
+      "theoretical_loss": 3.437961476598092,
+      "tokens_seen": 2153185280
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035098700048146364,
+      "loss": 1.2659,
+      "theoretical_loss": 3.437944422390447,
+      "tokens_seen": 2153316352
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035094687851067245,
+      "loss": 1.2468,
+      "theoretical_loss": 3.4379273695115007,
+      "tokens_seen": 2153447424
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035090675653988126,
+      "loss": 1.2326,
+      "theoretical_loss": 3.4379103179610695,
+      "tokens_seen": 2153578496
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035086663456909,
+      "loss": 1.3597,
+      "theoretical_loss": 3.437893267738969,
+      "tokens_seen": 2153709568
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035082651259829887,
+      "loss": 1.2884,
+      "theoretical_loss": 3.4378762188450143,
+      "tokens_seen": 2153840640
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003507863906275076,
+      "loss": 1.2793,
+      "theoretical_loss": 3.4378591712790216,
+      "tokens_seen": 2153971712
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003507462686567164,
+      "loss": 1.2698,
+      "theoretical_loss": 3.437842125040807,
+      "tokens_seen": 2154102784
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035070614668592523,
+      "loss": 1.2641,
+      "theoretical_loss": 3.4378250801301853,
+      "tokens_seen": 2154233856
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035066602471513404,
+      "loss": 1.2059,
+      "theoretical_loss": 3.4378080365469734,
+      "tokens_seen": 2154364928
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003506259027443428,
+      "loss": 1.3117,
+      "theoretical_loss": 3.4377909942909866,
+      "tokens_seen": 2154496000
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 724229,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.218306064605713,
+      "objective/train/theoretical_loss": 3.437773953362041,
+      "objective/train/tokens_used": 525086176,
+      "theoretical_loss": 3.437773953362041,
+      "tokens_seen": 2154627072
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003505857807735516,
+      "loss": 1.302,
+      "theoretical_loss": 3.437773953362041,
+      "tokens_seen": 2154627072
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003505456588027604,
+      "loss": 1.3157,
+      "theoretical_loss": 3.4377569137599524,
+      "tokens_seen": 2154758144
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035050553683196916,
+      "loss": 1.2819,
+      "theoretical_loss": 3.4377398754845365,
+      "tokens_seen": 2154889216
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.000350465414861178,
+      "loss": 1.2683,
+      "theoretical_loss": 3.43772283853561,
+      "tokens_seen": 2155020288
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035042529289038677,
+      "loss": 1.276,
+      "theoretical_loss": 3.4377058029129888,
+      "tokens_seen": 2155151360
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003503851709195956,
+      "loss": 1.2697,
+      "theoretical_loss": 3.4376887686164883,
+      "tokens_seen": 2155282432
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003503450489488044,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4376717356459254,
+      "tokens_seen": 2155413504
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003503049269780132,
+      "loss": 1.3243,
+      "theoretical_loss": 3.4376547040011163,
+      "tokens_seen": 2155544576
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035026480500722194,
+      "loss": 1.2689,
+      "theoretical_loss": 3.437637673681876,
+      "tokens_seen": 2155675648
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035022468303643075,
+      "loss": 1.2127,
+      "theoretical_loss": 3.437620644688023,
+      "tokens_seen": 2155806720
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035018456106563955,
+      "loss": 1.3105,
+      "theoretical_loss": 3.437603617019372,
+      "tokens_seen": 2155937792
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035014443909484836,
+      "loss": 1.2941,
+      "theoretical_loss": 3.4375865906757386,
+      "tokens_seen": 2156068864
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035010431712405717,
+      "loss": 1.3113,
+      "theoretical_loss": 3.4375695656569407,
+      "tokens_seen": 2156199936
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003500641951532659,
+      "loss": 1.3328,
+      "theoretical_loss": 3.437552541962794,
+      "tokens_seen": 2156331008
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003500240731824748,
+      "loss": 1.2858,
+      "theoretical_loss": 3.437535519593115,
+      "tokens_seen": 2156462080
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034998395121168353,
+      "loss": 1.2929,
+      "theoretical_loss": 3.4375184985477203,
+      "tokens_seen": 2156593152
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003499438292408923,
+      "loss": 1.3151,
+      "theoretical_loss": 3.4375014788264266,
+      "tokens_seen": 2156724224
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034990370727010114,
+      "loss": 1.2893,
+      "theoretical_loss": 3.43748446042905,
+      "tokens_seen": 2156855296
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003498635852993099,
+      "loss": 1.3212,
+      "theoretical_loss": 3.437467443355407,
+      "tokens_seen": 2156986368
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003498234633285187,
+      "loss": 1.2047,
+      "theoretical_loss": 3.4374504276053144,
+      "tokens_seen": 2157117440
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003497833413577275,
+      "loss": 1.3036,
+      "theoretical_loss": 3.4374334131785886,
+      "tokens_seen": 2157248512
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003497432193869363,
+      "loss": 1.3149,
+      "theoretical_loss": 3.4374164000750467,
+      "tokens_seen": 2157379584
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034970309741614507,
+      "loss": 1.3402,
+      "theoretical_loss": 3.437399388294506,
+      "tokens_seen": 2157510656
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034966297544535387,
+      "loss": 1.2656,
+      "theoretical_loss": 3.4373823778367814,
+      "tokens_seen": 2157641728
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003496228534745627,
+      "loss": 1.3154,
+      "theoretical_loss": 3.437365368701691,
+      "tokens_seen": 2157772800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034958273150377143,
+      "loss": 1.2686,
+      "theoretical_loss": 3.437348360889052,
+      "tokens_seen": 2157903872
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003495426095329803,
+      "loss": 1.2775,
+      "theoretical_loss": 3.4373313543986805,
+      "tokens_seen": 2158034944
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034950248756218904,
+      "loss": 1.2651,
+      "theoretical_loss": 3.4373143492303933,
+      "tokens_seen": 2158166016
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003494623655913979,
+      "loss": 1.3369,
+      "theoretical_loss": 3.4372973453840077,
+      "tokens_seen": 2158297088
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034942224362060666,
+      "loss": 1.2771,
+      "theoretical_loss": 3.4372803428593404,
+      "tokens_seen": 2158428160
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003493821216498154,
+      "loss": 1.2488,
+      "theoretical_loss": 3.437263341656209,
+      "tokens_seen": 2158559232
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034934199967902427,
+      "loss": 1.2848,
+      "theoretical_loss": 3.43724634177443,
+      "tokens_seen": 2158690304
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.000349301877708233,
+      "loss": 1.284,
+      "theoretical_loss": 3.4372293432138203,
+      "tokens_seen": 2158821376
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003492617557374418,
+      "loss": 1.2621,
+      "theoretical_loss": 3.437212345974198,
+      "tokens_seen": 2158952448
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034922163376665063,
+      "loss": 1.3345,
+      "theoretical_loss": 3.437195350055379,
+      "tokens_seen": 2159083520
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034918151179585944,
+      "loss": 1.2163,
+      "theoretical_loss": 3.4371783554571813,
+      "tokens_seen": 2159214592
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003491413898250682,
+      "loss": 1.3833,
+      "theoretical_loss": 3.437161362179422,
+      "tokens_seen": 2159345664
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.000349101267854277,
+      "loss": 1.3518,
+      "theoretical_loss": 3.4371443702219184,
+      "tokens_seen": 2159476736
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003490611458834858,
+      "loss": 1.3384,
+      "theoretical_loss": 3.437127379584487,
+      "tokens_seen": 2159607808
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034902102391269456,
+      "loss": 1.3191,
+      "theoretical_loss": 3.4371103902669464,
+      "tokens_seen": 2159738880
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003489809019419034,
+      "loss": 1.3035,
+      "theoretical_loss": 3.4370934022691135,
+      "tokens_seen": 2159869952
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034894077997111217,
+      "loss": 1.2917,
+      "theoretical_loss": 3.4370764155908056,
+      "tokens_seen": 2160001024
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034890065800032103,
+      "loss": 1.2877,
+      "theoretical_loss": 3.43705943023184,
+      "tokens_seen": 2160132096
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003488605360295298,
+      "loss": 1.2506,
+      "theoretical_loss": 3.4370424461920344,
+      "tokens_seen": 2160263168
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003488204140587386,
+      "loss": 1.221,
+      "theoretical_loss": 3.437025463471206,
+      "tokens_seen": 2160394240
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003487802920879474,
+      "loss": 1.268,
+      "theoretical_loss": 3.4370084820691726,
+      "tokens_seen": 2160525312
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034874017011715615,
+      "loss": 1.2365,
+      "theoretical_loss": 3.436991501985752,
+      "tokens_seen": 2160656384
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034870004814636495,
+      "loss": 1.4085,
+      "theoretical_loss": 3.4369745232207616,
+      "tokens_seen": 2160787456
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034865992617557376,
+      "loss": 1.326,
+      "theoretical_loss": 3.436957545774019,
+      "tokens_seen": 2160918528
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034861980420478256,
+      "loss": 1.3001,
+      "theoretical_loss": 3.436940569645342,
+      "tokens_seen": 2161049600
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 726764,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2695492506027222,
+      "objective/train/theoretical_loss": 3.436923594834548,
+      "objective/train/tokens_used": 531639776,
+      "theoretical_loss": 3.436923594834548,
+      "tokens_seen": 2161180672
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003485796822339913,
+      "loss": 1.3093,
+      "theoretical_loss": 3.436923594834548,
+      "tokens_seen": 2161180672
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003485395602632002,
+      "loss": 1.2421,
+      "theoretical_loss": 3.436906621341456,
+      "tokens_seen": 2161311744
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034849943829240893,
+      "loss": 1.2918,
+      "theoretical_loss": 3.4368896491658822,
+      "tokens_seen": 2161442816
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003484593163216177,
+      "loss": 1.3016,
+      "theoretical_loss": 3.4368726783076453,
+      "tokens_seen": 2161573888
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034841919435082654,
+      "loss": 1.2377,
+      "theoretical_loss": 3.4368557087665628,
+      "tokens_seen": 2161704960
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003483790723800353,
+      "loss": 1.2463,
+      "theoretical_loss": 3.436838740542453,
+      "tokens_seen": 2161836032
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003483389504092441,
+      "loss": 1.3295,
+      "theoretical_loss": 3.436821773635134,
+      "tokens_seen": 2161967104
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003482988284384529,
+      "loss": 1.2969,
+      "theoretical_loss": 3.4368048080444233,
+      "tokens_seen": 2162098176
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003482587064676617,
+      "loss": 1.2596,
+      "theoretical_loss": 3.436787843770139,
+      "tokens_seen": 2162229248
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003482185844968705,
+      "loss": 1.2153,
+      "theoretical_loss": 3.4367708808120994,
+      "tokens_seen": 2162360320
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034817846252607927,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4367539191701226,
+      "tokens_seen": 2162491392
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003481383405552881,
+      "loss": 1.2427,
+      "theoretical_loss": 3.436736958844026,
+      "tokens_seen": 2162622464
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003480982185844969,
+      "loss": 1.2699,
+      "theoretical_loss": 3.4367199998336293,
+      "tokens_seen": 2162753536
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003480580966137057,
+      "loss": 1.2624,
+      "theoretical_loss": 3.436703042138749,
+      "tokens_seen": 2162884608
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034801797464291444,
+      "loss": 1.2991,
+      "theoretical_loss": 3.4366860857592045,
+      "tokens_seen": 2163015680
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003479778526721233,
+      "loss": 1.2543,
+      "theoretical_loss": 3.436669130694814,
+      "tokens_seen": 2163146752
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034793773070133206,
+      "loss": 1.3223,
+      "theoretical_loss": 3.436652176945395,
+      "tokens_seen": 2163277824
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003478976087305408,
+      "loss": 1.2393,
+      "theoretical_loss": 3.4366352245107668,
+      "tokens_seen": 2163408896
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034785748675974967,
+      "loss": 1.2552,
+      "theoretical_loss": 3.436618273390747,
+      "tokens_seen": 2163539968
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003478173647889584,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4366013235851547,
+      "tokens_seen": 2163671040
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003477772428181672,
+      "loss": 1.2705,
+      "theoretical_loss": 3.4365843750938074,
+      "tokens_seen": 2163802112
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034773712084737603,
+      "loss": 1.2651,
+      "theoretical_loss": 3.436567427916525,
+      "tokens_seen": 2163933184
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034769699887658484,
+      "loss": 1.2406,
+      "theoretical_loss": 3.436550482053125,
+      "tokens_seen": 2164064256
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003476568769057936,
+      "loss": 1.2858,
+      "theoretical_loss": 3.436533537503426,
+      "tokens_seen": 2164195328
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003476167549350024,
+      "loss": 1.3094,
+      "theoretical_loss": 3.436516594267247,
+      "tokens_seen": 2164326400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003475766329642112,
+      "loss": 1.2504,
+      "theoretical_loss": 3.436499652344407,
+      "tokens_seen": 2164457472
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034753651099342,
+      "loss": 1.2441,
+      "theoretical_loss": 3.4364827117347234,
+      "tokens_seen": 2164588544
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003474963890226288,
+      "loss": 1.3021,
+      "theoretical_loss": 3.436465772438016,
+      "tokens_seen": 2164719616
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034745626705183757,
+      "loss": 1.3363,
+      "theoretical_loss": 3.4364488344541035,
+      "tokens_seen": 2164850688
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034741614508104643,
+      "loss": 1.2043,
+      "theoretical_loss": 3.4364318977828043,
+      "tokens_seen": 2164981760
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003473760231102552,
+      "loss": 1.2493,
+      "theoretical_loss": 3.436414962423937,
+      "tokens_seen": 2165112832
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034733590113946393,
+      "loss": 1.3216,
+      "theoretical_loss": 3.436398028377321,
+      "tokens_seen": 2165243904
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003472957791686728,
+      "loss": 1.2797,
+      "theoretical_loss": 3.436381095642775,
+      "tokens_seen": 2165374976
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034725565719788155,
+      "loss": 1.226,
+      "theoretical_loss": 3.4363641642201177,
+      "tokens_seen": 2165506048
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034721553522709035,
+      "loss": 1.3246,
+      "theoretical_loss": 3.4363472341091685,
+      "tokens_seen": 2165637120
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034717541325629916,
+      "loss": 1.2735,
+      "theoretical_loss": 3.436330305309746,
+      "tokens_seen": 2165768192
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034713529128550796,
+      "loss": 1.2162,
+      "theoretical_loss": 3.4363133778216692,
+      "tokens_seen": 2165899264
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003470951693147167,
+      "loss": 1.2993,
+      "theoretical_loss": 3.4362964516447576,
+      "tokens_seen": 2166030336
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003470550473439256,
+      "loss": 1.3255,
+      "theoretical_loss": 3.43627952677883,
+      "tokens_seen": 2166161408
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034701492537313433,
+      "loss": 1.2285,
+      "theoretical_loss": 3.4362626032237062,
+      "tokens_seen": 2166292480
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034697480340234314,
+      "loss": 1.3035,
+      "theoretical_loss": 3.436245680979204,
+      "tokens_seen": 2166423552
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034693468143155194,
+      "loss": 1.2511,
+      "theoretical_loss": 3.436228760045144,
+      "tokens_seen": 2166554624
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003468945594607607,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4362118404213446,
+      "tokens_seen": 2166685696
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034685443748996955,
+      "loss": 1.3092,
+      "theoretical_loss": 3.4361949221076253,
+      "tokens_seen": 2166816768
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003468143155191783,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4361780051038058,
+      "tokens_seen": 2166947840
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003467741935483871,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4361610894097048,
+      "tokens_seen": 2167078912
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003467340715775959,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4361441750251425,
+      "tokens_seen": 2167209984
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034669394960680467,
+      "loss": 1.2629,
+      "theoretical_loss": 3.4361272619499372,
+      "tokens_seen": 2167341056
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003466538276360135,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4361103501839096,
+      "tokens_seen": 2167472128
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003466137056652223,
+      "loss": 1.2532,
+      "theoretical_loss": 3.4360934397268785,
+      "tokens_seen": 2167603200
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 728956,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.24422025680542,
+      "objective/train/theoretical_loss": 3.436076530578663,
+      "objective/train/tokens_used": 538193376,
+      "theoretical_loss": 3.436076530578663,
+      "tokens_seen": 2167734272
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003465735836944311,
+      "loss": 1.2635,
+      "theoretical_loss": 3.436076530578663,
+      "tokens_seen": 2167734272
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034653346172363984,
+      "loss": 1.2585,
+      "theoretical_loss": 3.4360596227390836,
+      "tokens_seen": 2167865344
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003464933397528487,
+      "loss": 1.299,
+      "theoretical_loss": 3.4360427162079596,
+      "tokens_seen": 2167996416
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034645321778205745,
+      "loss": 1.2167,
+      "theoretical_loss": 3.436025810985111,
+      "tokens_seen": 2168127488
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003464130958112662,
+      "loss": 1.3034,
+      "theoretical_loss": 3.4360089070703568,
+      "tokens_seen": 2168258560
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034637297384047507,
+      "loss": 1.3188,
+      "theoretical_loss": 3.4359920044635164,
+      "tokens_seen": 2168389632
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003463328518696838,
+      "loss": 1.2912,
+      "theoretical_loss": 3.435975103164411,
+      "tokens_seen": 2168520704
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003462927298988927,
+      "loss": 1.2963,
+      "theoretical_loss": 3.4359582031728593,
+      "tokens_seen": 2168651776
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034625260792810143,
+      "loss": 1.2735,
+      "theoretical_loss": 3.435941304488681,
+      "tokens_seen": 2168782848
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034621248595731024,
+      "loss": 1.2759,
+      "theoretical_loss": 3.4359244071116963,
+      "tokens_seen": 2168913920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034617236398651904,
+      "loss": 1.2582,
+      "theoretical_loss": 3.4359075110417256,
+      "tokens_seen": 2169044992
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003461322420157278,
+      "loss": 1.2869,
+      "theoretical_loss": 3.435890616278588,
+      "tokens_seen": 2169176064
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003460921200449366,
+      "loss": 1.2838,
+      "theoretical_loss": 3.435873722822104,
+      "tokens_seen": 2169307136
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003460519980741454,
+      "loss": 1.2497,
+      "theoretical_loss": 3.435856830672093,
+      "tokens_seen": 2169438208
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003460118761033542,
+      "loss": 1.3238,
+      "theoretical_loss": 3.435839939828376,
+      "tokens_seen": 2169569280
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034597175413256297,
+      "loss": 1.2926,
+      "theoretical_loss": 3.4358230502907725,
+      "tokens_seen": 2169700352
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034593163216177183,
+      "loss": 1.2725,
+      "theoretical_loss": 3.435806162059102,
+      "tokens_seen": 2169831424
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003458915101909806,
+      "loss": 1.3193,
+      "theoretical_loss": 3.4357892751331858,
+      "tokens_seen": 2169962496
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034585138822018933,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4357723895128434,
+      "tokens_seen": 2170093568
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003458112662493982,
+      "loss": 1.3142,
+      "theoretical_loss": 3.4357555051978954,
+      "tokens_seen": 2170224640
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034577114427860695,
+      "loss": 1.2678,
+      "theoretical_loss": 3.4357386221881616,
+      "tokens_seen": 2170355712
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003457310223078158,
+      "loss": 1.2974,
+      "theoretical_loss": 3.4357217404834626,
+      "tokens_seen": 2170486784
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034569090033702456,
+      "loss": 1.2758,
+      "theoretical_loss": 3.4357048600836184,
+      "tokens_seen": 2170617856
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034565077836623336,
+      "loss": 1.1928,
+      "theoretical_loss": 3.4356879809884497,
+      "tokens_seen": 2170748928
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034561065639544217,
+      "loss": 1.286,
+      "theoretical_loss": 3.435671103197777,
+      "tokens_seen": 2170880000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000345570534424651,
+      "loss": 1.2141,
+      "theoretical_loss": 3.4356542267114203,
+      "tokens_seen": 2171011072
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034553041245385973,
+      "loss": 1.3459,
+      "theoretical_loss": 3.4356373515292002,
+      "tokens_seen": 2171142144
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034549029048306854,
+      "loss": 1.2512,
+      "theoretical_loss": 3.4356204776509376,
+      "tokens_seen": 2171273216
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034545016851227734,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4356036050764525,
+      "tokens_seen": 2171404288
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003454100465414861,
+      "loss": 1.2776,
+      "theoretical_loss": 3.4355867338055655,
+      "tokens_seen": 2171535360
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034536992457069495,
+      "loss": 1.246,
+      "theoretical_loss": 3.435569863838098,
+      "tokens_seen": 2171666432
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003453298025999037,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4355529951738695,
+      "tokens_seen": 2171797504
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003452896806291125,
+      "loss": 1.3446,
+      "theoretical_loss": 3.4355361278127017,
+      "tokens_seen": 2171928576
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003452495586583213,
+      "loss": 1.3074,
+      "theoretical_loss": 3.4355192617544144,
+      "tokens_seen": 2172059648
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034520943668753007,
+      "loss": 1.3051,
+      "theoretical_loss": 3.435502396998829,
+      "tokens_seen": 2172190720
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003451693147167389,
+      "loss": 1.2116,
+      "theoretical_loss": 3.4354855335457657,
+      "tokens_seen": 2172321792
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003451291927459477,
+      "loss": 1.2965,
+      "theoretical_loss": 3.435468671395046,
+      "tokens_seen": 2172452864
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003450890707751565,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4354518105464904,
+      "tokens_seen": 2172583936
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003450489488043653,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4354349509999196,
+      "tokens_seen": 2172715008
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003450088268335741,
+      "loss": 1.3123,
+      "theoretical_loss": 3.435418092755155,
+      "tokens_seen": 2172846080
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034496870486278285,
+      "loss": 1.3499,
+      "theoretical_loss": 3.4354012358120176,
+      "tokens_seen": 2172977152
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034492858289199166,
+      "loss": 1.2697,
+      "theoretical_loss": 3.4353843801703277,
+      "tokens_seen": 2173108224
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034488846092120047,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4353675258299066,
+      "tokens_seen": 2173239296
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003448483389504092,
+      "loss": 1.2738,
+      "theoretical_loss": 3.435350672790576,
+      "tokens_seen": 2173370368
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003448082169796181,
+      "loss": 1.2419,
+      "theoretical_loss": 3.435333821052156,
+      "tokens_seen": 2173501440
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034476809500882683,
+      "loss": 1.2335,
+      "theoretical_loss": 3.435316970614469,
+      "tokens_seen": 2173632512
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034472797303803564,
+      "loss": 1.3056,
+      "theoretical_loss": 3.4353001214773347,
+      "tokens_seen": 2173763584
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034468785106724444,
+      "loss": 1.2336,
+      "theoretical_loss": 3.435283273640575,
+      "tokens_seen": 2173894656
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003446477290964532,
+      "loss": 1.2104,
+      "theoretical_loss": 3.435266427104012,
+      "tokens_seen": 2174025728
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000344607607125662,
+      "loss": 1.3063,
+      "theoretical_loss": 3.4352495818674655,
+      "tokens_seen": 2174156800
+    },
+    {
+      "epoch": 0.32,
+      "objective/train/docs_used": 731141,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2807493209838867,
+      "objective/train/theoretical_loss": 3.435232737930758,
+      "objective/train/tokens_used": 544746976,
+      "theoretical_loss": 3.435232737930758,
+      "tokens_seen": 2174287872
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003445674851548708,
+      "loss": 1.3092,
+      "theoretical_loss": 3.435232737930758,
+      "tokens_seen": 2174287872
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003445273631840796,
+      "loss": 1.1953,
+      "theoretical_loss": 3.4352158952937097,
+      "tokens_seen": 2174418944
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034448724121328837,
+      "loss": 1.2201,
+      "theoretical_loss": 3.435199053956143,
+      "tokens_seen": 2174550016
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034444711924249723,
+      "loss": 1.3015,
+      "theoretical_loss": 3.435182213917879,
+      "tokens_seen": 2174681088
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000344406997271706,
+      "loss": 1.2582,
+      "theoretical_loss": 3.435165375178739,
+      "tokens_seen": 2174812160
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003443668753009148,
+      "loss": 1.284,
+      "theoretical_loss": 3.4351485377385447,
+      "tokens_seen": 2174943232
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003443267533301236,
+      "loss": 1.3273,
+      "theoretical_loss": 3.4351317015971174,
+      "tokens_seen": 2175074304
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034428663135933234,
+      "loss": 1.2206,
+      "theoretical_loss": 3.4351148667542786,
+      "tokens_seen": 2175205376
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003442465093885412,
+      "loss": 1.2588,
+      "theoretical_loss": 3.4350980332098504,
+      "tokens_seen": 2175336448
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034420638741774996,
+      "loss": 1.3438,
+      "theoretical_loss": 3.435081200963654,
+      "tokens_seen": 2175467520
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034416626544695876,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4350643700155112,
+      "tokens_seen": 2175598592
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034412614347616757,
+      "loss": 1.247,
+      "theoretical_loss": 3.4350475403652436,
+      "tokens_seen": 2175729664
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003440860215053764,
+      "loss": 1.229,
+      "theoretical_loss": 3.435030712012673,
+      "tokens_seen": 2175860736
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034404589953458513,
+      "loss": 1.3057,
+      "theoretical_loss": 3.4350138849576215,
+      "tokens_seen": 2175991808
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034400577756379393,
+      "loss": 1.2405,
+      "theoretical_loss": 3.4349970591999104,
+      "tokens_seen": 2176122880
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034396565559300274,
+      "loss": 1.2893,
+      "theoretical_loss": 3.4349802347393616,
+      "tokens_seen": 2176253952
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003439255336222115,
+      "loss": 1.2248,
+      "theoretical_loss": 3.4349634115757977,
+      "tokens_seen": 2176385024
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034388541165142035,
+      "loss": 1.3162,
+      "theoretical_loss": 3.4349465897090394,
+      "tokens_seen": 2176516096
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003438452896806291,
+      "loss": 1.2994,
+      "theoretical_loss": 3.43492976913891,
+      "tokens_seen": 2176647168
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034380516770983797,
+      "loss": 1.2801,
+      "theoretical_loss": 3.4349129498652298,
+      "tokens_seen": 2176778240
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003437650457390467,
+      "loss": 1.2067,
+      "theoretical_loss": 3.4348961318878226,
+      "tokens_seen": 2176909312
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034372492376825547,
+      "loss": 1.2852,
+      "theoretical_loss": 3.4348793152065094,
+      "tokens_seen": 2177040384
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034368480179746433,
+      "loss": 1.2336,
+      "theoretical_loss": 3.4348624998211124,
+      "tokens_seen": 2177171456
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003436446798266731,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4348456857314544,
+      "tokens_seen": 2177302528
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003436045578558819,
+      "loss": 1.324,
+      "theoretical_loss": 3.4348288729373566,
+      "tokens_seen": 2177433600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003435644358850907,
+      "loss": 1.2283,
+      "theoretical_loss": 3.4348120614386417,
+      "tokens_seen": 2177564672
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003435243139142995,
+      "loss": 1.2915,
+      "theoretical_loss": 3.434795251235132,
+      "tokens_seen": 2177695744
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034348419194350825,
+      "loss": 1.2552,
+      "theoretical_loss": 3.4347784423266496,
+      "tokens_seen": 2177826816
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034344406997271706,
+      "loss": 1.2935,
+      "theoretical_loss": 3.4347616347130168,
+      "tokens_seen": 2177957888
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034340394800192587,
+      "loss": 1.2417,
+      "theoretical_loss": 3.434744828394056,
+      "tokens_seen": 2178088960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003433638260311346,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4347280233695896,
+      "tokens_seen": 2178220032
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003433237040603435,
+      "loss": 1.3331,
+      "theoretical_loss": 3.4347112196394396,
+      "tokens_seen": 2178351104
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034328358208955223,
+      "loss": 1.286,
+      "theoretical_loss": 3.434694417203429,
+      "tokens_seen": 2178482176
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034324346011876104,
+      "loss": 1.16,
+      "theoretical_loss": 3.4346776160613803,
+      "tokens_seen": 2178613248
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034320333814796984,
+      "loss": 1.3073,
+      "theoretical_loss": 3.434660816213116,
+      "tokens_seen": 2178744320
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003431632161771786,
+      "loss": 1.2699,
+      "theoretical_loss": 3.4346440176584574,
+      "tokens_seen": 2178875392
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034312309420638746,
+      "loss": 1.1886,
+      "theoretical_loss": 3.434627220397229,
+      "tokens_seen": 2179006464
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003430829722355962,
+      "loss": 1.323,
+      "theoretical_loss": 3.4346104244292524,
+      "tokens_seen": 2179137536
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000343042850264805,
+      "loss": 1.2475,
+      "theoretical_loss": 3.43459362975435,
+      "tokens_seen": 2179268608
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003430027282940138,
+      "loss": 1.2352,
+      "theoretical_loss": 3.4345768363723455,
+      "tokens_seen": 2179399680
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034296260632322263,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4345600442830606,
+      "tokens_seen": 2179530752
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003429224843524314,
+      "loss": 1.3206,
+      "theoretical_loss": 3.4345432534863183,
+      "tokens_seen": 2179661824
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003428823623816402,
+      "loss": 1.2818,
+      "theoretical_loss": 3.4345264639819417,
+      "tokens_seen": 2179792896
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000342842240410849,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4345096757697533,
+      "tokens_seen": 2179923968
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034280211844005774,
+      "loss": 1.1995,
+      "theoretical_loss": 3.4344928888495767,
+      "tokens_seen": 2180055040
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003427619964692666,
+      "loss": 1.2668,
+      "theoretical_loss": 3.4344761032212334,
+      "tokens_seen": 2180186112
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034272187449847536,
+      "loss": 1.1949,
+      "theoretical_loss": 3.4344593188845476,
+      "tokens_seen": 2180317184
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034268175252768416,
+      "loss": 1.246,
+      "theoretical_loss": 3.434442535839342,
+      "tokens_seen": 2180448256
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034264163055689297,
+      "loss": 1.3463,
+      "theoretical_loss": 3.4344257540854395,
+      "tokens_seen": 2180579328
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003426015085861017,
+      "loss": 1.2654,
+      "theoretical_loss": 3.4344089736226624,
+      "tokens_seen": 2180710400
+    },
+    {
+      "epoch": 0.32,
+      "objective/train/docs_used": 733364,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2431890964508057,
+      "objective/train/theoretical_loss": 3.434392194450835,
+      "objective/train/tokens_used": 551300576,
+      "theoretical_loss": 3.434392194450835,
+      "tokens_seen": 2180841472
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034256138661531053,
+      "loss": 1.2928,
+      "theoretical_loss": 3.434392194450835,
+      "tokens_seen": 2180841472
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034252126464451933,
+      "loss": 1.2669,
+      "theoretical_loss": 3.4343754165697797,
+      "tokens_seen": 2180972544
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034248114267372814,
+      "loss": 1.2365,
+      "theoretical_loss": 3.43435863997932,
+      "tokens_seen": 2181103616
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034244102070293695,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4343418646792783,
+      "tokens_seen": 2181234688
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034240089873214575,
+      "loss": 1.3107,
+      "theoretical_loss": 3.4343250906694784,
+      "tokens_seen": 2181365760
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003423607767613545,
+      "loss": 1.249,
+      "theoretical_loss": 3.434308317949744,
+      "tokens_seen": 2181496832
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034232065479056337,
+      "loss": 1.3147,
+      "theoretical_loss": 3.4342915465198978,
+      "tokens_seen": 2181627904
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003422805328197721,
+      "loss": 1.3485,
+      "theoretical_loss": 3.434274776379763,
+      "tokens_seen": 2181758976
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034224041084898087,
+      "loss": 1.2119,
+      "theoretical_loss": 3.434258007529163,
+      "tokens_seen": 2181890048
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034220028887818973,
+      "loss": 1.2904,
+      "theoretical_loss": 3.4342412399679216,
+      "tokens_seen": 2182021120
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003421601669073985,
+      "loss": 1.2219,
+      "theoretical_loss": 3.434224473695862,
+      "tokens_seen": 2182152192
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003421200449366073,
+      "loss": 1.2096,
+      "theoretical_loss": 3.4342077087128073,
+      "tokens_seen": 2182283264
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003420799229658161,
+      "loss": 1.3216,
+      "theoretical_loss": 3.4341909450185817,
+      "tokens_seen": 2182414336
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003420398009950249,
+      "loss": 1.2897,
+      "theoretical_loss": 3.434174182613008,
+      "tokens_seen": 2182545408
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034199967902423365,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4341574214959105,
+      "tokens_seen": 2182676480
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034195955705344246,
+      "loss": 1.3154,
+      "theoretical_loss": 3.434140661667112,
+      "tokens_seen": 2182807552
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034191943508265127,
+      "loss": 1.2526,
+      "theoretical_loss": 3.434123903126437,
+      "tokens_seen": 2182938624
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034187931311186007,
+      "loss": 1.295,
+      "theoretical_loss": 3.434107145873708,
+      "tokens_seen": 2183069696
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003418391911410689,
+      "loss": 1.2993,
+      "theoretical_loss": 3.4340903899087496,
+      "tokens_seen": 2183200768
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034179906917027763,
+      "loss": 1.2712,
+      "theoretical_loss": 3.4340736352313854,
+      "tokens_seen": 2183331840
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003417589471994865,
+      "loss": 1.2767,
+      "theoretical_loss": 3.434056881841439,
+      "tokens_seen": 2183462912
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034171882522869524,
+      "loss": 1.2886,
+      "theoretical_loss": 3.4340401297387344,
+      "tokens_seen": 2183593984
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000341678703257904,
+      "loss": 1.2523,
+      "theoretical_loss": 3.434023378923095,
+      "tokens_seen": 2183725056
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034163858128711286,
+      "loss": 1.261,
+      "theoretical_loss": 3.434006629394345,
+      "tokens_seen": 2183856128
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003415984593163216,
+      "loss": 1.3406,
+      "theoretical_loss": 3.433989881152309,
+      "tokens_seen": 2183987200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003415583373455304,
+      "loss": 1.2502,
+      "theoretical_loss": 3.433973134196809,
+      "tokens_seen": 2184118272
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003415182153747392,
+      "loss": 1.2188,
+      "theoretical_loss": 3.433956388527671,
+      "tokens_seen": 2184249344
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034147809340394803,
+      "loss": 1.2696,
+      "theoretical_loss": 3.433939644144718,
+      "tokens_seen": 2184380416
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003414379714331568,
+      "loss": 1.3149,
+      "theoretical_loss": 3.4339229010477745,
+      "tokens_seen": 2184511488
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003413978494623656,
+      "loss": 1.2477,
+      "theoretical_loss": 3.433906159236664,
+      "tokens_seen": 2184642560
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003413577274915744,
+      "loss": 1.2957,
+      "theoretical_loss": 3.4338894187112112,
+      "tokens_seen": 2184773632
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034131760552078314,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4338726794712398,
+      "tokens_seen": 2184904704
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000341277483549992,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4338559415165744,
+      "tokens_seen": 2185035776
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034123736157920076,
+      "loss": 1.258,
+      "theoretical_loss": 3.4338392048470388,
+      "tokens_seen": 2185166848
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003411972396084096,
+      "loss": 1.3045,
+      "theoretical_loss": 3.4338224694624575,
+      "tokens_seen": 2185297920
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034115711763761837,
+      "loss": 1.2517,
+      "theoretical_loss": 3.4338057353626548,
+      "tokens_seen": 2185428992
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003411169956668271,
+      "loss": 1.2314,
+      "theoretical_loss": 3.4337890025474547,
+      "tokens_seen": 2185560064
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000341076873696036,
+      "loss": 1.2228,
+      "theoretical_loss": 3.433772271016682,
+      "tokens_seen": 2185691136
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034103675172524473,
+      "loss": 1.2893,
+      "theoretical_loss": 3.4337555407701608,
+      "tokens_seen": 2185822208
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034099662975445354,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4337388118077157,
+      "tokens_seen": 2185953280
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034095650778366235,
+      "loss": 1.2402,
+      "theoretical_loss": 3.433722084129171,
+      "tokens_seen": 2186084352
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034091638581287115,
+      "loss": 1.2564,
+      "theoretical_loss": 3.4337053577343513,
+      "tokens_seen": 2186215424
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003408762638420799,
+      "loss": 1.2939,
+      "theoretical_loss": 3.433688632623081,
+      "tokens_seen": 2186346496
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034083614187128877,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4336719087951852,
+      "tokens_seen": 2186477568
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003407960199004975,
+      "loss": 1.2252,
+      "theoretical_loss": 3.433655186250488,
+      "tokens_seen": 2186608640
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034075589792970627,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4336384649888134,
+      "tokens_seen": 2186739712
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034071577595891513,
+      "loss": 1.2214,
+      "theoretical_loss": 3.4336217450099875,
+      "tokens_seen": 2186870784
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003406756539881239,
+      "loss": 1.2232,
+      "theoretical_loss": 3.433605026313834,
+      "tokens_seen": 2187001856
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034063553201733274,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4335883089001777,
+      "tokens_seen": 2187132928
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003405954100465415,
+      "loss": 1.2488,
+      "theoretical_loss": 3.4335715927688435,
+      "tokens_seen": 2187264000
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 735627,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2050080299377441,
+      "objective/train/theoretical_loss": 3.4335548779196565,
+      "objective/train/tokens_used": 557854176,
+      "theoretical_loss": 3.4335548779196565,
+      "tokens_seen": 2187395072
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003405552880757503,
+      "loss": 1.224,
+      "theoretical_loss": 3.4335548779196565,
+      "tokens_seen": 2187395072
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003405151661049591,
+      "loss": 1.1952,
+      "theoretical_loss": 3.4335381643524414,
+      "tokens_seen": 2187526144
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034047504413416786,
+      "loss": 1.27,
+      "theoretical_loss": 3.4335214520670228,
+      "tokens_seen": 2187657216
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034043492216337667,
+      "loss": 1.248,
+      "theoretical_loss": 3.433504741063226,
+      "tokens_seen": 2187788288
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034039480019258547,
+      "loss": 1.3022,
+      "theoretical_loss": 3.4334880313408753,
+      "tokens_seen": 2187919360
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003403546782217943,
+      "loss": 1.3015,
+      "theoretical_loss": 3.4334713228997966,
+      "tokens_seen": 2188050432
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034031455625100303,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4334546157398145,
+      "tokens_seen": 2188181504
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003402744342802119,
+      "loss": 1.1834,
+      "theoretical_loss": 3.4334379098607535,
+      "tokens_seen": 2188312576
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034023431230942064,
+      "loss": 1.3267,
+      "theoretical_loss": 3.433421205262439,
+      "tokens_seen": 2188443648
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003401941903386294,
+      "loss": 1.2233,
+      "theoretical_loss": 3.433404501944697,
+      "tokens_seen": 2188574720
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034015406836783826,
+      "loss": 1.2269,
+      "theoretical_loss": 3.433387799907351,
+      "tokens_seen": 2188705792
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000340113946397047,
+      "loss": 1.1717,
+      "theoretical_loss": 3.4333710991502278,
+      "tokens_seen": 2188836864
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003400738244262558,
+      "loss": 1.2367,
+      "theoretical_loss": 3.433354399673152,
+      "tokens_seen": 2188967936
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003400337024554646,
+      "loss": 1.244,
+      "theoretical_loss": 3.4333377014759483,
+      "tokens_seen": 2189099008
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003399935804846734,
+      "loss": 1.154,
+      "theoretical_loss": 3.433321004558443,
+      "tokens_seen": 2189230080
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033995345851388223,
+      "loss": 1.1756,
+      "theoretical_loss": 3.43330430892046,
+      "tokens_seen": 2189361152
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000339913336543091,
+      "loss": 1.2843,
+      "theoretical_loss": 3.433287614561826,
+      "tokens_seen": 2189492224
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003398732145722998,
+      "loss": 1.292,
+      "theoretical_loss": 3.4332709214823653,
+      "tokens_seen": 2189623296
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003398330926015086,
+      "loss": 1.2891,
+      "theoretical_loss": 3.433254229681905,
+      "tokens_seen": 2189754368
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003397929706307174,
+      "loss": 1.2978,
+      "theoretical_loss": 3.4332375391602685,
+      "tokens_seen": 2189885440
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033975284865992616,
+      "loss": 1.3255,
+      "theoretical_loss": 3.4332208499172827,
+      "tokens_seen": 2190016512
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000339712726689135,
+      "loss": 1.2932,
+      "theoretical_loss": 3.433204161952773,
+      "tokens_seen": 2190147584
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033967260471834377,
+      "loss": 1.3197,
+      "theoretical_loss": 3.433187475266564,
+      "tokens_seen": 2190278656
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003396324827475525,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4331707898584822,
+      "tokens_seen": 2190409728
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003395923607767614,
+      "loss": 1.2029,
+      "theoretical_loss": 3.4331541057283532,
+      "tokens_seen": 2190540800
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033955223880597013,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4331374228760025,
+      "tokens_seen": 2190671872
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033951211683517894,
+      "loss": 1.2752,
+      "theoretical_loss": 3.433120741301255,
+      "tokens_seen": 2190802944
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033947199486438775,
+      "loss": 1.3607,
+      "theoretical_loss": 3.433104061003938,
+      "tokens_seen": 2190934016
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033943187289359655,
+      "loss": 1.3273,
+      "theoretical_loss": 3.433087381983876,
+      "tokens_seen": 2191065088
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003393917509228053,
+      "loss": 1.1846,
+      "theoretical_loss": 3.4330707042408957,
+      "tokens_seen": 2191196160
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033935162895201417,
+      "loss": 1.2413,
+      "theoretical_loss": 3.433054027774822,
+      "tokens_seen": 2191327232
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003393115069812229,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4330373525854814,
+      "tokens_seen": 2191458304
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003392713850104317,
+      "loss": 1.3037,
+      "theoretical_loss": 3.4330206786726993,
+      "tokens_seen": 2191589376
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033923126303964053,
+      "loss": 1.3202,
+      "theoretical_loss": 3.4330040060363025,
+      "tokens_seen": 2191720448
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003391911410688493,
+      "loss": 1.2275,
+      "theoretical_loss": 3.432987334676116,
+      "tokens_seen": 2191851520
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033915101909805814,
+      "loss": 1.3004,
+      "theoretical_loss": 3.4329706645919664,
+      "tokens_seen": 2191982592
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003391108971272669,
+      "loss": 1.308,
+      "theoretical_loss": 3.43295399578368,
+      "tokens_seen": 2192113664
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003390707751564757,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4329373282510818,
+      "tokens_seen": 2192244736
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003390306531856845,
+      "loss": 1.2403,
+      "theoretical_loss": 3.432920661993999,
+      "tokens_seen": 2192375808
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033899053121489326,
+      "loss": 1.2609,
+      "theoretical_loss": 3.432903997012257,
+      "tokens_seen": 2192506880
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033895040924410207,
+      "loss": 1.2204,
+      "theoretical_loss": 3.432887333305682,
+      "tokens_seen": 2192637952
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033891028727331087,
+      "loss": 1.28,
+      "theoretical_loss": 3.432870670874101,
+      "tokens_seen": 2192769024
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003388701653025197,
+      "loss": 1.298,
+      "theoretical_loss": 3.4328540097173397,
+      "tokens_seen": 2192900096
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033883004333172843,
+      "loss": 1.2296,
+      "theoretical_loss": 3.432837349835224,
+      "tokens_seen": 2193031168
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003387899213609373,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4328206912275814,
+      "tokens_seen": 2193162240
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033874979939014604,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4328040338942367,
+      "tokens_seen": 2193293312
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033870967741935485,
+      "loss": 1.2856,
+      "theoretical_loss": 3.4327873778350173,
+      "tokens_seen": 2193424384
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033866955544856366,
+      "loss": 1.2462,
+      "theoretical_loss": 3.432770723049749,
+      "tokens_seen": 2193555456
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003386294334777724,
+      "loss": 1.2821,
+      "theoretical_loss": 3.4327540695382592,
+      "tokens_seen": 2193686528
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033858931150698127,
+      "loss": 1.2789,
+      "theoretical_loss": 3.4327374173003733,
+      "tokens_seen": 2193817600
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 737824,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2934893369674683,
+      "objective/train/theoretical_loss": 3.4327207663359185,
+      "objective/train/tokens_used": 564407776,
+      "theoretical_loss": 3.4327207663359185,
+      "tokens_seen": 2193948672
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033854918953619,
+      "loss": 1.2133,
+      "theoretical_loss": 3.4327207663359185,
+      "tokens_seen": 2193948672
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003385090675653988,
+      "loss": 1.2092,
+      "theoretical_loss": 3.432704116644721,
+      "tokens_seen": 2194079744
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033846894559460763,
+      "loss": 1.2012,
+      "theoretical_loss": 3.4326874682266078,
+      "tokens_seen": 2194210816
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003384288236238164,
+      "loss": 1.2658,
+      "theoretical_loss": 3.432670821081405,
+      "tokens_seen": 2194341888
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003383887016530252,
+      "loss": 1.2194,
+      "theoretical_loss": 3.4326541752089397,
+      "tokens_seen": 2194472960
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000338348579682234,
+      "loss": 1.2078,
+      "theoretical_loss": 3.432637530609038,
+      "tokens_seen": 2194604032
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003383084577114428,
+      "loss": 1.2808,
+      "theoretical_loss": 3.4326208872815274,
+      "tokens_seen": 2194735104
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033826833574065156,
+      "loss": 1.2379,
+      "theoretical_loss": 3.432604245226234,
+      "tokens_seen": 2194866176
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003382282137698604,
+      "loss": 1.2962,
+      "theoretical_loss": 3.432587604442985,
+      "tokens_seen": 2194997248
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033818809179906917,
+      "loss": 1.3036,
+      "theoretical_loss": 3.4325709649316067,
+      "tokens_seen": 2195128320
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003381479698282779,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4325543266919265,
+      "tokens_seen": 2195259392
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003381078478574868,
+      "loss": 1.248,
+      "theoretical_loss": 3.4325376897237714,
+      "tokens_seen": 2195390464
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033806772588669553,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4325210540269677,
+      "tokens_seen": 2195521536
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003380276039159044,
+      "loss": 1.2622,
+      "theoretical_loss": 3.432504419601343,
+      "tokens_seen": 2195652608
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033798748194511315,
+      "loss": 1.3169,
+      "theoretical_loss": 3.4324877864467234,
+      "tokens_seen": 2195783680
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033794735997432195,
+      "loss": 1.2541,
+      "theoretical_loss": 3.4324711545629367,
+      "tokens_seen": 2195914752
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033790723800353076,
+      "loss": 1.2793,
+      "theoretical_loss": 3.43245452394981,
+      "tokens_seen": 2196045824
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003378671160327395,
+      "loss": 1.2162,
+      "theoretical_loss": 3.43243789460717,
+      "tokens_seen": 2196176896
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003378269940619483,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4324212665348437,
+      "tokens_seen": 2196307968
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003377868720911571,
+      "loss": 1.2584,
+      "theoretical_loss": 3.432404639732659,
+      "tokens_seen": 2196439040
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033774675012036593,
+      "loss": 1.2872,
+      "theoretical_loss": 3.432388014200442,
+      "tokens_seen": 2196570112
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003377066281495747,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4323713899380213,
+      "tokens_seen": 2196701184
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033766650617878354,
+      "loss": 1.2395,
+      "theoretical_loss": 3.432354766945223,
+      "tokens_seen": 2196832256
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003376263842079923,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4323381452218746,
+      "tokens_seen": 2196963328
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003375862622372011,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4323215247678043,
+      "tokens_seen": 2197094400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003375461402664099,
+      "loss": 1.1947,
+      "theoretical_loss": 3.4323049055828383,
+      "tokens_seen": 2197225472
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033750601829561866,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4322882876668044,
+      "tokens_seen": 2197356544
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003374658963248275,
+      "loss": 1.319,
+      "theoretical_loss": 3.4322716710195302,
+      "tokens_seen": 2197487616
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033742577435403627,
+      "loss": 1.2975,
+      "theoretical_loss": 3.432255055640843,
+      "tokens_seen": 2197618688
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003373856523832451,
+      "loss": 1.2602,
+      "theoretical_loss": 3.4322384415305707,
+      "tokens_seen": 2197749760
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003373455304124539,
+      "loss": 1.2546,
+      "theoretical_loss": 3.43222182868854,
+      "tokens_seen": 2197880832
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003373054084416627,
+      "loss": 1.3298,
+      "theoretical_loss": 3.4322052171145794,
+      "tokens_seen": 2198011904
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033726528647087144,
+      "loss": 1.2113,
+      "theoretical_loss": 3.432188606808516,
+      "tokens_seen": 2198142976
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033722516450008025,
+      "loss": 1.2229,
+      "theoretical_loss": 3.432171997770177,
+      "tokens_seen": 2198274048
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033718504252928906,
+      "loss": 1.2532,
+      "theoretical_loss": 3.4321553899993913,
+      "tokens_seen": 2198405120
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003371449205584978,
+      "loss": 1.3119,
+      "theoretical_loss": 3.4321387834959856,
+      "tokens_seen": 2198536192
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033710479858770667,
+      "loss": 1.341,
+      "theoretical_loss": 3.432122178259788,
+      "tokens_seen": 2198667264
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003370646766169154,
+      "loss": 1.2026,
+      "theoretical_loss": 3.4321055742906257,
+      "tokens_seen": 2198798336
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003370245546461242,
+      "loss": 1.248,
+      "theoretical_loss": 3.4320889715883274,
+      "tokens_seen": 2198929408
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033698443267533303,
+      "loss": 1.2272,
+      "theoretical_loss": 3.4320723701527203,
+      "tokens_seen": 2199060480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003369443107045418,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4320557699836325,
+      "tokens_seen": 2199191552
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003369041887337506,
+      "loss": 1.2877,
+      "theoretical_loss": 3.4320391710808917,
+      "tokens_seen": 2199322624
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003368640667629594,
+      "loss": 1.2874,
+      "theoretical_loss": 3.432022573444326,
+      "tokens_seen": 2199453696
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003368239447921682,
+      "loss": 1.2333,
+      "theoretical_loss": 3.432005977073764,
+      "tokens_seen": 2199584768
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000336783822821377,
+      "loss": 1.2823,
+      "theoretical_loss": 3.4319893819690326,
+      "tokens_seen": 2199715840
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003367437008505858,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4319727881299604,
+      "tokens_seen": 2199846912
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033670357887979457,
+      "loss": 1.3115,
+      "theoretical_loss": 3.431956195556375,
+      "tokens_seen": 2199977984
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003366634569090034,
+      "loss": 1.2818,
+      "theoretical_loss": 3.4319396042481056,
+      "tokens_seen": 2200109056
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003366233349382122,
+      "loss": 1.236,
+      "theoretical_loss": 3.4319230142049792,
+      "tokens_seen": 2200240128
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033658321296742093,
+      "loss": 1.2976,
+      "theoretical_loss": 3.4319064254268246,
+      "tokens_seen": 2200371200
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 740003,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1852833032608032,
+      "objective/train/theoretical_loss": 3.4318898379134697,
+      "objective/train/tokens_used": 570961376,
+      "theoretical_loss": 3.4318898379134697,
+      "tokens_seen": 2200502272
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003365430909966298,
+      "loss": 1.2229,
+      "theoretical_loss": 3.4318898379134697,
+      "tokens_seen": 2200502272
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033650296902583855,
+      "loss": 1.2376,
+      "theoretical_loss": 3.4318732516647428,
+      "tokens_seen": 2200633344
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033646284705504735,
+      "loss": 1.3616,
+      "theoretical_loss": 3.431856666680472,
+      "tokens_seen": 2200764416
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033642272508425616,
+      "loss": 1.2628,
+      "theoretical_loss": 3.431840082960486,
+      "tokens_seen": 2200895488
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003363826031134649,
+      "loss": 1.1774,
+      "theoretical_loss": 3.431823500504613,
+      "tokens_seen": 2201026560
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003363424811426737,
+      "loss": 1.3272,
+      "theoretical_loss": 3.431806919312681,
+      "tokens_seen": 2201157632
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003363023591718825,
+      "loss": 1.2136,
+      "theoretical_loss": 3.431790339384519,
+      "tokens_seen": 2201288704
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033626223720109133,
+      "loss": 1.245,
+      "theoretical_loss": 3.431773760719955,
+      "tokens_seen": 2201419776
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003362221152303001,
+      "loss": 1.2504,
+      "theoretical_loss": 3.431757183318818,
+      "tokens_seen": 2201550848
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033618199325950894,
+      "loss": 1.3018,
+      "theoretical_loss": 3.431740607180936,
+      "tokens_seen": 2201681920
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003361418712887177,
+      "loss": 1.2221,
+      "theoretical_loss": 3.4317240323061378,
+      "tokens_seen": 2201812992
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033610174931792655,
+      "loss": 1.301,
+      "theoretical_loss": 3.4317074586942518,
+      "tokens_seen": 2201944064
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003360616273471353,
+      "loss": 1.2351,
+      "theoretical_loss": 3.4316908863451063,
+      "tokens_seen": 2202075136
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033602150537634406,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4316743152585305,
+      "tokens_seen": 2202206208
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003359813834055529,
+      "loss": 1.2207,
+      "theoretical_loss": 3.431657745434353,
+      "tokens_seen": 2202337280
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033594126143476167,
+      "loss": 1.234,
+      "theoretical_loss": 3.431641176872402,
+      "tokens_seen": 2202468352
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003359011394639705,
+      "loss": 1.2592,
+      "theoretical_loss": 3.431624609572507,
+      "tokens_seen": 2202599424
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003358610174931793,
+      "loss": 1.3115,
+      "theoretical_loss": 3.431608043534496,
+      "tokens_seen": 2202730496
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003358208955223881,
+      "loss": 1.2683,
+      "theoretical_loss": 3.4315914787581985,
+      "tokens_seen": 2202861568
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033578077355159684,
+      "loss": 1.2405,
+      "theoretical_loss": 3.431574915243443,
+      "tokens_seen": 2202992640
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033574065158080565,
+      "loss": 1.2937,
+      "theoretical_loss": 3.431558352990059,
+      "tokens_seen": 2203123712
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033570052961001445,
+      "loss": 1.3401,
+      "theoretical_loss": 3.431541791997874,
+      "tokens_seen": 2203254784
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003356604076392232,
+      "loss": 1.187,
+      "theoretical_loss": 3.431525232266718,
+      "tokens_seen": 2203385856
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033562028566843207,
+      "loss": 1.2694,
+      "theoretical_loss": 3.4315086737964196,
+      "tokens_seen": 2203516928
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003355801636976408,
+      "loss": 1.2307,
+      "theoretical_loss": 3.431492116586808,
+      "tokens_seen": 2203648000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003355400417268497,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4314755606377125,
+      "tokens_seen": 2203779072
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033549991975605843,
+      "loss": 1.2993,
+      "theoretical_loss": 3.4314590059489616,
+      "tokens_seen": 2203910144
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003354597977852672,
+      "loss": 1.2679,
+      "theoretical_loss": 3.431442452520385,
+      "tokens_seen": 2204041216
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033541967581447604,
+      "loss": 1.2647,
+      "theoretical_loss": 3.431425900351811,
+      "tokens_seen": 2204172288
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003353795538436848,
+      "loss": 1.3296,
+      "theoretical_loss": 3.4314093494430695,
+      "tokens_seen": 2204303360
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003353394318728936,
+      "loss": 1.2822,
+      "theoretical_loss": 3.4313927997939895,
+      "tokens_seen": 2204434432
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003352993099021024,
+      "loss": 1.2888,
+      "theoretical_loss": 3.4313762514044,
+      "tokens_seen": 2204565504
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003352591879313112,
+      "loss": 1.3583,
+      "theoretical_loss": 3.4313597042741306,
+      "tokens_seen": 2204696576
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033521906596051997,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4313431584030107,
+      "tokens_seen": 2204827648
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003351789439897288,
+      "loss": 1.3173,
+      "theoretical_loss": 3.431326613790869,
+      "tokens_seen": 2204958720
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003351388220189376,
+      "loss": 1.2439,
+      "theoretical_loss": 3.4313100704375357,
+      "tokens_seen": 2205089792
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033509870004814633,
+      "loss": 1.308,
+      "theoretical_loss": 3.43129352834284,
+      "tokens_seen": 2205220864
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003350585780773552,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4312769875066107,
+      "tokens_seen": 2205351936
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033501845610656395,
+      "loss": 1.2397,
+      "theoretical_loss": 3.431260447928678,
+      "tokens_seen": 2205483008
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033497833413577275,
+      "loss": 1.271,
+      "theoretical_loss": 3.431243909608871,
+      "tokens_seen": 2205614080
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033493821216498156,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4312273725470197,
+      "tokens_seen": 2205745152
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003348980901941903,
+      "loss": 1.276,
+      "theoretical_loss": 3.4312108367429532,
+      "tokens_seen": 2205876224
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033485796822339917,
+      "loss": 1.2949,
+      "theoretical_loss": 3.4311943021965012,
+      "tokens_seen": 2206007296
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003348178462526079,
+      "loss": 1.24,
+      "theoretical_loss": 3.4311777689074936,
+      "tokens_seen": 2206138368
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033477772428181673,
+      "loss": 1.3202,
+      "theoretical_loss": 3.43116123687576,
+      "tokens_seen": 2206269440
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033473760231102554,
+      "loss": 1.2459,
+      "theoretical_loss": 3.4311447061011293,
+      "tokens_seen": 2206400512
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033469748034023434,
+      "loss": 1.2577,
+      "theoretical_loss": 3.4311281765834325,
+      "tokens_seen": 2206531584
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003346573583694431,
+      "loss": 1.2947,
+      "theoretical_loss": 3.431111648322499,
+      "tokens_seen": 2206662656
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033461723639865195,
+      "loss": 1.2037,
+      "theoretical_loss": 3.4310951213181577,
+      "tokens_seen": 2206793728
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003345771144278607,
+      "loss": 1.1921,
+      "theoretical_loss": 3.4310785955702396,
+      "tokens_seen": 2206924800
+    },
+    {
+      "epoch": 0.34,
+      "objective/train/docs_used": 742158,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.292722463607788,
+      "objective/train/theoretical_loss": 3.431062071078574,
+      "objective/train/tokens_used": 577514976,
+      "theoretical_loss": 3.431062071078574,
+      "tokens_seen": 2207055872
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033453699245706946,
+      "loss": 1.2984,
+      "theoretical_loss": 3.431062071078574,
+      "tokens_seen": 2207055872
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003344968704862783,
+      "loss": 1.2485,
+      "theoretical_loss": 3.4310455478429907,
+      "tokens_seen": 2207186944
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033445674851548707,
+      "loss": 1.284,
+      "theoretical_loss": 3.43102902586332,
+      "tokens_seen": 2207318016
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003344166265446959,
+      "loss": 1.2095,
+      "theoretical_loss": 3.4310125051393916,
+      "tokens_seen": 2207449088
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003343765045739047,
+      "loss": 1.2956,
+      "theoretical_loss": 3.430995985671036,
+      "tokens_seen": 2207580160
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003343363826031135,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4309794674580827,
+      "tokens_seen": 2207711232
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033429626063232224,
+      "loss": 1.3173,
+      "theoretical_loss": 3.430962950500362,
+      "tokens_seen": 2207842304
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033425613866153105,
+      "loss": 1.2809,
+      "theoretical_loss": 3.4309464347977037,
+      "tokens_seen": 2207973376
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033421601669073985,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4309299203499384,
+      "tokens_seen": 2208104448
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033417589471994866,
+      "loss": 1.2918,
+      "theoretical_loss": 3.430913407156896,
+      "tokens_seen": 2208235520
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033413577274915747,
+      "loss": 1.2224,
+      "theoretical_loss": 3.4308968952184067,
+      "tokens_seen": 2208366592
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003340956507783662,
+      "loss": 1.2457,
+      "theoretical_loss": 3.430880384534301,
+      "tokens_seen": 2208497664
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003340555288075751,
+      "loss": 1.2771,
+      "theoretical_loss": 3.430863875104409,
+      "tokens_seen": 2208628736
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033401540683678383,
+      "loss": 1.2186,
+      "theoretical_loss": 3.4308473669285604,
+      "tokens_seen": 2208759808
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003339752848659926,
+      "loss": 1.2688,
+      "theoretical_loss": 3.430830860006586,
+      "tokens_seen": 2208890880
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033393516289520144,
+      "loss": 1.2647,
+      "theoretical_loss": 3.430814354338317,
+      "tokens_seen": 2209021952
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003338950409244102,
+      "loss": 1.3687,
+      "theoretical_loss": 3.4307978499235827,
+      "tokens_seen": 2209153024
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000333854918953619,
+      "loss": 1.2244,
+      "theoretical_loss": 3.430781346762214,
+      "tokens_seen": 2209284096
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003338147969828278,
+      "loss": 1.3301,
+      "theoretical_loss": 3.4307648448540413,
+      "tokens_seen": 2209415168
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003337746750120366,
+      "loss": 1.3161,
+      "theoretical_loss": 3.4307483441988946,
+      "tokens_seen": 2209546240
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033373455304124537,
+      "loss": 1.236,
+      "theoretical_loss": 3.4307318447966058,
+      "tokens_seen": 2209677312
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003336944310704542,
+      "loss": 1.2923,
+      "theoretical_loss": 3.4307153466470037,
+      "tokens_seen": 2209808384
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000333654309099663,
+      "loss": 1.2672,
+      "theoretical_loss": 3.4306988497499202,
+      "tokens_seen": 2209939456
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003336141871288718,
+      "loss": 1.3431,
+      "theoretical_loss": 3.430682354105185,
+      "tokens_seen": 2210070528
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003335740651580806,
+      "loss": 1.2382,
+      "theoretical_loss": 3.43066585971263,
+      "tokens_seen": 2210201600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033353394318728934,
+      "loss": 1.1575,
+      "theoretical_loss": 3.4306493665720845,
+      "tokens_seen": 2210332672
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003334938212164982,
+      "loss": 1.1748,
+      "theoretical_loss": 3.43063287468338,
+      "tokens_seen": 2210463744
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033345369924570696,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4306163840463473,
+      "tokens_seen": 2210594816
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003334135772749157,
+      "loss": 1.3101,
+      "theoretical_loss": 3.4305998946608174,
+      "tokens_seen": 2210725888
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033337345530412457,
+      "loss": 1.2473,
+      "theoretical_loss": 3.4305834065266203,
+      "tokens_seen": 2210856960
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003333333333333333,
+      "loss": 1.3129,
+      "theoretical_loss": 3.4305669196435873,
+      "tokens_seen": 2210988032
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033329321136254213,
+      "loss": 1.1899,
+      "theoretical_loss": 3.4305504340115496,
+      "tokens_seen": 2211119104
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033325308939175093,
+      "loss": 1.2572,
+      "theoretical_loss": 3.4305339496303375,
+      "tokens_seen": 2211250176
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033321296742095974,
+      "loss": 1.2195,
+      "theoretical_loss": 3.430517466499783,
+      "tokens_seen": 2211381248
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003331728454501685,
+      "loss": 1.1716,
+      "theoretical_loss": 3.430500984619716,
+      "tokens_seen": 2211512320
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003331327234793773,
+      "loss": 1.2598,
+      "theoretical_loss": 3.430484503989968,
+      "tokens_seen": 2211643392
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003330926015085861,
+      "loss": 1.2726,
+      "theoretical_loss": 3.4304680246103696,
+      "tokens_seen": 2211774464
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033305247953779486,
+      "loss": 1.2122,
+      "theoretical_loss": 3.4304515464807523,
+      "tokens_seen": 2211905536
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003330123575670037,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4304350696009474,
+      "tokens_seen": 2212036608
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033297223559621247,
+      "loss": 1.2789,
+      "theoretical_loss": 3.430418593970786,
+      "tokens_seen": 2212167680
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033293211362542133,
+      "loss": 1.2565,
+      "theoretical_loss": 3.4304021195900996,
+      "tokens_seen": 2212298752
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003328919916546301,
+      "loss": 1.2096,
+      "theoretical_loss": 3.430385646458718,
+      "tokens_seen": 2212429824
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003328518696838389,
+      "loss": 1.3028,
+      "theoretical_loss": 3.430369174576474,
+      "tokens_seen": 2212560896
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003328117477130477,
+      "loss": 1.2517,
+      "theoretical_loss": 3.4303527039431985,
+      "tokens_seen": 2212691968
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033277162574225645,
+      "loss": 1.2644,
+      "theoretical_loss": 3.4303362345587223,
+      "tokens_seen": 2212823040
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033273150377146525,
+      "loss": 1.2317,
+      "theoretical_loss": 3.4303197664228775,
+      "tokens_seen": 2212954112
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033269138180067406,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4303032995354945,
+      "tokens_seen": 2213085184
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033265125982988287,
+      "loss": 1.3288,
+      "theoretical_loss": 3.430286833896406,
+      "tokens_seen": 2213216256
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003326111378590916,
+      "loss": 1.2871,
+      "theoretical_loss": 3.4302703695054424,
+      "tokens_seen": 2213347328
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003325710158883005,
+      "loss": 1.3018,
+      "theoretical_loss": 3.4302539063624358,
+      "tokens_seen": 2213478400
+    },
+    {
+      "epoch": 0.34,
+      "objective/train/docs_used": 744341,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2135040760040283,
+      "objective/train/theoretical_loss": 3.4302374444672172,
+      "objective/train/tokens_used": 584068576,
+      "theoretical_loss": 3.4302374444672172,
+      "tokens_seen": 2213609472
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033253089391750923,
+      "loss": 1.2435,
+      "theoretical_loss": 3.4302374444672172,
+      "tokens_seen": 2213609472
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000332490771946718,
+      "loss": 1.2138,
+      "theoretical_loss": 3.4302209838196185,
+      "tokens_seen": 2213740544
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033245064997592684,
+      "loss": 1.2108,
+      "theoretical_loss": 3.430204524419471,
+      "tokens_seen": 2213871616
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003324105280051356,
+      "loss": 1.2302,
+      "theoretical_loss": 3.4301880662666067,
+      "tokens_seen": 2214002688
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033237040603434446,
+      "loss": 1.2283,
+      "theoretical_loss": 3.4301716093608574,
+      "tokens_seen": 2214133760
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003323302840635532,
+      "loss": 1.2632,
+      "theoretical_loss": 3.430155153702054,
+      "tokens_seen": 2214264832
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000332290162092762,
+      "loss": 1.2756,
+      "theoretical_loss": 3.430138699290029,
+      "tokens_seen": 2214395904
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003322500401219708,
+      "loss": 1.2479,
+      "theoretical_loss": 3.4301222461246135,
+      "tokens_seen": 2214526976
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003322099181511796,
+      "loss": 1.299,
+      "theoretical_loss": 3.4301057942056397,
+      "tokens_seen": 2214658048
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003321697961803884,
+      "loss": 1.2283,
+      "theoretical_loss": 3.4300893435329396,
+      "tokens_seen": 2214789120
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003321296742095972,
+      "loss": 1.2604,
+      "theoretical_loss": 3.4300728941063445,
+      "tokens_seen": 2214920192
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000332089552238806,
+      "loss": 1.2967,
+      "theoretical_loss": 3.430056445925686,
+      "tokens_seen": 2215051264
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033204943026801474,
+      "loss": 1.2197,
+      "theoretical_loss": 3.4300399989907975,
+      "tokens_seen": 2215182336
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003320093082972236,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4300235533015093,
+      "tokens_seen": 2215313408
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033196918632643236,
+      "loss": 1.2614,
+      "theoretical_loss": 3.4300071088576547,
+      "tokens_seen": 2215444480
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003319290643556411,
+      "loss": 1.2627,
+      "theoretical_loss": 3.429990665659065,
+      "tokens_seen": 2215575552
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033188894238484997,
+      "loss": 1.2833,
+      "theoretical_loss": 3.429974223705572,
+      "tokens_seen": 2215706624
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003318488204140587,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4299577829970076,
+      "tokens_seen": 2215837696
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033180869844326753,
+      "loss": 1.2322,
+      "theoretical_loss": 3.429941343533205,
+      "tokens_seen": 2215968768
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033176857647247633,
+      "loss": 1.1806,
+      "theoretical_loss": 3.4299249053139955,
+      "tokens_seen": 2216099840
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033172845450168514,
+      "loss": 1.2761,
+      "theoretical_loss": 3.429908468339212,
+      "tokens_seen": 2216230912
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033168833253089395,
+      "loss": 1.248,
+      "theoretical_loss": 3.4298920326086857,
+      "tokens_seen": 2216361984
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003316482105601027,
+      "loss": 1.2445,
+      "theoretical_loss": 3.4298755981222495,
+      "tokens_seen": 2216493056
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003316080885893115,
+      "loss": 1.2657,
+      "theoretical_loss": 3.429859164879735,
+      "tokens_seen": 2216624128
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003315679666185203,
+      "loss": 1.199,
+      "theoretical_loss": 3.429842732880976,
+      "tokens_seen": 2216755200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003315278446477291,
+      "loss": 1.2873,
+      "theoretical_loss": 3.4298263021258033,
+      "tokens_seen": 2216886272
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033148772267693787,
+      "loss": 1.277,
+      "theoretical_loss": 3.42980987261405,
+      "tokens_seen": 2217017344
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033144760070614673,
+      "loss": 1.2381,
+      "theoretical_loss": 3.429793444345548,
+      "tokens_seen": 2217148416
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003314074787353555,
+      "loss": 1.2344,
+      "theoretical_loss": 3.4297770173201303,
+      "tokens_seen": 2217279488
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003313673567645643,
+      "loss": 1.2745,
+      "theoretical_loss": 3.429760591537629,
+      "tokens_seen": 2217410560
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003313272347937731,
+      "loss": 1.23,
+      "theoretical_loss": 3.4297441669978768,
+      "tokens_seen": 2217541632
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033128711282298185,
+      "loss": 1.2737,
+      "theoretical_loss": 3.429727743700706,
+      "tokens_seen": 2217672704
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033124699085219065,
+      "loss": 1.3049,
+      "theoretical_loss": 3.429711321645949,
+      "tokens_seen": 2217803776
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033120686888139946,
+      "loss": 1.1986,
+      "theoretical_loss": 3.4296949008334394,
+      "tokens_seen": 2217934848
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033116674691060827,
+      "loss": 1.2815,
+      "theoretical_loss": 3.4296784812630086,
+      "tokens_seen": 2218065920
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000331126624939817,
+      "loss": 1.3338,
+      "theoretical_loss": 3.42966206293449,
+      "tokens_seen": 2218196992
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003310865029690259,
+      "loss": 1.1824,
+      "theoretical_loss": 3.429645645847716,
+      "tokens_seen": 2218328064
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033104638099823463,
+      "loss": 1.297,
+      "theoretical_loss": 3.4296292300025195,
+      "tokens_seen": 2218459136
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033100625902744344,
+      "loss": 1.2841,
+      "theoretical_loss": 3.429612815398733,
+      "tokens_seen": 2218590208
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033096613705665224,
+      "loss": 1.2643,
+      "theoretical_loss": 3.4295964020361893,
+      "tokens_seen": 2218721280
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000330926015085861,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4295799899147217,
+      "tokens_seen": 2218852352
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033088589311506986,
+      "loss": 1.2317,
+      "theoretical_loss": 3.4295635790341623,
+      "tokens_seen": 2218983424
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003308457711442786,
+      "loss": 1.2133,
+      "theoretical_loss": 3.4295471693943447,
+      "tokens_seen": 2219114496
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003308056491734874,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4295307609951013,
+      "tokens_seen": 2219245568
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003307655272026962,
+      "loss": 1.275,
+      "theoretical_loss": 3.4295143538362654,
+      "tokens_seen": 2219376640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000330725405231905,
+      "loss": 1.2549,
+      "theoretical_loss": 3.4294979479176697,
+      "tokens_seen": 2219507712
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003306852832611138,
+      "loss": 1.2694,
+      "theoretical_loss": 3.4294815432391474,
+      "tokens_seen": 2219638784
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003306451612903226,
+      "loss": 1.3288,
+      "theoretical_loss": 3.4294651398005316,
+      "tokens_seen": 2219769856
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003306050393195314,
+      "loss": 1.2559,
+      "theoretical_loss": 3.429448737601655,
+      "tokens_seen": 2219900928
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033056491734874014,
+      "loss": 1.2273,
+      "theoretical_loss": 3.4294323366423516,
+      "tokens_seen": 2220032000
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 746497,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2674071788787842,
+      "objective/train/theoretical_loss": 3.4294159369224535,
+      "objective/train/tokens_used": 590622176,
+      "theoretical_loss": 3.4294159369224535,
+      "tokens_seen": 2220163072
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000330524795377949,
+      "loss": 1.2121,
+      "theoretical_loss": 3.4294159369224535,
+      "tokens_seen": 2220163072
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033048467340715776,
+      "loss": 1.2253,
+      "theoretical_loss": 3.4293995384417943,
+      "tokens_seen": 2220294144
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033044455143636656,
+      "loss": 1.2662,
+      "theoretical_loss": 3.4293831412002076,
+      "tokens_seen": 2220425216
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033040442946557537,
+      "loss": 1.323,
+      "theoretical_loss": 3.429366745197526,
+      "tokens_seen": 2220556288
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003303643074947841,
+      "loss": 1.2648,
+      "theoretical_loss": 3.429350350433583,
+      "tokens_seen": 2220687360
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000330324185523993,
+      "loss": 1.2721,
+      "theoretical_loss": 3.429333956908212,
+      "tokens_seen": 2220818432
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033028406355320173,
+      "loss": 1.1987,
+      "theoretical_loss": 3.429317564621246,
+      "tokens_seen": 2220949504
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033024394158241054,
+      "loss": 1.2219,
+      "theoretical_loss": 3.4293011735725187,
+      "tokens_seen": 2221080576
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033020381961161935,
+      "loss": 1.3176,
+      "theoretical_loss": 3.429284783761864,
+      "tokens_seen": 2221211648
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003301636976408281,
+      "loss": 1.1767,
+      "theoretical_loss": 3.4292683951891143,
+      "tokens_seen": 2221342720
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003301235756700369,
+      "loss": 1.2821,
+      "theoretical_loss": 3.4292520078541036,
+      "tokens_seen": 2221473792
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003300834536992457,
+      "loss": 1.2481,
+      "theoretical_loss": 3.4292356217566655,
+      "tokens_seen": 2221604864
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003300433317284545,
+      "loss": 1.2988,
+      "theoretical_loss": 3.429219236896633,
+      "tokens_seen": 2221735936
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033000320975766327,
+      "loss": 1.2383,
+      "theoretical_loss": 3.429202853273841,
+      "tokens_seen": 2221867008
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032996308778687213,
+      "loss": 1.3197,
+      "theoretical_loss": 3.429186470888121,
+      "tokens_seen": 2221998080
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003299229658160809,
+      "loss": 1.2813,
+      "theoretical_loss": 3.4291700897393085,
+      "tokens_seen": 2222129152
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003298828438452897,
+      "loss": 1.3042,
+      "theoretical_loss": 3.4291537098272364,
+      "tokens_seen": 2222260224
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003298427218744985,
+      "loss": 1.2718,
+      "theoretical_loss": 3.4291373311517384,
+      "tokens_seen": 2222391296
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032980259990370725,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4291209537126477,
+      "tokens_seen": 2222522368
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003297624779329161,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4291045775097992,
+      "tokens_seen": 2222653440
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032972235596212486,
+      "loss": 1.2266,
+      "theoretical_loss": 3.429088202543026,
+      "tokens_seen": 2222784512
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032968223399133367,
+      "loss": 1.2316,
+      "theoretical_loss": 3.429071828812162,
+      "tokens_seen": 2222915584
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032964211202054247,
+      "loss": 1.2608,
+      "theoretical_loss": 3.429055456317041,
+      "tokens_seen": 2223046656
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003296019900497513,
+      "loss": 1.2804,
+      "theoretical_loss": 3.429039085057497,
+      "tokens_seen": 2223177728
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032956186807896003,
+      "loss": 1.2499,
+      "theoretical_loss": 3.429022715033364,
+      "tokens_seen": 2223308800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032952174610816884,
+      "loss": 1.3429,
+      "theoretical_loss": 3.4290063462444755,
+      "tokens_seen": 2223439872
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032948162413737764,
+      "loss": 1.219,
+      "theoretical_loss": 3.428989978690666,
+      "tokens_seen": 2223570944
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003294415021665864,
+      "loss": 1.224,
+      "theoretical_loss": 3.4289736123717693,
+      "tokens_seen": 2223702016
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032940138019579526,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4289572472876193,
+      "tokens_seen": 2223833088
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000329361258225004,
+      "loss": 1.2958,
+      "theoretical_loss": 3.4289408834380506,
+      "tokens_seen": 2223964160
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003293211362542128,
+      "loss": 1.2587,
+      "theoretical_loss": 3.4289245208228967,
+      "tokens_seen": 2224095232
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003292810142834216,
+      "loss": 1.2687,
+      "theoretical_loss": 3.428908159441992,
+      "tokens_seen": 2224226304
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032924089231263037,
+      "loss": 1.3132,
+      "theoretical_loss": 3.428891799295171,
+      "tokens_seen": 2224357376
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032920077034183923,
+      "loss": 1.2551,
+      "theoretical_loss": 3.428875440382267,
+      "tokens_seen": 2224488448
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000329160648371048,
+      "loss": 1.2371,
+      "theoretical_loss": 3.4288590827031156,
+      "tokens_seen": 2224619520
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003291205264002568,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4288427262575496,
+      "tokens_seen": 2224750592
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003290804044294656,
+      "loss": 1.2738,
+      "theoretical_loss": 3.428826371045404,
+      "tokens_seen": 2224881664
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003290402824586744,
+      "loss": 1.2677,
+      "theoretical_loss": 3.4288100170665134,
+      "tokens_seen": 2225012736
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032900016048788316,
+      "loss": 1.2205,
+      "theoretical_loss": 3.428793664320712,
+      "tokens_seen": 2225143808
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032896003851709196,
+      "loss": 1.2149,
+      "theoretical_loss": 3.428777312807834,
+      "tokens_seen": 2225274880
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032891991654630077,
+      "loss": 1.2927,
+      "theoretical_loss": 3.4287609625277136,
+      "tokens_seen": 2225405952
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003288797945755095,
+      "loss": 1.2213,
+      "theoretical_loss": 3.4287446134801858,
+      "tokens_seen": 2225537024
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003288396726047184,
+      "loss": 1.2424,
+      "theoretical_loss": 3.428728265665085,
+      "tokens_seen": 2225668096
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032879955063392713,
+      "loss": 1.2964,
+      "theoretical_loss": 3.4287119190822457,
+      "tokens_seen": 2225799168
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032875942866313594,
+      "loss": 1.2818,
+      "theoretical_loss": 3.4286955737315017,
+      "tokens_seen": 2225930240
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032871930669234475,
+      "loss": 1.3178,
+      "theoretical_loss": 3.4286792296126887,
+      "tokens_seen": 2226061312
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003286791847215535,
+      "loss": 1.2908,
+      "theoretical_loss": 3.428662886725641,
+      "tokens_seen": 2226192384
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003286390627507623,
+      "loss": 1.2822,
+      "theoretical_loss": 3.428646545070193,
+      "tokens_seen": 2226323456
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003285989407799711,
+      "loss": 1.2694,
+      "theoretical_loss": 3.428630204646179,
+      "tokens_seen": 2226454528
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003285588188091799,
+      "loss": 1.2912,
+      "theoretical_loss": 3.428613865453435,
+      "tokens_seen": 2226585600
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 748781,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2060874700546265,
+      "objective/train/theoretical_loss": 3.428597527491795,
+      "objective/train/tokens_used": 597175776,
+      "theoretical_loss": 3.428597527491795,
+      "tokens_seen": 2226716672
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003285186968383887,
+      "loss": 1.2343,
+      "theoretical_loss": 3.428597527491795,
+      "tokens_seen": 2226716672
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032847857486759753,
+      "loss": 1.2449,
+      "theoretical_loss": 3.4285811907610935,
+      "tokens_seen": 2226847744
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003284384528968063,
+      "loss": 1.2223,
+      "theoretical_loss": 3.428564855261165,
+      "tokens_seen": 2226978816
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003283983309260151,
+      "loss": 1.2711,
+      "theoretical_loss": 3.428548520991846,
+      "tokens_seen": 2227109888
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003283582089552239,
+      "loss": 1.294,
+      "theoretical_loss": 3.4285321879529693,
+      "tokens_seen": 2227240960
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032831808698443265,
+      "loss": 1.3291,
+      "theoretical_loss": 3.4285158561443714,
+      "tokens_seen": 2227372032
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003282779650136415,
+      "loss": 1.244,
+      "theoretical_loss": 3.4284995255658863,
+      "tokens_seen": 2227503104
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032823784304285026,
+      "loss": 1.2932,
+      "theoretical_loss": 3.42848319621735,
+      "tokens_seen": 2227634176
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032819772107205907,
+      "loss": 1.2768,
+      "theoretical_loss": 3.428466868098596,
+      "tokens_seen": 2227765248
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032815759910126787,
+      "loss": 1.3205,
+      "theoretical_loss": 3.4284505412094606,
+      "tokens_seen": 2227896320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003281174771304767,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4284342155497782,
+      "tokens_seen": 2228027392
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032807735515968543,
+      "loss": 1.2344,
+      "theoretical_loss": 3.428417891119384,
+      "tokens_seen": 2228158464
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032803723318889424,
+      "loss": 1.3398,
+      "theoretical_loss": 3.428401567918114,
+      "tokens_seen": 2228289536
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032799711121810304,
+      "loss": 1.2718,
+      "theoretical_loss": 3.428385245945802,
+      "tokens_seen": 2228420608
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003279569892473118,
+      "loss": 1.2401,
+      "theoretical_loss": 3.428368925202284,
+      "tokens_seen": 2228551680
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032791686727652066,
+      "loss": 1.1965,
+      "theoretical_loss": 3.4283526056873947,
+      "tokens_seen": 2228682752
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003278767453057294,
+      "loss": 1.2756,
+      "theoretical_loss": 3.4283362874009704,
+      "tokens_seen": 2228813824
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032783662333493827,
+      "loss": 1.2706,
+      "theoretical_loss": 3.4283199703428453,
+      "tokens_seen": 2228944896
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000327796501364147,
+      "loss": 1.2467,
+      "theoretical_loss": 3.428303654512855,
+      "tokens_seen": 2229075968
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032775637939335577,
+      "loss": 1.2545,
+      "theoretical_loss": 3.428287339910835,
+      "tokens_seen": 2229207040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032771625742256463,
+      "loss": 1.3052,
+      "theoretical_loss": 3.4282710265366205,
+      "tokens_seen": 2229338112
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003276761354517734,
+      "loss": 1.2819,
+      "theoretical_loss": 3.4282547143900475,
+      "tokens_seen": 2229469184
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003276360134809822,
+      "loss": 1.2328,
+      "theoretical_loss": 3.428238403470951,
+      "tokens_seen": 2229600256
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000327595891510191,
+      "loss": 1.2318,
+      "theoretical_loss": 3.4282220937791665,
+      "tokens_seen": 2229731328
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003275557695393998,
+      "loss": 1.2626,
+      "theoretical_loss": 3.428205785314529,
+      "tokens_seen": 2229862400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032751564756860856,
+      "loss": 1.2631,
+      "theoretical_loss": 3.428189478076875,
+      "tokens_seen": 2229993472
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032747552559781736,
+      "loss": 1.273,
+      "theoretical_loss": 3.4281731720660398,
+      "tokens_seen": 2230124544
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032743540362702617,
+      "loss": 1.3022,
+      "theoretical_loss": 3.4281568672818588,
+      "tokens_seen": 2230255616
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003273952816562349,
+      "loss": 1.2037,
+      "theoretical_loss": 3.4281405637241678,
+      "tokens_seen": 2230386688
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003273551596854438,
+      "loss": 1.2908,
+      "theoretical_loss": 3.428124261392802,
+      "tokens_seen": 2230517760
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032731503771465253,
+      "loss": 1.2709,
+      "theoretical_loss": 3.4281079602875977,
+      "tokens_seen": 2230648832
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003272749157438614,
+      "loss": 1.258,
+      "theoretical_loss": 3.42809166040839,
+      "tokens_seen": 2230779904
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032723479377307015,
+      "loss": 1.2098,
+      "theoretical_loss": 3.4280753617550155,
+      "tokens_seen": 2230910976
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003271946718022789,
+      "loss": 1.2306,
+      "theoretical_loss": 3.4280590643273094,
+      "tokens_seen": 2231042048
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032715454983148776,
+      "loss": 1.2493,
+      "theoretical_loss": 3.428042768125108,
+      "tokens_seen": 2231173120
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003271144278606965,
+      "loss": 1.2235,
+      "theoretical_loss": 3.4280264731482464,
+      "tokens_seen": 2231304192
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003270743058899053,
+      "loss": 1.26,
+      "theoretical_loss": 3.428010179396561,
+      "tokens_seen": 2231435264
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003270341839191141,
+      "loss": 1.2196,
+      "theoretical_loss": 3.427993886869888,
+      "tokens_seen": 2231566336
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032699406194832293,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4279775955680627,
+      "tokens_seen": 2231697408
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003269539399775317,
+      "loss": 1.2912,
+      "theoretical_loss": 3.427961305490922,
+      "tokens_seen": 2231828480
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003269138180067405,
+      "loss": 1.219,
+      "theoretical_loss": 3.4279450166383008,
+      "tokens_seen": 2231959552
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003268736960359493,
+      "loss": 1.3383,
+      "theoretical_loss": 3.427928729010036,
+      "tokens_seen": 2232090624
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032683357406515805,
+      "loss": 1.2845,
+      "theoretical_loss": 3.427912442605963,
+      "tokens_seen": 2232221696
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003267934520943669,
+      "loss": 1.2408,
+      "theoretical_loss": 3.4278961574259186,
+      "tokens_seen": 2232352768
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032675333012357566,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4278798734697387,
+      "tokens_seen": 2232483840
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032671320815278447,
+      "loss": 1.3582,
+      "theoretical_loss": 3.4278635907372594,
+      "tokens_seen": 2232614912
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032667308618199327,
+      "loss": 1.2484,
+      "theoretical_loss": 3.4278473092283166,
+      "tokens_seen": 2232745984
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003266329642112021,
+      "loss": 1.2418,
+      "theoretical_loss": 3.427831028942747,
+      "tokens_seen": 2232877056
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003265928422404109,
+      "loss": 1.2592,
+      "theoretical_loss": 3.427814749880387,
+      "tokens_seen": 2233008128
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032655272026961964,
+      "loss": 1.3228,
+      "theoretical_loss": 3.4277984720410726,
+      "tokens_seen": 2233139200
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 751008,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.261589527130127,
+      "objective/train/theoretical_loss": 3.4277821954246397,
+      "objective/train/tokens_used": 603729376,
+      "theoretical_loss": 3.4277821954246397,
+      "tokens_seen": 2233270272
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032651259829882844,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4277821954246397,
+      "tokens_seen": 2233270272
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032647247632803725,
+      "loss": 1.2552,
+      "theoretical_loss": 3.4277659200309256,
+      "tokens_seen": 2233401344
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032643235435724606,
+      "loss": 1.2317,
+      "theoretical_loss": 3.427749645859766,
+      "tokens_seen": 2233532416
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003263922323864548,
+      "loss": 1.2448,
+      "theoretical_loss": 3.427733372910998,
+      "tokens_seen": 2233663488
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032635211041566367,
+      "loss": 1.2383,
+      "theoretical_loss": 3.427717101184457,
+      "tokens_seen": 2233794560
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003263119884448724,
+      "loss": 1.277,
+      "theoretical_loss": 3.4277008306799805,
+      "tokens_seen": 2233925632
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032627186647408117,
+      "loss": 1.2165,
+      "theoretical_loss": 3.4276845613974047,
+      "tokens_seen": 2234056704
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032623174450329003,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4276682933365663,
+      "tokens_seen": 2234187776
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003261916225324988,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4276520264973014,
+      "tokens_seen": 2234318848
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003261515005617076,
+      "loss": 1.3174,
+      "theoretical_loss": 3.4276357608794465,
+      "tokens_seen": 2234449920
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003261113785909164,
+      "loss": 1.2113,
+      "theoretical_loss": 3.4276194964828393,
+      "tokens_seen": 2234580992
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003260712566201252,
+      "loss": 1.1964,
+      "theoretical_loss": 3.4276032333073156,
+      "tokens_seen": 2234712064
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032603113464933396,
+      "loss": 1.269,
+      "theoretical_loss": 3.4275869713527123,
+      "tokens_seen": 2234843136
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032599101267854276,
+      "loss": 1.3167,
+      "theoretical_loss": 3.427570710618866,
+      "tokens_seen": 2234974208
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032595089070775157,
+      "loss": 1.2557,
+      "theoretical_loss": 3.427554451105614,
+      "tokens_seen": 2235105280
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003259107687369604,
+      "loss": 1.2466,
+      "theoretical_loss": 3.427538192812793,
+      "tokens_seen": 2235236352
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003258706467661692,
+      "loss": 1.2968,
+      "theoretical_loss": 3.427521935740239,
+      "tokens_seen": 2235367424
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032583052479537793,
+      "loss": 1.3053,
+      "theoretical_loss": 3.42750567988779,
+      "tokens_seen": 2235498496
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003257904028245868,
+      "loss": 1.2397,
+      "theoretical_loss": 3.427489425255282,
+      "tokens_seen": 2235629568
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032575028085379555,
+      "loss": 1.3088,
+      "theoretical_loss": 3.4274731718425526,
+      "tokens_seen": 2235760640
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003257101588830043,
+      "loss": 1.3026,
+      "theoretical_loss": 3.4274569196494387,
+      "tokens_seen": 2235891712
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032567003691221316,
+      "loss": 1.2838,
+      "theoretical_loss": 3.427440668675777,
+      "tokens_seen": 2236022784
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003256299149414219,
+      "loss": 1.2236,
+      "theoretical_loss": 3.427424418921404,
+      "tokens_seen": 2236153856
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003255897929706307,
+      "loss": 1.282,
+      "theoretical_loss": 3.427408170386158,
+      "tokens_seen": 2236284928
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003255496709998395,
+      "loss": 1.2522,
+      "theoretical_loss": 3.4273919230698753,
+      "tokens_seen": 2236416000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032550954902904833,
+      "loss": 1.2827,
+      "theoretical_loss": 3.4273756769723933,
+      "tokens_seen": 2236547072
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003254694270582571,
+      "loss": 1.2767,
+      "theoretical_loss": 3.4273594320935485,
+      "tokens_seen": 2236678144
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003254293050874659,
+      "loss": 1.2516,
+      "theoretical_loss": 3.427343188433179,
+      "tokens_seen": 2236809216
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003253891831166747,
+      "loss": 1.3028,
+      "theoretical_loss": 3.4273269459911218,
+      "tokens_seen": 2236940288
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003253490611458835,
+      "loss": 1.2559,
+      "theoretical_loss": 3.427310704767214,
+      "tokens_seen": 2237071360
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003253089391750923,
+      "loss": 1.293,
+      "theoretical_loss": 3.4272944647612924,
+      "tokens_seen": 2237202432
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032526881720430106,
+      "loss": 1.2727,
+      "theoretical_loss": 3.4272782259731946,
+      "tokens_seen": 2237333504
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003252286952335099,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4272619884027584,
+      "tokens_seen": 2237464576
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032518857326271867,
+      "loss": 1.2505,
+      "theoretical_loss": 3.427245752049821,
+      "tokens_seen": 2237595648
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003251484512919275,
+      "loss": 1.2653,
+      "theoretical_loss": 3.4272295169142195,
+      "tokens_seen": 2237726720
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003251083293211363,
+      "loss": 1.3262,
+      "theoretical_loss": 3.4272132829957918,
+      "tokens_seen": 2237857792
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032506820735034504,
+      "loss": 1.2693,
+      "theoretical_loss": 3.427197050294375,
+      "tokens_seen": 2237988864
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032502808537955384,
+      "loss": 1.2219,
+      "theoretical_loss": 3.4271808188098065,
+      "tokens_seen": 2238119936
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032498796340876265,
+      "loss": 1.2608,
+      "theoretical_loss": 3.427164588541924,
+      "tokens_seen": 2238251008
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032494784143797145,
+      "loss": 1.2967,
+      "theoretical_loss": 3.427148359490565,
+      "tokens_seen": 2238382080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003249077194671802,
+      "loss": 1.3314,
+      "theoretical_loss": 3.427132131655567,
+      "tokens_seen": 2238513152
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032486759749638907,
+      "loss": 1.2731,
+      "theoretical_loss": 3.427115905036768,
+      "tokens_seen": 2238644224
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003248274755255978,
+      "loss": 1.2939,
+      "theoretical_loss": 3.427099679634005,
+      "tokens_seen": 2238775296
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032478735355480657,
+      "loss": 1.2575,
+      "theoretical_loss": 3.427083455447117,
+      "tokens_seen": 2238906368
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032474723158401543,
+      "loss": 1.2869,
+      "theoretical_loss": 3.42706723247594,
+      "tokens_seen": 2239037440
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003247071096132242,
+      "loss": 1.2637,
+      "theoretical_loss": 3.4270510107203127,
+      "tokens_seen": 2239168512
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032466698764243304,
+      "loss": 1.1878,
+      "theoretical_loss": 3.427034790180073,
+      "tokens_seen": 2239299584
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003246268656716418,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4270185708550582,
+      "tokens_seen": 2239430656
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003245867437008506,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4270023527451063,
+      "tokens_seen": 2239561728
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003245466217300594,
+      "loss": 1.29,
+      "theoretical_loss": 3.4269861358500555,
+      "tokens_seen": 2239692800
+    },
+    {
+      "epoch": 0.36,
+      "objective/train/docs_used": 752999,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1702591180801392,
+      "objective/train/theoretical_loss": 3.4269699201697432,
+      "objective/train/tokens_used": 610282976,
+      "theoretical_loss": 3.4269699201697432,
+      "tokens_seen": 2239823872
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032450649975926816,
+      "loss": 1.2072,
+      "theoretical_loss": 3.4269699201697432,
+      "tokens_seen": 2239823872
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032446637778847697,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4269537057040074,
+      "tokens_seen": 2239954944
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003244262558176858,
+      "loss": 1.2933,
+      "theoretical_loss": 3.4269374924526868,
+      "tokens_seen": 2240086016
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003243861338468946,
+      "loss": 1.257,
+      "theoretical_loss": 3.4269212804156184,
+      "tokens_seen": 2240217088
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032434601187610333,
+      "loss": 1.3576,
+      "theoretical_loss": 3.4269050695926406,
+      "tokens_seen": 2240348160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003243058899053122,
+      "loss": 1.221,
+      "theoretical_loss": 3.426888859983592,
+      "tokens_seen": 2240479232
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032426576793452095,
+      "loss": 1.2743,
+      "theoretical_loss": 3.42687265158831,
+      "tokens_seen": 2240610304
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003242256459637297,
+      "loss": 1.2868,
+      "theoretical_loss": 3.4268564444066327,
+      "tokens_seen": 2240741376
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032418552399293856,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4268402384383982,
+      "tokens_seen": 2240872448
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003241454020221473,
+      "loss": 1.3356,
+      "theoretical_loss": 3.4268240336834457,
+      "tokens_seen": 2241003520
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032410528005135617,
+      "loss": 1.2819,
+      "theoretical_loss": 3.426807830141612,
+      "tokens_seen": 2241134592
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003240651580805649,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4267916278127366,
+      "tokens_seen": 2241265664
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032402503610977373,
+      "loss": 1.2631,
+      "theoretical_loss": 3.426775426696657,
+      "tokens_seen": 2241396736
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032398491413898254,
+      "loss": 1.2302,
+      "theoretical_loss": 3.4267592267932114,
+      "tokens_seen": 2241527808
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003239447921681913,
+      "loss": 1.2961,
+      "theoretical_loss": 3.4267430281022384,
+      "tokens_seen": 2241658880
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003239046701974001,
+      "loss": 1.2627,
+      "theoretical_loss": 3.4267268306235765,
+      "tokens_seen": 2241789952
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003238645482266089,
+      "loss": 1.282,
+      "theoretical_loss": 3.426710634357064,
+      "tokens_seen": 2241921024
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003238244262558177,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4266944393025396,
+      "tokens_seen": 2242052096
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032378430428502646,
+      "loss": 1.2736,
+      "theoretical_loss": 3.426678245459841,
+      "tokens_seen": 2242183168
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003237441823142353,
+      "loss": 1.2812,
+      "theoretical_loss": 3.4266620528288074,
+      "tokens_seen": 2242314240
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032370406034344407,
+      "loss": 1.2065,
+      "theoretical_loss": 3.426645861409277,
+      "tokens_seen": 2242445312
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003236639383726528,
+      "loss": 1.2833,
+      "theoretical_loss": 3.4266296712010886,
+      "tokens_seen": 2242576384
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003236238164018617,
+      "loss": 1.3419,
+      "theoretical_loss": 3.4266134822040804,
+      "tokens_seen": 2242707456
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032358369443107044,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4265972944180914,
+      "tokens_seen": 2242838528
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032354357246027924,
+      "loss": 1.2337,
+      "theoretical_loss": 3.42658110784296,
+      "tokens_seen": 2242969600
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032350345048948805,
+      "loss": 1.284,
+      "theoretical_loss": 3.4265649224785246,
+      "tokens_seen": 2243100672
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032346332851869685,
+      "loss": 1.255,
+      "theoretical_loss": 3.426548738324625,
+      "tokens_seen": 2243231744
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032342320654790566,
+      "loss": 1.254,
+      "theoretical_loss": 3.4265325553810984,
+      "tokens_seen": 2243362816
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032338308457711447,
+      "loss": 1.238,
+      "theoretical_loss": 3.426516373647785,
+      "tokens_seen": 2243493888
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003233429626063232,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4265001931245225,
+      "tokens_seen": 2243624960
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.000323302840635532,
+      "loss": 1.2087,
+      "theoretical_loss": 3.4264840138111508,
+      "tokens_seen": 2243756032
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032326271866474083,
+      "loss": 1.2974,
+      "theoretical_loss": 3.426467835707508,
+      "tokens_seen": 2243887104
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003232225966939496,
+      "loss": 1.2843,
+      "theoretical_loss": 3.4264516588134324,
+      "tokens_seen": 2244018176
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032318247472315844,
+      "loss": 1.1882,
+      "theoretical_loss": 3.4264354831287642,
+      "tokens_seen": 2244149248
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003231423527523672,
+      "loss": 1.2808,
+      "theoretical_loss": 3.426419308653342,
+      "tokens_seen": 2244280320
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.000323102230781576,
+      "loss": 1.3104,
+      "theoretical_loss": 3.4264031353870044,
+      "tokens_seen": 2244411392
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003230621088107848,
+      "loss": 1.2211,
+      "theoretical_loss": 3.4263869633295907,
+      "tokens_seen": 2244542464
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032302198683999356,
+      "loss": 1.3181,
+      "theoretical_loss": 3.42637079248094,
+      "tokens_seen": 2244673536
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032298186486920237,
+      "loss": 1.2284,
+      "theoretical_loss": 3.426354622840891,
+      "tokens_seen": 2244804608
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003229417428984112,
+      "loss": 1.2835,
+      "theoretical_loss": 3.426338454409283,
+      "tokens_seen": 2244935680
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032290162092762,
+      "loss": 1.182,
+      "theoretical_loss": 3.4263222871859558,
+      "tokens_seen": 2245066752
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032286149895682873,
+      "loss": 1.3005,
+      "theoretical_loss": 3.4263061211707475,
+      "tokens_seen": 2245197824
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003228213769860376,
+      "loss": 1.209,
+      "theoretical_loss": 3.426289956363498,
+      "tokens_seen": 2245328896
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032278125501524634,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4262737927640456,
+      "tokens_seen": 2245459968
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032274113304445515,
+      "loss": 1.2511,
+      "theoretical_loss": 3.426257630372231,
+      "tokens_seen": 2245591040
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032270101107366396,
+      "loss": 1.3342,
+      "theoretical_loss": 3.4262414691878926,
+      "tokens_seen": 2245722112
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003226608891028727,
+      "loss": 1.319,
+      "theoretical_loss": 3.42622530921087,
+      "tokens_seen": 2245853184
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032262076713208157,
+      "loss": 1.2844,
+      "theoretical_loss": 3.4262091504410024,
+      "tokens_seen": 2245984256
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003225806451612903,
+      "loss": 1.3213,
+      "theoretical_loss": 3.426192992878129,
+      "tokens_seen": 2246115328
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032254052319049913,
+      "loss": 1.2988,
+      "theoretical_loss": 3.4261768365220897,
+      "tokens_seen": 2246246400
+    },
+    {
+      "epoch": 0.36,
+      "objective/train/docs_used": 755049,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2806779146194458,
+      "objective/train/theoretical_loss": 3.4261606813727234,
+      "objective/train/tokens_used": 616836576,
+      "theoretical_loss": 3.4261606813727234,
+      "tokens_seen": 2246377472
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032250040121970793,
+      "loss": 1.2685,
+      "theoretical_loss": 3.4261606813727234,
+      "tokens_seen": 2246377472
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003224602792489167,
+      "loss": 1.2164,
+      "theoretical_loss": 3.42614452742987,
+      "tokens_seen": 2246508544
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003224201572781255,
+      "loss": 1.2632,
+      "theoretical_loss": 3.426128374693369,
+      "tokens_seen": 2246639616
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003223800353073343,
+      "loss": 1.3213,
+      "theoretical_loss": 3.4261122231630594,
+      "tokens_seen": 2246770688
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003223399133365431,
+      "loss": 1.3073,
+      "theoretical_loss": 3.4260960728387815,
+      "tokens_seen": 2246901760
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032229979136575186,
+      "loss": 1.3067,
+      "theoretical_loss": 3.4260799237203745,
+      "tokens_seen": 2247032832
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003222596693949607,
+      "loss": 1.2549,
+      "theoretical_loss": 3.4260637758076786,
+      "tokens_seen": 2247163904
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032221954742416947,
+      "loss": 1.193,
+      "theoretical_loss": 3.426047629100532,
+      "tokens_seen": 2247294976
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003221794254533782,
+      "loss": 1.2594,
+      "theoretical_loss": 3.426031483598776,
+      "tokens_seen": 2247426048
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003221393034825871,
+      "loss": 1.2127,
+      "theoretical_loss": 3.42601533930225,
+      "tokens_seen": 2247557120
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032209918151179584,
+      "loss": 1.2897,
+      "theoretical_loss": 3.425999196210793,
+      "tokens_seen": 2247688192
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003220590595410047,
+      "loss": 1.2832,
+      "theoretical_loss": 3.425983054324245,
+      "tokens_seen": 2247819264
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032201893757021345,
+      "loss": 1.2537,
+      "theoretical_loss": 3.425966913642447,
+      "tokens_seen": 2247950336
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032197881559942225,
+      "loss": 1.3046,
+      "theoretical_loss": 3.425950774165237,
+      "tokens_seen": 2248081408
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032193869362863106,
+      "loss": 1.2487,
+      "theoretical_loss": 3.425934635892456,
+      "tokens_seen": 2248212480
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032189857165783987,
+      "loss": 1.2543,
+      "theoretical_loss": 3.4259184988239433,
+      "tokens_seen": 2248343552
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003218584496870486,
+      "loss": 1.2535,
+      "theoretical_loss": 3.4259023629595395,
+      "tokens_seen": 2248474624
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003218183277162574,
+      "loss": 1.2558,
+      "theoretical_loss": 3.4258862282990847,
+      "tokens_seen": 2248605696
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032177820574546623,
+      "loss": 1.293,
+      "theoretical_loss": 3.425870094842418,
+      "tokens_seen": 2248736768
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.000321738083774675,
+      "loss": 1.3058,
+      "theoretical_loss": 3.4258539625893802,
+      "tokens_seen": 2248867840
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032169796180388384,
+      "loss": 1.2737,
+      "theoretical_loss": 3.425837831539811,
+      "tokens_seen": 2248998912
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003216578398330926,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4258217016935504,
+      "tokens_seen": 2249129984
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003216177178623014,
+      "loss": 1.239,
+      "theoretical_loss": 3.4258055730504386,
+      "tokens_seen": 2249261056
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003215775958915102,
+      "loss": 1.2125,
+      "theoretical_loss": 3.425789445610316,
+      "tokens_seen": 2249392128
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032153747392071896,
+      "loss": 1.2238,
+      "theoretical_loss": 3.4257733193730227,
+      "tokens_seen": 2249523200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003214973519499278,
+      "loss": 1.267,
+      "theoretical_loss": 3.4257571943383986,
+      "tokens_seen": 2249654272
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003214572299791366,
+      "loss": 1.2894,
+      "theoretical_loss": 3.4257410705062847,
+      "tokens_seen": 2249785344
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003214171080083454,
+      "loss": 1.2867,
+      "theoretical_loss": 3.42572494787652,
+      "tokens_seen": 2249916416
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003213769860375542,
+      "loss": 1.2842,
+      "theoretical_loss": 3.425708826448946,
+      "tokens_seen": 2250047488
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.000321336864066763,
+      "loss": 1.2363,
+      "theoretical_loss": 3.425692706223402,
+      "tokens_seen": 2250178560
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032129674209597174,
+      "loss": 1.3567,
+      "theoretical_loss": 3.42567658719973,
+      "tokens_seen": 2250309632
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032125662012518055,
+      "loss": 1.3174,
+      "theoretical_loss": 3.4256604693777684,
+      "tokens_seen": 2250440704
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032121649815438936,
+      "loss": 1.1966,
+      "theoretical_loss": 3.425644352757359,
+      "tokens_seen": 2250571776
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003211763761835981,
+      "loss": 1.3118,
+      "theoretical_loss": 3.425628237338342,
+      "tokens_seen": 2250702848
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032113625421280697,
+      "loss": 1.3062,
+      "theoretical_loss": 3.4256121231205574,
+      "tokens_seen": 2250833920
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003210961322420157,
+      "loss": 1.1588,
+      "theoretical_loss": 3.4255960101038463,
+      "tokens_seen": 2250964992
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032105601027122453,
+      "loss": 1.2852,
+      "theoretical_loss": 3.4255798982880488,
+      "tokens_seen": 2251096064
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032101588830043333,
+      "loss": 1.255,
+      "theoretical_loss": 3.4255637876730054,
+      "tokens_seen": 2251227136
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003209757663296421,
+      "loss": 1.2236,
+      "theoretical_loss": 3.425547678258557,
+      "tokens_seen": 2251358208
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003209356443588509,
+      "loss": 1.2111,
+      "theoretical_loss": 3.425531570044545,
+      "tokens_seen": 2251489280
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003208955223880597,
+      "loss": 1.3077,
+      "theoretical_loss": 3.4255154630308082,
+      "tokens_seen": 2251620352
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003208554004172685,
+      "loss": 1.2043,
+      "theoretical_loss": 3.425499357217189,
+      "tokens_seen": 2251751424
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003208152784464773,
+      "loss": 1.1858,
+      "theoretical_loss": 3.4254832526035273,
+      "tokens_seen": 2251882496
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003207751564756861,
+      "loss": 1.271,
+      "theoretical_loss": 3.4254671491896644,
+      "tokens_seen": 2252013568
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032073503450489487,
+      "loss": 1.24,
+      "theoretical_loss": 3.4254510469754407,
+      "tokens_seen": 2252144640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003206949125341037,
+      "loss": 1.2095,
+      "theoretical_loss": 3.4254349459606965,
+      "tokens_seen": 2252275712
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003206547905633125,
+      "loss": 1.2508,
+      "theoretical_loss": 3.425418846145274,
+      "tokens_seen": 2252406784
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032061466859252123,
+      "loss": 1.2266,
+      "theoretical_loss": 3.425402747529013,
+      "tokens_seen": 2252537856
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003205745466217301,
+      "loss": 1.1815,
+      "theoretical_loss": 3.4253866501117547,
+      "tokens_seen": 2252668928
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032053442465093885,
+      "loss": 1.2501,
+      "theoretical_loss": 3.42537055389334,
+      "tokens_seen": 2252800000
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 757120,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.264780879020691,
+      "objective/train/theoretical_loss": 3.42535445887361,
+      "objective/train/tokens_used": 623390176,
+      "theoretical_loss": 3.42535445887361,
+      "tokens_seen": 2252931072
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032049430268014765,
+      "loss": 1.2734,
+      "theoretical_loss": 3.42535445887361,
+      "tokens_seen": 2252931072
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032045418070935646,
+      "loss": 1.3318,
+      "theoretical_loss": 3.4253383650524056,
+      "tokens_seen": 2253062144
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003204140587385652,
+      "loss": 1.3159,
+      "theoretical_loss": 3.425322272429568,
+      "tokens_seen": 2253193216
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000320373936767774,
+      "loss": 1.2206,
+      "theoretical_loss": 3.425306181004938,
+      "tokens_seen": 2253324288
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003203338147969828,
+      "loss": 1.3463,
+      "theoretical_loss": 3.425290090778357,
+      "tokens_seen": 2253455360
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032029369282619163,
+      "loss": 1.2194,
+      "theoretical_loss": 3.425274001749666,
+      "tokens_seen": 2253586432
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032025357085540044,
+      "loss": 1.2366,
+      "theoretical_loss": 3.4252579139187063,
+      "tokens_seen": 2253717504
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032021344888460924,
+      "loss": 1.2233,
+      "theoretical_loss": 3.4252418272853187,
+      "tokens_seen": 2253848576
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000320173326913818,
+      "loss": 1.2019,
+      "theoretical_loss": 3.4252257418493444,
+      "tokens_seen": 2253979648
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032013320494302686,
+      "loss": 1.2598,
+      "theoretical_loss": 3.4252096576106252,
+      "tokens_seen": 2254110720
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003200930829722356,
+      "loss": 1.299,
+      "theoretical_loss": 3.425193574569002,
+      "tokens_seen": 2254241792
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032005296100144436,
+      "loss": 1.2469,
+      "theoretical_loss": 3.425177492724316,
+      "tokens_seen": 2254372864
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003200128390306532,
+      "loss": 1.2884,
+      "theoretical_loss": 3.4251614120764087,
+      "tokens_seen": 2254503936
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000319972717059862,
+      "loss": 1.1848,
+      "theoretical_loss": 3.4251453326251218,
+      "tokens_seen": 2254635008
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003199325950890708,
+      "loss": 1.1761,
+      "theoretical_loss": 3.425129254370296,
+      "tokens_seen": 2254766080
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003198924731182796,
+      "loss": 1.2862,
+      "theoretical_loss": 3.4251131773117733,
+      "tokens_seen": 2254897152
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003198523511474884,
+      "loss": 1.2377,
+      "theoretical_loss": 3.425097101449395,
+      "tokens_seen": 2255028224
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031981222917669714,
+      "loss": 1.2932,
+      "theoretical_loss": 3.4250810267830025,
+      "tokens_seen": 2255159296
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031977210720590595,
+      "loss": 1.2869,
+      "theoretical_loss": 3.4250649533124373,
+      "tokens_seen": 2255290368
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031973198523511476,
+      "loss": 1.2289,
+      "theoretical_loss": 3.425048881037541,
+      "tokens_seen": 2255421440
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003196918632643235,
+      "loss": 1.2891,
+      "theoretical_loss": 3.4250328099581555,
+      "tokens_seen": 2255552512
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031965174129353237,
+      "loss": 1.2277,
+      "theoretical_loss": 3.4250167400741214,
+      "tokens_seen": 2255683584
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003196116193227411,
+      "loss": 1.2299,
+      "theoretical_loss": 3.4250006713852814,
+      "tokens_seen": 2255814656
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031957149735195,
+      "loss": 1.271,
+      "theoretical_loss": 3.424984603891477,
+      "tokens_seen": 2255945728
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031953137538115873,
+      "loss": 1.202,
+      "theoretical_loss": 3.42496853759255,
+      "tokens_seen": 2256076800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003194912534103675,
+      "loss": 1.2403,
+      "theoretical_loss": 3.424952472488341,
+      "tokens_seen": 2256207872
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031945113143957635,
+      "loss": 1.2396,
+      "theoretical_loss": 3.4249364085786933,
+      "tokens_seen": 2256338944
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003194110094687851,
+      "loss": 1.2263,
+      "theoretical_loss": 3.4249203458634474,
+      "tokens_seen": 2256470016
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003193708874979939,
+      "loss": 1.2739,
+      "theoretical_loss": 3.424904284342446,
+      "tokens_seen": 2256601088
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003193307655272027,
+      "loss": 1.3718,
+      "theoretical_loss": 3.4248882240155307,
+      "tokens_seen": 2256732160
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003192906435564115,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4248721648825433,
+      "tokens_seen": 2256863232
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031925052158562027,
+      "loss": 1.3054,
+      "theoretical_loss": 3.424856106943326,
+      "tokens_seen": 2256994304
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003192103996148291,
+      "loss": 1.3204,
+      "theoretical_loss": 3.4248400501977203,
+      "tokens_seen": 2257125376
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003191702776440379,
+      "loss": 1.1691,
+      "theoretical_loss": 3.424823994645568,
+      "tokens_seen": 2257256448
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031913015567324663,
+      "loss": 1.3074,
+      "theoretical_loss": 3.424807940286712,
+      "tokens_seen": 2257387520
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003190900337024555,
+      "loss": 1.2384,
+      "theoretical_loss": 3.4247918871209935,
+      "tokens_seen": 2257518592
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031904991173166425,
+      "loss": 1.2924,
+      "theoretical_loss": 3.424775835148255,
+      "tokens_seen": 2257649664
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003190097897608731,
+      "loss": 1.2488,
+      "theoretical_loss": 3.424759784368338,
+      "tokens_seen": 2257780736
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031896966779008186,
+      "loss": 1.2753,
+      "theoretical_loss": 3.424743734781086,
+      "tokens_seen": 2257911808
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003189295458192906,
+      "loss": 1.2747,
+      "theoretical_loss": 3.4247276863863396,
+      "tokens_seen": 2258042880
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031888942384849947,
+      "loss": 1.2788,
+      "theoretical_loss": 3.4247116391839416,
+      "tokens_seen": 2258173952
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003188493018777082,
+      "loss": 1.2473,
+      "theoretical_loss": 3.4246955931737344,
+      "tokens_seen": 2258305024
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031880917990691703,
+      "loss": 1.2618,
+      "theoretical_loss": 3.42467954835556,
+      "tokens_seen": 2258436096
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031876905793612584,
+      "loss": 1.2725,
+      "theoretical_loss": 3.424663504729261,
+      "tokens_seen": 2258567168
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031872893596533464,
+      "loss": 1.2526,
+      "theoretical_loss": 3.424647462294679,
+      "tokens_seen": 2258698240
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003186888139945434,
+      "loss": 1.2259,
+      "theoretical_loss": 3.424631421051657,
+      "tokens_seen": 2258829312
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031864869202375226,
+      "loss": 1.308,
+      "theoretical_loss": 3.424615381000037,
+      "tokens_seen": 2258960384
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000318608570052961,
+      "loss": 1.2447,
+      "theoretical_loss": 3.424599342139661,
+      "tokens_seen": 2259091456
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031856844808216976,
+      "loss": 1.2906,
+      "theoretical_loss": 3.424583304470373,
+      "tokens_seen": 2259222528
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003185283261113786,
+      "loss": 1.2693,
+      "theoretical_loss": 3.424567267992014,
+      "tokens_seen": 2259353600
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 759180,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2880125045776367,
+      "objective/train/theoretical_loss": 3.4245512327044265,
+      "objective/train/tokens_used": 629943776,
+      "theoretical_loss": 3.4245512327044265,
+      "tokens_seen": 2259484672
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003184882041405874,
+      "loss": 1.2847,
+      "theoretical_loss": 3.4245512327044265,
+      "tokens_seen": 2259484672
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003184480821697962,
+      "loss": 1.2761,
+      "theoretical_loss": 3.4245351986074537,
+      "tokens_seen": 2259615744
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000318407960199005,
+      "loss": 1.1973,
+      "theoretical_loss": 3.424519165700938,
+      "tokens_seen": 2259746816
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003183678382282138,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4245031339847216,
+      "tokens_seen": 2259877888
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003183277162574226,
+      "loss": 1.2573,
+      "theoretical_loss": 3.424487103458647,
+      "tokens_seen": 2260008960
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031828759428663135,
+      "loss": 1.3014,
+      "theoretical_loss": 3.4244710741225575,
+      "tokens_seen": 2260140032
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031824747231584016,
+      "loss": 1.243,
+      "theoretical_loss": 3.4244550459762957,
+      "tokens_seen": 2260271104
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031820735034504896,
+      "loss": 1.1952,
+      "theoretical_loss": 3.4244390190197036,
+      "tokens_seen": 2260402176
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031816722837425777,
+      "loss": 1.224,
+      "theoretical_loss": 3.424422993252624,
+      "tokens_seen": 2260533248
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003181271064034665,
+      "loss": 1.2379,
+      "theoretical_loss": 3.4244069686749006,
+      "tokens_seen": 2260664320
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003180869844326754,
+      "loss": 1.2818,
+      "theoretical_loss": 3.424390945286375,
+      "tokens_seen": 2260795392
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031804686246188413,
+      "loss": 1.2808,
+      "theoretical_loss": 3.4243749230868907,
+      "tokens_seen": 2260926464
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003180067404910929,
+      "loss": 1.246,
+      "theoretical_loss": 3.424358902076291,
+      "tokens_seen": 2261057536
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031796661852030175,
+      "loss": 1.3164,
+      "theoretical_loss": 3.4243428822544173,
+      "tokens_seen": 2261188608
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003179264965495105,
+      "loss": 1.2687,
+      "theoretical_loss": 3.424326863621114,
+      "tokens_seen": 2261319680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003178863745787193,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4243108461762226,
+      "tokens_seen": 2261450752
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003178462526079281,
+      "loss": 1.2616,
+      "theoretical_loss": 3.424294829919588,
+      "tokens_seen": 2261581824
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003178061306371369,
+      "loss": 1.2326,
+      "theoretical_loss": 3.4242788148510512,
+      "tokens_seen": 2261712896
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031776600866634567,
+      "loss": 1.1933,
+      "theoretical_loss": 3.4242628009704563,
+      "tokens_seen": 2261843968
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003177258866955545,
+      "loss": 1.2215,
+      "theoretical_loss": 3.424246788277646,
+      "tokens_seen": 2261975040
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003176857647247633,
+      "loss": 1.2381,
+      "theoretical_loss": 3.424230776772464,
+      "tokens_seen": 2262106112
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003176456427539721,
+      "loss": 1.2904,
+      "theoretical_loss": 3.424214766454752,
+      "tokens_seen": 2262237184
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003176055207831809,
+      "loss": 1.2329,
+      "theoretical_loss": 3.424198757324355,
+      "tokens_seen": 2262368256
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031756539881238965,
+      "loss": 1.3046,
+      "theoretical_loss": 3.4241827493811146,
+      "tokens_seen": 2262499328
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003175252768415985,
+      "loss": 1.2541,
+      "theoretical_loss": 3.424166742624875,
+      "tokens_seen": 2262630400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031748515487080726,
+      "loss": 1.2984,
+      "theoretical_loss": 3.4241507370554785,
+      "tokens_seen": 2262761472
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000317445032900016,
+      "loss": 1.2579,
+      "theoretical_loss": 3.424134732672769,
+      "tokens_seen": 2262892544
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031740491092922487,
+      "loss": 1.294,
+      "theoretical_loss": 3.4241187294765902,
+      "tokens_seen": 2263023616
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003173647889584336,
+      "loss": 1.3159,
+      "theoretical_loss": 3.4241027274667846,
+      "tokens_seen": 2263154688
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031732466698764243,
+      "loss": 1.2231,
+      "theoretical_loss": 3.4240867266431962,
+      "tokens_seen": 2263285760
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031728454501685124,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4240707270056676,
+      "tokens_seen": 2263416832
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031724442304606004,
+      "loss": 1.2676,
+      "theoretical_loss": 3.424054728554043,
+      "tokens_seen": 2263547904
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003172043010752688,
+      "loss": 1.2891,
+      "theoretical_loss": 3.424038731288165,
+      "tokens_seen": 2263678976
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031716417910447766,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4240227352078776,
+      "tokens_seen": 2263810048
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003171240571336864,
+      "loss": 1.29,
+      "theoretical_loss": 3.4240067403130245,
+      "tokens_seen": 2263941120
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003170839351628952,
+      "loss": 1.2655,
+      "theoretical_loss": 3.423990746603449,
+      "tokens_seen": 2264072192
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000317043813192104,
+      "loss": 1.2168,
+      "theoretical_loss": 3.4239747540789947,
+      "tokens_seen": 2264203264
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031700369122131277,
+      "loss": 1.3182,
+      "theoretical_loss": 3.423958762739505,
+      "tokens_seen": 2264334336
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031696356925052163,
+      "loss": 1.1928,
+      "theoretical_loss": 3.4239427725848235,
+      "tokens_seen": 2264465408
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003169234472797304,
+      "loss": 1.2086,
+      "theoretical_loss": 3.423926783614794,
+      "tokens_seen": 2264596480
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003168833253089392,
+      "loss": 1.2883,
+      "theoretical_loss": 3.4239107958292605,
+      "tokens_seen": 2264727552
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000316843203338148,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4238948092280657,
+      "tokens_seen": 2264858624
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031680308136735675,
+      "loss": 1.3322,
+      "theoretical_loss": 3.4238788238110542,
+      "tokens_seen": 2264989696
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031676295939656556,
+      "loss": 1.253,
+      "theoretical_loss": 3.42386283957807,
+      "tokens_seen": 2265120768
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031672283742577436,
+      "loss": 1.2279,
+      "theoretical_loss": 3.4238468565289564,
+      "tokens_seen": 2265251840
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031668271545498317,
+      "loss": 1.2926,
+      "theoretical_loss": 3.423830874663557,
+      "tokens_seen": 2265382912
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003166425934841919,
+      "loss": 1.2373,
+      "theoretical_loss": 3.423814893981716,
+      "tokens_seen": 2265513984
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003166024715134008,
+      "loss": 1.2457,
+      "theoretical_loss": 3.423798914483277,
+      "tokens_seen": 2265645056
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031656234954260953,
+      "loss": 1.2519,
+      "theoretical_loss": 3.423782936168084,
+      "tokens_seen": 2265776128
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003165222275718183,
+      "loss": 1.2053,
+      "theoretical_loss": 3.4237669590359814,
+      "tokens_seen": 2265907200
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 761497,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.302661657333374,
+      "objective/train/theoretical_loss": 3.423750983086813,
+      "objective/train/tokens_used": 636497376,
+      "theoretical_loss": 3.423750983086813,
+      "tokens_seen": 2266038272
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031648210560102715,
+      "loss": 1.2916,
+      "theoretical_loss": 3.423750983086813,
+      "tokens_seen": 2266038272
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003164419836302359,
+      "loss": 1.2368,
+      "theoretical_loss": 3.4237350083204223,
+      "tokens_seen": 2266169344
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031640186165944476,
+      "loss": 1.2541,
+      "theoretical_loss": 3.4237190347366537,
+      "tokens_seen": 2266300416
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003163617396886535,
+      "loss": 1.2175,
+      "theoretical_loss": 3.4237030623353513,
+      "tokens_seen": 2266431488
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003163216177178623,
+      "loss": 1.2855,
+      "theoretical_loss": 3.423687091116359,
+      "tokens_seen": 2266562560
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003162814957470711,
+      "loss": 1.2408,
+      "theoretical_loss": 3.423671121079521,
+      "tokens_seen": 2266693632
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003162413737762799,
+      "loss": 1.2301,
+      "theoretical_loss": 3.423655152224682,
+      "tokens_seen": 2266824704
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003162012518054887,
+      "loss": 1.2758,
+      "theoretical_loss": 3.4236391845516847,
+      "tokens_seen": 2266955776
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003161611298346975,
+      "loss": 1.2717,
+      "theoretical_loss": 3.423623218060375,
+      "tokens_seen": 2267086848
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003161210078639063,
+      "loss": 1.2884,
+      "theoretical_loss": 3.423607252750596,
+      "tokens_seen": 2267217920
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031608088589311505,
+      "loss": 1.2144,
+      "theoretical_loss": 3.423591288622193,
+      "tokens_seen": 2267348992
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003160407639223239,
+      "loss": 1.2614,
+      "theoretical_loss": 3.423575325675009,
+      "tokens_seen": 2267480064
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031600064195153266,
+      "loss": 1.1983,
+      "theoretical_loss": 3.4235593639088893,
+      "tokens_seen": 2267611136
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003159605199807414,
+      "loss": 1.2197,
+      "theoretical_loss": 3.423543403323678,
+      "tokens_seen": 2267742208
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031592039800995027,
+      "loss": 1.2165,
+      "theoretical_loss": 3.423527443919219,
+      "tokens_seen": 2267873280
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000315880276039159,
+      "loss": 1.3355,
+      "theoretical_loss": 3.4235114856953577,
+      "tokens_seen": 2268004352
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003158401540683679,
+      "loss": 1.229,
+      "theoretical_loss": 3.4234955286519377,
+      "tokens_seen": 2268135424
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031580003209757664,
+      "loss": 1.2386,
+      "theoretical_loss": 3.423479572788804,
+      "tokens_seen": 2268266496
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031575991012678544,
+      "loss": 1.1791,
+      "theoretical_loss": 3.4234636181058002,
+      "tokens_seen": 2268397568
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031571978815599425,
+      "loss": 1.286,
+      "theoretical_loss": 3.4234476646027723,
+      "tokens_seen": 2268528640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000315679666185203,
+      "loss": 1.2116,
+      "theoretical_loss": 3.4234317122795637,
+      "tokens_seen": 2268659712
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003156395442144118,
+      "loss": 1.254,
+      "theoretical_loss": 3.4234157611360194,
+      "tokens_seen": 2268790784
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003155994222436206,
+      "loss": 1.184,
+      "theoretical_loss": 3.4233998111719837,
+      "tokens_seen": 2268921856
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003155593002728294,
+      "loss": 1.1607,
+      "theoretical_loss": 3.423383862387302,
+      "tokens_seen": 2269052928
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031551917830203817,
+      "loss": 1.1939,
+      "theoretical_loss": 3.423367914781818,
+      "tokens_seen": 2269184000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031547905633124703,
+      "loss": 1.3212,
+      "theoretical_loss": 3.4233519683553775,
+      "tokens_seen": 2269315072
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003154389343604558,
+      "loss": 1.2223,
+      "theoretical_loss": 3.4233360231078245,
+      "tokens_seen": 2269446144
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003153988123896646,
+      "loss": 1.1765,
+      "theoretical_loss": 3.4233200790390037,
+      "tokens_seen": 2269577216
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003153586904188734,
+      "loss": 1.2051,
+      "theoretical_loss": 3.42330413614876,
+      "tokens_seen": 2269708288
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031531856844808215,
+      "loss": 1.2246,
+      "theoretical_loss": 3.423288194436939,
+      "tokens_seen": 2269839360
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031527844647729096,
+      "loss": 1.2887,
+      "theoretical_loss": 3.423272253903384,
+      "tokens_seen": 2269970432
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031523832450649976,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4232563145479413,
+      "tokens_seen": 2270101504
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031519820253570857,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4232403763704555,
+      "tokens_seen": 2270232576
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003151580805649174,
+      "loss": 1.3262,
+      "theoretical_loss": 3.4232244393707707,
+      "tokens_seen": 2270363648
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003151179585941262,
+      "loss": 1.2216,
+      "theoretical_loss": 3.423208503548733,
+      "tokens_seen": 2270494720
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031507783662333493,
+      "loss": 1.317,
+      "theoretical_loss": 3.423192568904187,
+      "tokens_seen": 2270625792
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031503771465254374,
+      "loss": 1.2436,
+      "theoretical_loss": 3.4231766354369775,
+      "tokens_seen": 2270756864
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031499759268175255,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4231607031469498,
+      "tokens_seen": 2270887936
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003149574707109613,
+      "loss": 1.2708,
+      "theoretical_loss": 3.4231447720339485,
+      "tokens_seen": 2271019008
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031491734874017016,
+      "loss": 1.2376,
+      "theoretical_loss": 3.423128842097819,
+      "tokens_seen": 2271150080
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003148772267693789,
+      "loss": 1.2462,
+      "theoretical_loss": 3.4231129133384073,
+      "tokens_seen": 2271281152
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003148371047985877,
+      "loss": 1.3108,
+      "theoretical_loss": 3.4230969857555578,
+      "tokens_seen": 2271412224
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003147969828277965,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4230810593491157,
+      "tokens_seen": 2271543296
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003147568608570053,
+      "loss": 1.375,
+      "theoretical_loss": 3.423065134118926,
+      "tokens_seen": 2271674368
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003147167388862141,
+      "loss": 1.1838,
+      "theoretical_loss": 3.423049210064834,
+      "tokens_seen": 2271805440
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003146766169154229,
+      "loss": 1.2117,
+      "theoretical_loss": 3.423033287186686,
+      "tokens_seen": 2271936512
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003146364949446317,
+      "loss": 1.2655,
+      "theoretical_loss": 3.423017365484326,
+      "tokens_seen": 2272067584
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031459637297384045,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4230014449576003,
+      "tokens_seen": 2272198656
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003145562510030493,
+      "loss": 1.3196,
+      "theoretical_loss": 3.4229855256063537,
+      "tokens_seen": 2272329728
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031451612903225806,
+      "loss": 1.3246,
+      "theoretical_loss": 3.4229696074304323,
+      "tokens_seen": 2272460800
+    },
+    {
+      "epoch": 0.38,
+      "objective/train/docs_used": 763592,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2745773792266846,
+      "objective/train/theoretical_loss": 3.4229536904296807,
+      "objective/train/tokens_used": 643050976,
+      "theoretical_loss": 3.4229536904296807,
+      "tokens_seen": 2272591872
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031447600706146687,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4229536904296807,
+      "tokens_seen": 2272591872
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031443588509067567,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4229377746039447,
+      "tokens_seen": 2272722944
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003143957631198844,
+      "loss": 1.2534,
+      "theoretical_loss": 3.42292185995307,
+      "tokens_seen": 2272854016
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003143556411490933,
+      "loss": 1.1746,
+      "theoretical_loss": 3.422905946476902,
+      "tokens_seen": 2272985088
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031431551917830204,
+      "loss": 1.1887,
+      "theoretical_loss": 3.4228900341752864,
+      "tokens_seen": 2273116160
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031427539720751084,
+      "loss": 1.3101,
+      "theoretical_loss": 3.422874123048068,
+      "tokens_seen": 2273247232
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031423527523671965,
+      "loss": 1.3049,
+      "theoretical_loss": 3.4228582130950937,
+      "tokens_seen": 2273378304
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003141951532659284,
+      "loss": 1.2566,
+      "theoretical_loss": 3.4228423043162084,
+      "tokens_seen": 2273509376
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003141550312951372,
+      "loss": 1.2119,
+      "theoretical_loss": 3.4228263967112578,
+      "tokens_seen": 2273640448
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.000314114909324346,
+      "loss": 1.275,
+      "theoretical_loss": 3.4228104902800873,
+      "tokens_seen": 2273771520
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003140747873535548,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4227945850225434,
+      "tokens_seen": 2273902592
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031403466538276357,
+      "loss": 1.2586,
+      "theoretical_loss": 3.4227786809384715,
+      "tokens_seen": 2274033664
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031399454341197243,
+      "loss": 1.2008,
+      "theoretical_loss": 3.4227627780277174,
+      "tokens_seen": 2274164736
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003139544214411812,
+      "loss": 1.3135,
+      "theoretical_loss": 3.4227468762901267,
+      "tokens_seen": 2274295808
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031391429947039004,
+      "loss": 1.3182,
+      "theoretical_loss": 3.4227309757255457,
+      "tokens_seen": 2274426880
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003138741774995988,
+      "loss": 1.2913,
+      "theoretical_loss": 3.42271507633382,
+      "tokens_seen": 2274557952
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031383405552880755,
+      "loss": 1.2648,
+      "theoretical_loss": 3.4226991781147955,
+      "tokens_seen": 2274689024
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003137939335580164,
+      "loss": 1.2939,
+      "theoretical_loss": 3.4226832810683177,
+      "tokens_seen": 2274820096
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031375381158722516,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4226673851942335,
+      "tokens_seen": 2274951168
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031371368961643397,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4226514904923886,
+      "tokens_seen": 2275082240
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003136735676456428,
+      "loss": 1.2174,
+      "theoretical_loss": 3.4226355969626283,
+      "tokens_seen": 2275213312
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003136334456748516,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4226197046048,
+      "tokens_seen": 2275344384
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031359332370406033,
+      "loss": 1.2595,
+      "theoretical_loss": 3.4226038134187484,
+      "tokens_seen": 2275475456
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031355320173326914,
+      "loss": 1.2302,
+      "theoretical_loss": 3.42258792340432,
+      "tokens_seen": 2275606528
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031351307976247795,
+      "loss": 1.2333,
+      "theoretical_loss": 3.422572034561362,
+      "tokens_seen": 2275737600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003134729577916867,
+      "loss": 1.2786,
+      "theoretical_loss": 3.422556146889719,
+      "tokens_seen": 2275868672
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031343283582089556,
+      "loss": 1.2335,
+      "theoretical_loss": 3.4225402603892383,
+      "tokens_seen": 2275999744
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003133927138501043,
+      "loss": 1.2145,
+      "theoretical_loss": 3.4225243750597656,
+      "tokens_seen": 2276130816
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003133525918793131,
+      "loss": 1.2588,
+      "theoretical_loss": 3.4225084909011474,
+      "tokens_seen": 2276261888
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003133124699085219,
+      "loss": 1.3203,
+      "theoretical_loss": 3.42249260791323,
+      "tokens_seen": 2276392960
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003132723479377307,
+      "loss": 1.2623,
+      "theoretical_loss": 3.4224767260958595,
+      "tokens_seen": 2276524032
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031323222596693954,
+      "loss": 1.3195,
+      "theoretical_loss": 3.422460845448882,
+      "tokens_seen": 2276655104
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003131921039961483,
+      "loss": 1.1767,
+      "theoretical_loss": 3.4224449659721445,
+      "tokens_seen": 2276786176
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003131519820253571,
+      "loss": 1.3408,
+      "theoretical_loss": 3.4224290876654937,
+      "tokens_seen": 2276917248
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003131118600545659,
+      "loss": 1.236,
+      "theoretical_loss": 3.422413210528775,
+      "tokens_seen": 2277048320
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003130717380837747,
+      "loss": 1.2799,
+      "theoretical_loss": 3.4223973345618353,
+      "tokens_seen": 2277179392
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031303161611298346,
+      "loss": 1.221,
+      "theoretical_loss": 3.4223814597645212,
+      "tokens_seen": 2277310464
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031299149414219226,
+      "loss": 1.2008,
+      "theoretical_loss": 3.422365586136679,
+      "tokens_seen": 2277441536
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031295137217140107,
+      "loss": 1.2452,
+      "theoretical_loss": 3.4223497136781558,
+      "tokens_seen": 2277572608
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003129112502006098,
+      "loss": 1.2605,
+      "theoretical_loss": 3.422333842388797,
+      "tokens_seen": 2277703680
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003128711282298187,
+      "loss": 1.1957,
+      "theoretical_loss": 3.4223179722684507,
+      "tokens_seen": 2277834752
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031283100625902744,
+      "loss": 1.1924,
+      "theoretical_loss": 3.4223021033169627,
+      "tokens_seen": 2277965824
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031279088428823624,
+      "loss": 1.2186,
+      "theoretical_loss": 3.4222862355341794,
+      "tokens_seen": 2278096896
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031275076231744505,
+      "loss": 1.2821,
+      "theoretical_loss": 3.4222703689199485,
+      "tokens_seen": 2278227968
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003127106403466538,
+      "loss": 1.1781,
+      "theoretical_loss": 3.4222545034741154,
+      "tokens_seen": 2278359040
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003126705183758626,
+      "loss": 1.225,
+      "theoretical_loss": 3.4222386391965283,
+      "tokens_seen": 2278490112
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003126303964050714,
+      "loss": 1.2973,
+      "theoretical_loss": 3.422222776087033,
+      "tokens_seen": 2278621184
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003125902744342802,
+      "loss": 1.2723,
+      "theoretical_loss": 3.422206914145476,
+      "tokens_seen": 2278752256
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.000312550152463489,
+      "loss": 1.2558,
+      "theoretical_loss": 3.422191053371705,
+      "tokens_seen": 2278883328
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031251003049269783,
+      "loss": 1.3006,
+      "theoretical_loss": 3.422175193765567,
+      "tokens_seen": 2279014400
+    },
+    {
+      "epoch": 0.38,
+      "objective/train/docs_used": 765727,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2588951587677002,
+      "objective/train/theoretical_loss": 3.4221593353269077,
+      "objective/train/tokens_used": 649604576,
+      "theoretical_loss": 3.4221593353269077,
+      "tokens_seen": 2279145472
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003124699085219066,
+      "loss": 1.2761,
+      "theoretical_loss": 3.4221593353269077,
+      "tokens_seen": 2279145472
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031242978655111544,
+      "loss": 1.2675,
+      "theoretical_loss": 3.422143478055575,
+      "tokens_seen": 2279276544
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003123896645803242,
+      "loss": 1.2831,
+      "theoretical_loss": 3.422127621951416,
+      "tokens_seen": 2279407616
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031234954260953295,
+      "loss": 1.1775,
+      "theoretical_loss": 3.422111767014277,
+      "tokens_seen": 2279538688
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003123094206387418,
+      "loss": 1.231,
+      "theoretical_loss": 3.422095913244005,
+      "tokens_seen": 2279669760
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031226929866795056,
+      "loss": 1.2887,
+      "theoretical_loss": 3.4220800606404476,
+      "tokens_seen": 2279800832
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031222917669715937,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4220642092034517,
+      "tokens_seen": 2279931904
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003121890547263682,
+      "loss": 1.3433,
+      "theoretical_loss": 3.422048358932864,
+      "tokens_seen": 2280062976
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.000312148932755577,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4220325098285325,
+      "tokens_seen": 2280194048
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031210881078478573,
+      "loss": 1.271,
+      "theoretical_loss": 3.4220166618903036,
+      "tokens_seen": 2280325120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031206868881399454,
+      "loss": 1.255,
+      "theoretical_loss": 3.4220008151180243,
+      "tokens_seen": 2280456192
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031202856684320335,
+      "loss": 1.277,
+      "theoretical_loss": 3.4219849695115427,
+      "tokens_seen": 2280587264
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031198844487241215,
+      "loss": 1.2993,
+      "theoretical_loss": 3.421969125070705,
+      "tokens_seen": 2280718336
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031194832290162096,
+      "loss": 1.2432,
+      "theoretical_loss": 3.4219532817953597,
+      "tokens_seen": 2280849408
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003119082009308297,
+      "loss": 1.2038,
+      "theoretical_loss": 3.421937439685353,
+      "tokens_seen": 2280980480
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031186807896003857,
+      "loss": 1.241,
+      "theoretical_loss": 3.4219215987405325,
+      "tokens_seen": 2281111552
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003118279569892473,
+      "loss": 1.2519,
+      "theoretical_loss": 3.421905758960746,
+      "tokens_seen": 2281242624
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003117878350184561,
+      "loss": 1.1797,
+      "theoretical_loss": 3.42188992034584,
+      "tokens_seen": 2281373696
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031174771304766494,
+      "loss": 1.1591,
+      "theoretical_loss": 3.421874082895663,
+      "tokens_seen": 2281504768
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003117075910768737,
+      "loss": 1.3045,
+      "theoretical_loss": 3.421858246610062,
+      "tokens_seen": 2281635840
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003116674691060825,
+      "loss": 1.2348,
+      "theoretical_loss": 3.421842411488884,
+      "tokens_seen": 2281766912
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003116273471352913,
+      "loss": 1.3049,
+      "theoretical_loss": 3.421826577531977,
+      "tokens_seen": 2281897984
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003115872251645001,
+      "loss": 1.1635,
+      "theoretical_loss": 3.421810744739189,
+      "tokens_seen": 2282029056
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031154710319370886,
+      "loss": 1.1977,
+      "theoretical_loss": 3.421794913110366,
+      "tokens_seen": 2282160128
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031150698122291766,
+      "loss": 1.205,
+      "theoretical_loss": 3.421779082645357,
+      "tokens_seen": 2282291200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031146685925212647,
+      "loss": 1.2106,
+      "theoretical_loss": 3.4217632533440097,
+      "tokens_seen": 2282422272
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003114267372813352,
+      "loss": 1.2918,
+      "theoretical_loss": 3.4217474252061706,
+      "tokens_seen": 2282553344
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003113866153105441,
+      "loss": 1.2775,
+      "theoretical_loss": 3.4217315982316885,
+      "tokens_seen": 2282684416
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031134649333975284,
+      "loss": 1.2593,
+      "theoretical_loss": 3.42171577242041,
+      "tokens_seen": 2282815488
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003113063713689617,
+      "loss": 1.2379,
+      "theoretical_loss": 3.421699947772184,
+      "tokens_seen": 2282946560
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031126624939817045,
+      "loss": 1.2473,
+      "theoretical_loss": 3.421684124286857,
+      "tokens_seen": 2283077632
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003112261274273792,
+      "loss": 1.2326,
+      "theoretical_loss": 3.4216683019642784,
+      "tokens_seen": 2283208704
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031118600545658806,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4216524808042945,
+      "tokens_seen": 2283339776
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003111458834857968,
+      "loss": 1.2074,
+      "theoretical_loss": 3.421636660806754,
+      "tokens_seen": 2283470848
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003111057615150056,
+      "loss": 1.224,
+      "theoretical_loss": 3.4216208419715044,
+      "tokens_seen": 2283601920
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003110656395442144,
+      "loss": 1.1462,
+      "theoretical_loss": 3.421605024298394,
+      "tokens_seen": 2283732992
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031102551757342323,
+      "loss": 1.206,
+      "theoretical_loss": 3.42158920778727,
+      "tokens_seen": 2283864064
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.000310985395602632,
+      "loss": 1.1819,
+      "theoretical_loss": 3.421573392437981,
+      "tokens_seen": 2283995136
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003109452736318408,
+      "loss": 1.2169,
+      "theoretical_loss": 3.4215575782503747,
+      "tokens_seen": 2284126208
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003109051516610496,
+      "loss": 1.2346,
+      "theoretical_loss": 3.4215417652242994,
+      "tokens_seen": 2284257280
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031086502969025835,
+      "loss": 1.2419,
+      "theoretical_loss": 3.4215259533596027,
+      "tokens_seen": 2284388352
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003108249077194672,
+      "loss": 1.2671,
+      "theoretical_loss": 3.4215101426561336,
+      "tokens_seen": 2284519424
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031078478574867596,
+      "loss": 1.1758,
+      "theoretical_loss": 3.421494333113739,
+      "tokens_seen": 2284650496
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003107446637778848,
+      "loss": 1.1807,
+      "theoretical_loss": 3.4214785247322674,
+      "tokens_seen": 2284781568
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003107045418070936,
+      "loss": 1.2711,
+      "theoretical_loss": 3.4214627175115675,
+      "tokens_seen": 2284912640
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003106644198363024,
+      "loss": 1.3175,
+      "theoretical_loss": 3.4214469114514867,
+      "tokens_seen": 2285043712
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003106242978655112,
+      "loss": 1.1664,
+      "theoretical_loss": 3.421431106551874,
+      "tokens_seen": 2285174784
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031058417589471994,
+      "loss": 1.2281,
+      "theoretical_loss": 3.4214153028125773,
+      "tokens_seen": 2285305856
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031054405392392874,
+      "loss": 1.2299,
+      "theoretical_loss": 3.4213995002334445,
+      "tokens_seen": 2285436928
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031050393195313755,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4213836988143247,
+      "tokens_seen": 2285568000
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 767869,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1636757850646973,
+      "objective/train/theoretical_loss": 3.4213678985550655,
+      "objective/train/tokens_used": 656158176,
+      "theoretical_loss": 3.4213678985550655,
+      "tokens_seen": 2285699072
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031046380998234636,
+      "loss": 1.2419,
+      "theoretical_loss": 3.4213678985550655,
+      "tokens_seen": 2285699072
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003104236880115551,
+      "loss": 1.2967,
+      "theoretical_loss": 3.4213520994555155,
+      "tokens_seen": 2285830144
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031038356604076397,
+      "loss": 1.266,
+      "theoretical_loss": 3.4213363015155234,
+      "tokens_seen": 2285961216
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003103434440699727,
+      "loss": 1.2314,
+      "theoretical_loss": 3.421320504734937,
+      "tokens_seen": 2286092288
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003103033220991815,
+      "loss": 1.2444,
+      "theoretical_loss": 3.4213047091136053,
+      "tokens_seen": 2286223360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031026320012839033,
+      "loss": 1.2261,
+      "theoretical_loss": 3.4212889146513765,
+      "tokens_seen": 2286354432
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003102230781575991,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4212731213480994,
+      "tokens_seen": 2286485504
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003101829561868079,
+      "loss": 1.1399,
+      "theoretical_loss": 3.421257329203622,
+      "tokens_seen": 2286616576
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003101428342160167,
+      "loss": 1.2584,
+      "theoretical_loss": 3.421241538217793,
+      "tokens_seen": 2286747648
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003101027122452255,
+      "loss": 1.1973,
+      "theoretical_loss": 3.421225748390462,
+      "tokens_seen": 2286878720
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003100625902744343,
+      "loss": 1.2213,
+      "theoretical_loss": 3.421209959721476,
+      "tokens_seen": 2287009792
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031002246830364306,
+      "loss": 1.2097,
+      "theoretical_loss": 3.4211941722106847,
+      "tokens_seen": 2287140864
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030998234633285187,
+      "loss": 1.2991,
+      "theoretical_loss": 3.421178385857936,
+      "tokens_seen": 2287271936
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003099422243620607,
+      "loss": 1.3415,
+      "theoretical_loss": 3.4211626006630795,
+      "tokens_seen": 2287403008
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003099021023912695,
+      "loss": 1.202,
+      "theoretical_loss": 3.4211468166259635,
+      "tokens_seen": 2287534080
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030986198042047824,
+      "loss": 1.2,
+      "theoretical_loss": 3.4211310337464367,
+      "tokens_seen": 2287665152
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003098218584496871,
+      "loss": 1.2523,
+      "theoretical_loss": 3.421115252024348,
+      "tokens_seen": 2287796224
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030978173647889585,
+      "loss": 1.2427,
+      "theoretical_loss": 3.421099471459546,
+      "tokens_seen": 2287927296
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003097416145081046,
+      "loss": 1.2286,
+      "theoretical_loss": 3.42108369205188,
+      "tokens_seen": 2288058368
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030970149253731346,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4210679138011986,
+      "tokens_seen": 2288189440
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003096613705665222,
+      "loss": 1.2654,
+      "theoretical_loss": 3.4210521367073503,
+      "tokens_seen": 2288320512
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000309621248595731,
+      "loss": 1.2108,
+      "theoretical_loss": 3.421036360770185,
+      "tokens_seen": 2288451584
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003095811266249398,
+      "loss": 1.2567,
+      "theoretical_loss": 3.4210205859895506,
+      "tokens_seen": 2288582656
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030954100465414863,
+      "loss": 1.1791,
+      "theoretical_loss": 3.4210048123652963,
+      "tokens_seen": 2288713728
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003095008826833574,
+      "loss": 1.2271,
+      "theoretical_loss": 3.4209890398972718,
+      "tokens_seen": 2288844800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003094607607125662,
+      "loss": 1.3054,
+      "theoretical_loss": 3.4209732685853256,
+      "tokens_seen": 2288975872
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000309420638741775,
+      "loss": 1.2702,
+      "theoretical_loss": 3.420957498429307,
+      "tokens_seen": 2289106944
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003093805167709838,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4209417294290647,
+      "tokens_seen": 2289238016
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003093403948001926,
+      "loss": 1.1891,
+      "theoretical_loss": 3.4209259615844485,
+      "tokens_seen": 2289369088
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030930027282940136,
+      "loss": 1.2829,
+      "theoretical_loss": 3.4209101948953067,
+      "tokens_seen": 2289500160
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003092601508586102,
+      "loss": 1.246,
+      "theoretical_loss": 3.420894429361489,
+      "tokens_seen": 2289631232
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000309220028887819,
+      "loss": 1.2291,
+      "theoretical_loss": 3.420878664982845,
+      "tokens_seen": 2289762304
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003091799069170278,
+      "loss": 1.2514,
+      "theoretical_loss": 3.4208629017592234,
+      "tokens_seen": 2289893376
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003091397849462366,
+      "loss": 1.185,
+      "theoretical_loss": 3.420847139690473,
+      "tokens_seen": 2290024448
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030909966297544534,
+      "loss": 1.248,
+      "theoretical_loss": 3.420831378776444,
+      "tokens_seen": 2290155520
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030905954100465414,
+      "loss": 1.2636,
+      "theoretical_loss": 3.4208156190169854,
+      "tokens_seen": 2290286592
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030901941903386295,
+      "loss": 1.2546,
+      "theoretical_loss": 3.4207998604119467,
+      "tokens_seen": 2290417664
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030897929706307176,
+      "loss": 1.2105,
+      "theoretical_loss": 3.4207841029611767,
+      "tokens_seen": 2290548736
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003089391750922805,
+      "loss": 1.294,
+      "theoretical_loss": 3.420768346664526,
+      "tokens_seen": 2290679808
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030889905312148937,
+      "loss": 1.245,
+      "theoretical_loss": 3.4207525915218424,
+      "tokens_seen": 2290810880
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003088589311506981,
+      "loss": 1.247,
+      "theoretical_loss": 3.420736837532977,
+      "tokens_seen": 2290941952
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030881880917990693,
+      "loss": 1.2042,
+      "theoretical_loss": 3.420721084697778,
+      "tokens_seen": 2291073024
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030877868720911573,
+      "loss": 1.256,
+      "theoretical_loss": 3.4207053330160955,
+      "tokens_seen": 2291204096
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003087385652383245,
+      "loss": 1.3035,
+      "theoretical_loss": 3.420689582487779,
+      "tokens_seen": 2291335168
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030869844326753335,
+      "loss": 1.2397,
+      "theoretical_loss": 3.4206738331126783,
+      "tokens_seen": 2291466240
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003086583212967421,
+      "loss": 1.2662,
+      "theoretical_loss": 3.4206580848906425,
+      "tokens_seen": 2291597312
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003086181993259509,
+      "loss": 1.2449,
+      "theoretical_loss": 3.420642337821522,
+      "tokens_seen": 2291728384
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003085780773551597,
+      "loss": 1.2725,
+      "theoretical_loss": 3.4206265919051657,
+      "tokens_seen": 2291859456
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030853795538436846,
+      "loss": 1.2201,
+      "theoretical_loss": 3.4206108471414236,
+      "tokens_seen": 2291990528
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030849783341357727,
+      "loss": 1.2284,
+      "theoretical_loss": 3.420595103530145,
+      "tokens_seen": 2292121600
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 770014,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.290283441543579,
+      "objective/train/theoretical_loss": 3.4205793610711805,
+      "objective/train/tokens_used": 662711776,
+      "theoretical_loss": 3.4205793610711805,
+      "tokens_seen": 2292252672
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003084577114427861,
+      "loss": 1.3179,
+      "theoretical_loss": 3.4205793610711805,
+      "tokens_seen": 2292252672
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003084175894719949,
+      "loss": 1.2784,
+      "theoretical_loss": 3.4205636197643794,
+      "tokens_seen": 2292383744
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030837746750120363,
+      "loss": 1.269,
+      "theoretical_loss": 3.4205478796095914,
+      "tokens_seen": 2292514816
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003083373455304125,
+      "loss": 1.2617,
+      "theoretical_loss": 3.4205321406066664,
+      "tokens_seen": 2292645888
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030829722355962125,
+      "loss": 1.184,
+      "theoretical_loss": 3.420516402755455,
+      "tokens_seen": 2292776960
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030825710158883,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4205006660558057,
+      "tokens_seen": 2292908032
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030821697961803886,
+      "loss": 1.2024,
+      "theoretical_loss": 3.420484930507569,
+      "tokens_seen": 2293039104
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003081768576472476,
+      "loss": 1.2719,
+      "theoretical_loss": 3.420469196110595,
+      "tokens_seen": 2293170176
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030813673567645647,
+      "loss": 1.2687,
+      "theoretical_loss": 3.4204534628647343,
+      "tokens_seen": 2293301248
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003080966137056652,
+      "loss": 1.2554,
+      "theoretical_loss": 3.420437730769836,
+      "tokens_seen": 2293432320
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030805649173487403,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4204219998257503,
+      "tokens_seen": 2293563392
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030801636976408284,
+      "loss": 1.2382,
+      "theoretical_loss": 3.420406270032327,
+      "tokens_seen": 2293694464
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003079762477932916,
+      "loss": 1.2257,
+      "theoretical_loss": 3.420390541389417,
+      "tokens_seen": 2293825536
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003079361258225004,
+      "loss": 1.2103,
+      "theoretical_loss": 3.4203748138968697,
+      "tokens_seen": 2293956608
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003078960038517092,
+      "loss": 1.2162,
+      "theoretical_loss": 3.4203590875545355,
+      "tokens_seen": 2294087680
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000307855881880918,
+      "loss": 1.232,
+      "theoretical_loss": 3.420343362362265,
+      "tokens_seen": 2294218752
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030781575991012676,
+      "loss": 1.2031,
+      "theoretical_loss": 3.4203276383199075,
+      "tokens_seen": 2294349824
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003077756379393356,
+      "loss": 1.2204,
+      "theoretical_loss": 3.420311915427314,
+      "tokens_seen": 2294480896
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003077355159685444,
+      "loss": 1.2132,
+      "theoretical_loss": 3.420296193684334,
+      "tokens_seen": 2294611968
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003076953939977532,
+      "loss": 1.22,
+      "theoretical_loss": 3.4202804730908185,
+      "tokens_seen": 2294743040
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000307655272026962,
+      "loss": 1.2416,
+      "theoretical_loss": 3.4202647536466175,
+      "tokens_seen": 2294874112
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030761515005617074,
+      "loss": 1.1873,
+      "theoretical_loss": 3.4202490353515813,
+      "tokens_seen": 2295005184
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003075750280853796,
+      "loss": 1.2231,
+      "theoretical_loss": 3.42023331820556,
+      "tokens_seen": 2295136256
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030753490611458835,
+      "loss": 1.2354,
+      "theoretical_loss": 3.420217602208405,
+      "tokens_seen": 2295267328
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030749478414379716,
+      "loss": 1.1753,
+      "theoretical_loss": 3.420201887359966,
+      "tokens_seen": 2295398400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030745466217300596,
+      "loss": 1.2732,
+      "theoretical_loss": 3.420186173660093,
+      "tokens_seen": 2295529472
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030741454020221477,
+      "loss": 1.1583,
+      "theoretical_loss": 3.420170461108637,
+      "tokens_seen": 2295660544
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003073744182314235,
+      "loss": 1.2499,
+      "theoretical_loss": 3.420154749705449,
+      "tokens_seen": 2295791616
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030733429626063233,
+      "loss": 1.1715,
+      "theoretical_loss": 3.420139039450379,
+      "tokens_seen": 2295922688
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030729417428984113,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4201233303432774,
+      "tokens_seen": 2296053760
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003072540523190499,
+      "loss": 1.2427,
+      "theoretical_loss": 3.420107622383995,
+      "tokens_seen": 2296184832
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030721393034825875,
+      "loss": 1.2497,
+      "theoretical_loss": 3.420091915572382,
+      "tokens_seen": 2296315904
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003071738083774675,
+      "loss": 1.3123,
+      "theoretical_loss": 3.4200762099082898,
+      "tokens_seen": 2296446976
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003071336864066763,
+      "loss": 1.2345,
+      "theoretical_loss": 3.420060505391569,
+      "tokens_seen": 2296578048
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003070935644358851,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4200448020220695,
+      "tokens_seen": 2296709120
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030705344246509386,
+      "loss": 1.2624,
+      "theoretical_loss": 3.4200290997996428,
+      "tokens_seen": 2296840192
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030701332049430267,
+      "loss": 1.1837,
+      "theoretical_loss": 3.4200133987241395,
+      "tokens_seen": 2296971264
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003069731985235115,
+      "loss": 1.2682,
+      "theoretical_loss": 3.41999769879541,
+      "tokens_seen": 2297102336
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003069330765527203,
+      "loss": 1.2304,
+      "theoretical_loss": 3.419982000013306,
+      "tokens_seen": 2297233408
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003068929545819291,
+      "loss": 1.2214,
+      "theoretical_loss": 3.419966302377677,
+      "tokens_seen": 2297364480
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003068528326111379,
+      "loss": 1.2298,
+      "theoretical_loss": 3.419950605888375,
+      "tokens_seen": 2297495552
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030681271064034665,
+      "loss": 1.1918,
+      "theoretical_loss": 3.419934910545251,
+      "tokens_seen": 2297626624
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030677258866955545,
+      "loss": 1.203,
+      "theoretical_loss": 3.4199192163481547,
+      "tokens_seen": 2297757696
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030673246669876426,
+      "loss": 1.2743,
+      "theoretical_loss": 3.419903523296938,
+      "tokens_seen": 2297888768
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000306692344727973,
+      "loss": 1.2956,
+      "theoretical_loss": 3.419887831391452,
+      "tokens_seen": 2298019840
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030665222275718187,
+      "loss": 1.1846,
+      "theoretical_loss": 3.419872140631547,
+      "tokens_seen": 2298150912
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003066121007863906,
+      "loss": 1.1659,
+      "theoretical_loss": 3.4198564510170746,
+      "tokens_seen": 2298281984
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030657197881559943,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4198407625478855,
+      "tokens_seen": 2298413056
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030653185684480824,
+      "loss": 1.3229,
+      "theoretical_loss": 3.419825075223831,
+      "tokens_seen": 2298544128
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000306491734874017,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4198093890447625,
+      "tokens_seen": 2298675200
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 772248,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2339587211608887,
+      "objective/train/theoretical_loss": 3.419793704010531,
+      "objective/train/tokens_used": 669265376,
+      "theoretical_loss": 3.419793704010531,
+      "tokens_seen": 2298806272
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003064516129032258,
+      "loss": 1.1847,
+      "theoretical_loss": 3.419793704010531,
+      "tokens_seen": 2298806272
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003064114909324346,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4197780201209875,
+      "tokens_seen": 2298937344
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003063713689616434,
+      "loss": 1.2012,
+      "theoretical_loss": 3.4197623373759827,
+      "tokens_seen": 2299068416
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030633124699085216,
+      "loss": 1.2528,
+      "theoretical_loss": 3.419746655775369,
+      "tokens_seen": 2299199488
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000306291125020061,
+      "loss": 1.3133,
+      "theoretical_loss": 3.4197309753189966,
+      "tokens_seen": 2299330560
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030625100304926977,
+      "loss": 1.1973,
+      "theoretical_loss": 3.419715296006718,
+      "tokens_seen": 2299461632
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003062108810784786,
+      "loss": 1.2161,
+      "theoretical_loss": 3.4196996178383827,
+      "tokens_seen": 2299592704
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003061707591076874,
+      "loss": 1.2142,
+      "theoretical_loss": 3.4196839408138437,
+      "tokens_seen": 2299723776
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030613063713689614,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4196682649329517,
+      "tokens_seen": 2299854848
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000306090515166105,
+      "loss": 1.2466,
+      "theoretical_loss": 3.419652590195558,
+      "tokens_seen": 2299985920
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030605039319531375,
+      "loss": 1.1863,
+      "theoretical_loss": 3.4196369166015144,
+      "tokens_seen": 2300116992
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030601027122452256,
+      "loss": 1.2432,
+      "theoretical_loss": 3.4196212441506715,
+      "tokens_seen": 2300248064
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030597014925373136,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4196055728428822,
+      "tokens_seen": 2300379136
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030593002728294017,
+      "loss": 1.243,
+      "theoretical_loss": 3.419589902677997,
+      "tokens_seen": 2300510208
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003058899053121489,
+      "loss": 1.3147,
+      "theoretical_loss": 3.4195742336558674,
+      "tokens_seen": 2300641280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030584978334135773,
+      "loss": 1.2281,
+      "theoretical_loss": 3.4195585657763456,
+      "tokens_seen": 2300772352
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030580966137056653,
+      "loss": 1.1853,
+      "theoretical_loss": 3.419542899039283,
+      "tokens_seen": 2300903424
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003057695393997753,
+      "loss": 1.2627,
+      "theoretical_loss": 3.419527233444531,
+      "tokens_seen": 2301034496
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030572941742898415,
+      "loss": 1.2514,
+      "theoretical_loss": 3.4195115689919406,
+      "tokens_seen": 2301165568
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003056892954581929,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4194959056813645,
+      "tokens_seen": 2301296640
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030564917348740176,
+      "loss": 1.2062,
+      "theoretical_loss": 3.419480243512654,
+      "tokens_seen": 2301427712
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003056090515166105,
+      "loss": 1.3021,
+      "theoretical_loss": 3.4194645824856615,
+      "tokens_seen": 2301558784
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030556892954581926,
+      "loss": 1.2309,
+      "theoretical_loss": 3.4194489226002376,
+      "tokens_seen": 2301689856
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003055288075750281,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4194332638562352,
+      "tokens_seen": 2301820928
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003054886856042369,
+      "loss": 1.2536,
+      "theoretical_loss": 3.4194176062535053,
+      "tokens_seen": 2301952000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003054485636334457,
+      "loss": 1.3297,
+      "theoretical_loss": 3.4194019497919,
+      "tokens_seen": 2302083072
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003054084416626545,
+      "loss": 1.2539,
+      "theoretical_loss": 3.4193862944712707,
+      "tokens_seen": 2302214144
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003053683196918633,
+      "loss": 1.264,
+      "theoretical_loss": 3.4193706402914703,
+      "tokens_seen": 2302345216
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030532819772107205,
+      "loss": 1.2927,
+      "theoretical_loss": 3.4193549872523503,
+      "tokens_seen": 2302476288
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030528807575028085,
+      "loss": 1.2577,
+      "theoretical_loss": 3.4193393353537624,
+      "tokens_seen": 2302607360
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030524795377948966,
+      "loss": 1.2337,
+      "theoretical_loss": 3.4193236845955592,
+      "tokens_seen": 2302738432
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003052078318086984,
+      "loss": 1.2344,
+      "theoretical_loss": 3.4193080349775915,
+      "tokens_seen": 2302869504
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030516770983790727,
+      "loss": 1.2981,
+      "theoretical_loss": 3.4192923864997127,
+      "tokens_seen": 2303000576
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000305127587867116,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4192767391617735,
+      "tokens_seen": 2303131648
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030508746589632483,
+      "loss": 1.2564,
+      "theoretical_loss": 3.4192610929636276,
+      "tokens_seen": 2303262720
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030504734392553364,
+      "loss": 1.2628,
+      "theoretical_loss": 3.419245447905126,
+      "tokens_seen": 2303393792
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003050072219547424,
+      "loss": 1.2492,
+      "theoretical_loss": 3.419229803986121,
+      "tokens_seen": 2303524864
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030496709998395125,
+      "loss": 1.2906,
+      "theoretical_loss": 3.4192141612064653,
+      "tokens_seen": 2303655936
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030492697801316,
+      "loss": 1.2539,
+      "theoretical_loss": 3.4191985195660104,
+      "tokens_seen": 2303787008
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003048868560423688,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4191828790646084,
+      "tokens_seen": 2303918080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003048467340715776,
+      "loss": 1.1993,
+      "theoretical_loss": 3.4191672397021122,
+      "tokens_seen": 2304049152
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003048066121007864,
+      "loss": 1.1146,
+      "theoretical_loss": 3.4191516014783745,
+      "tokens_seen": 2304180224
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030476649012999517,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4191359643932464,
+      "tokens_seen": 2304311296
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000304726368159204,
+      "loss": 1.3035,
+      "theoretical_loss": 3.419120328446581,
+      "tokens_seen": 2304442368
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003046862461884128,
+      "loss": 1.2057,
+      "theoretical_loss": 3.4191046936382303,
+      "tokens_seen": 2304573440
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030464612421762154,
+      "loss": 1.238,
+      "theoretical_loss": 3.419089059968047,
+      "tokens_seen": 2304704512
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003046060022468304,
+      "loss": 1.2598,
+      "theoretical_loss": 3.4190734274358836,
+      "tokens_seen": 2304835584
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030456588027603915,
+      "loss": 1.2415,
+      "theoretical_loss": 3.419057796041592,
+      "tokens_seen": 2304966656
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030452575830524796,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4190421657850254,
+      "tokens_seen": 2305097728
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030448563633445676,
+      "loss": 1.2426,
+      "theoretical_loss": 3.4190265366660357,
+      "tokens_seen": 2305228800
+    },
+    {
+      "debugging/Compilability": 0.0,
+      "debugging/distinct-1-grams": 0.6824214379946716,
+      "debugging/entropy-1-grams": 5.290924378994747,
+      "debugging/length": 550.125,
+      "debugging/num_segments": 16,
+      "debugging/score": 0.059994086139135025,
+      "debugging/score_std": 0.019143948003452343,
+      "epoch": 0.4,
+      "objective/train/docs_used": 774449,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2741005420684814,
+      "objective/train/theoretical_loss": 3.4190109086844758,
+      "objective/train/tokens_used": 675818976,
+      "theoretical_loss": 3.4190109086844758,
+      "tokens_seen": 2305359872
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030444551436366557,
+      "loss": 1.213,
+      "theoretical_loss": 3.4190109086844758,
+      "tokens_seen": 2305359872
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003044053923928743,
+      "loss": 1.2825,
+      "theoretical_loss": 3.418995281840198,
+      "tokens_seen": 2305490944
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003043652704220831,
+      "loss": 1.2121,
+      "theoretical_loss": 3.4189796561330548,
+      "tokens_seen": 2305622016
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030432514845129193,
+      "loss": 1.2444,
+      "theoretical_loss": 3.4189640315628997,
+      "tokens_seen": 2305753088
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030428502648050074,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4189484081295842,
+      "tokens_seen": 2305884160
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030424490450970955,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4189327858329612,
+      "tokens_seen": 2306015232
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003042047825389183,
+      "loss": 1.227,
+      "theoretical_loss": 3.418917164672884,
+      "tokens_seen": 2306146304
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030416466056812716,
+      "loss": 1.216,
+      "theoretical_loss": 3.4189015446492053,
+      "tokens_seen": 2306277376
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003041245385973359,
+      "loss": 1.1913,
+      "theoretical_loss": 3.418885925761777,
+      "tokens_seen": 2306408448
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030408441662654466,
+      "loss": 1.191,
+      "theoretical_loss": 3.4188703080104528,
+      "tokens_seen": 2306539520
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003040442946557535,
+      "loss": 1.2163,
+      "theoretical_loss": 3.418854691395085,
+      "tokens_seen": 2306670592
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003040041726849623,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4188390759155265,
+      "tokens_seen": 2306801664
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003039640507141711,
+      "loss": 1.1851,
+      "theoretical_loss": 3.41882346157163,
+      "tokens_seen": 2306932736
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003039239287433799,
+      "loss": 1.1571,
+      "theoretical_loss": 3.4188078483632487,
+      "tokens_seen": 2307063808
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003038838067725887,
+      "loss": 1.3042,
+      "theoretical_loss": 3.418792236290235,
+      "tokens_seen": 2307194880
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030384368480179745,
+      "loss": 1.2224,
+      "theoretical_loss": 3.418776625352443,
+      "tokens_seen": 2307325952
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030380356283100625,
+      "loss": 1.1562,
+      "theoretical_loss": 3.418761015549725,
+      "tokens_seen": 2307457024
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030376344086021506,
+      "loss": 1.1764,
+      "theoretical_loss": 3.4187454068819334,
+      "tokens_seen": 2307588096
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030372331888942387,
+      "loss": 1.2248,
+      "theoretical_loss": 3.418729799348922,
+      "tokens_seen": 2307719168
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030368319691863267,
+      "loss": 1.2355,
+      "theoretical_loss": 3.4187141929505436,
+      "tokens_seen": 2307850240
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003036430749478414,
+      "loss": 1.2352,
+      "theoretical_loss": 3.418698587686651,
+      "tokens_seen": 2307981312
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003036029529770503,
+      "loss": 1.2252,
+      "theoretical_loss": 3.418682983557098,
+      "tokens_seen": 2308112384
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030356283100625904,
+      "loss": 1.2701,
+      "theoretical_loss": 3.418667380561737,
+      "tokens_seen": 2308243456
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003035227090354678,
+      "loss": 1.3274,
+      "theoretical_loss": 3.418651778700422,
+      "tokens_seen": 2308374528
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030348258706467665,
+      "loss": 1.2009,
+      "theoretical_loss": 3.418636177973005,
+      "tokens_seen": 2308505600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003034424650938854,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4186205783793406,
+      "tokens_seen": 2308636672
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003034023431230942,
+      "loss": 1.2594,
+      "theoretical_loss": 3.418604979919281,
+      "tokens_seen": 2308767744
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000303362221152303,
+      "loss": 1.2605,
+      "theoretical_loss": 3.4185893825926796,
+      "tokens_seen": 2308898816
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003033220991815118,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4185737863993904,
+      "tokens_seen": 2309029888
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030328197721072057,
+      "loss": 1.2553,
+      "theoretical_loss": 3.418558191339266,
+      "tokens_seen": 2309160960
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003032418552399294,
+      "loss": 1.24,
+      "theoretical_loss": 3.41854259741216,
+      "tokens_seen": 2309292032
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003032017332691382,
+      "loss": 1.2991,
+      "theoretical_loss": 3.4185270046179257,
+      "tokens_seen": 2309423104
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030316161129834694,
+      "loss": 1.1927,
+      "theoretical_loss": 3.418511412956417,
+      "tokens_seen": 2309554176
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003031214893275558,
+      "loss": 1.2462,
+      "theoretical_loss": 3.4184958224274866,
+      "tokens_seen": 2309685248
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030308136735676455,
+      "loss": 1.3047,
+      "theoretical_loss": 3.4184802330309885,
+      "tokens_seen": 2309816320
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003030412453859734,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4184646447667757,
+      "tokens_seen": 2309947392
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030300112341518216,
+      "loss": 1.2466,
+      "theoretical_loss": 3.418449057634702,
+      "tokens_seen": 2310078464
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030296100144439097,
+      "loss": 1.2782,
+      "theoretical_loss": 3.4184334716346214,
+      "tokens_seen": 2310209536
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003029208794735998,
+      "loss": 1.2901,
+      "theoretical_loss": 3.418417886766387,
+      "tokens_seen": 2310340608
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003028807575028085,
+      "loss": 1.2885,
+      "theoretical_loss": 3.418402303029852,
+      "tokens_seen": 2310471680
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030284063553201733,
+      "loss": 1.2333,
+      "theoretical_loss": 3.418386720424871,
+      "tokens_seen": 2310602752
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030280051356122614,
+      "loss": 1.2163,
+      "theoretical_loss": 3.4183711389512963,
+      "tokens_seen": 2310733824
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030276039159043495,
+      "loss": 1.2687,
+      "theoretical_loss": 3.418355558608983,
+      "tokens_seen": 2310864896
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003027202696196437,
+      "loss": 1.2857,
+      "theoretical_loss": 3.418339979397784,
+      "tokens_seen": 2310995968
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030268014764885256,
+      "loss": 1.1858,
+      "theoretical_loss": 3.418324401317553,
+      "tokens_seen": 2311127040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003026400256780613,
+      "loss": 1.2918,
+      "theoretical_loss": 3.4183088243681445,
+      "tokens_seen": 2311258112
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030259990370727006,
+      "loss": 1.2788,
+      "theoretical_loss": 3.4182932485494115,
+      "tokens_seen": 2311389184
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003025597817364789,
+      "loss": 1.1571,
+      "theoretical_loss": 3.4182776738612084,
+      "tokens_seen": 2311520256
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003025196597656877,
+      "loss": 1.3224,
+      "theoretical_loss": 3.4182621003033886,
+      "tokens_seen": 2311651328
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030247953779489654,
+      "loss": 1.1736,
+      "theoretical_loss": 3.418246527875806,
+      "tokens_seen": 2311782400
+    },
+    {
+      "epoch": 0.4,
+      "objective/train/docs_used": 776587,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2411270141601562,
+      "objective/train/theoretical_loss": 3.4182309565783147,
+      "objective/train/tokens_used": 682372576,
+      "theoretical_loss": 3.4182309565783147,
+      "tokens_seen": 2311913472
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003024394158241053,
+      "loss": 1.2307,
+      "theoretical_loss": 3.4182309565783147,
+      "tokens_seen": 2311913472
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003023992938533141,
+      "loss": 1.2594,
+      "theoretical_loss": 3.4182153864107687,
+      "tokens_seen": 2312044544
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003023591718825229,
+      "loss": 1.2965,
+      "theoretical_loss": 3.4181998173730217,
+      "tokens_seen": 2312175616
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030231904991173165,
+      "loss": 1.2169,
+      "theoretical_loss": 3.418184249464928,
+      "tokens_seen": 2312306688
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030227892794094046,
+      "loss": 1.1945,
+      "theoretical_loss": 3.418168682686341,
+      "tokens_seen": 2312437760
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030223880597014926,
+      "loss": 1.3126,
+      "theoretical_loss": 3.4181531170371153,
+      "tokens_seen": 2312568832
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030219868399935807,
+      "loss": 1.2671,
+      "theoretical_loss": 3.4181375525171047,
+      "tokens_seen": 2312699904
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003021585620285668,
+      "loss": 1.2785,
+      "theoretical_loss": 3.418121989126164,
+      "tokens_seen": 2312830976
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003021184400577757,
+      "loss": 1.2436,
+      "theoretical_loss": 3.4181064268641457,
+      "tokens_seen": 2312962048
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030207831808698444,
+      "loss": 1.2249,
+      "theoretical_loss": 3.418090865730906,
+      "tokens_seen": 2313093120
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003020381961161932,
+      "loss": 1.2463,
+      "theoretical_loss": 3.418075305726297,
+      "tokens_seen": 2313224192
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030199807414540205,
+      "loss": 1.2837,
+      "theoretical_loss": 3.4180597468501746,
+      "tokens_seen": 2313355264
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003019579521746108,
+      "loss": 1.2274,
+      "theoretical_loss": 3.418044189102392,
+      "tokens_seen": 2313486336
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003019178302038196,
+      "loss": 1.1948,
+      "theoretical_loss": 3.418028632482804,
+      "tokens_seen": 2313617408
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003018777082330284,
+      "loss": 1.2248,
+      "theoretical_loss": 3.4180130769912647,
+      "tokens_seen": 2313748480
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003018375862622372,
+      "loss": 1.2185,
+      "theoretical_loss": 3.417997522627628,
+      "tokens_seen": 2313879552
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000301797464291446,
+      "loss": 1.2009,
+      "theoretical_loss": 3.4179819693917493,
+      "tokens_seen": 2314010624
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003017573423206548,
+      "loss": 1.2233,
+      "theoretical_loss": 3.417966417283482,
+      "tokens_seen": 2314141696
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003017172203498636,
+      "loss": 1.2079,
+      "theoretical_loss": 3.4179508663026805,
+      "tokens_seen": 2314272768
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003016770983790724,
+      "loss": 1.2554,
+      "theoretical_loss": 3.4179353164491992,
+      "tokens_seen": 2314403840
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003016369764082812,
+      "loss": 1.2712,
+      "theoretical_loss": 3.4179197677228936,
+      "tokens_seen": 2314534912
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030159685443748995,
+      "loss": 1.1893,
+      "theoretical_loss": 3.4179042201236167,
+      "tokens_seen": 2314665984
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003015567324666988,
+      "loss": 1.2059,
+      "theoretical_loss": 3.417888673651224,
+      "tokens_seen": 2314797056
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030151661049590756,
+      "loss": 1.257,
+      "theoretical_loss": 3.41787312830557,
+      "tokens_seen": 2314928128
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003014764885251163,
+      "loss": 1.1675,
+      "theoretical_loss": 3.4178575840865086,
+      "tokens_seen": 2315059200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003014363665543252,
+      "loss": 1.2864,
+      "theoretical_loss": 3.4178420409938948,
+      "tokens_seen": 2315190272
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003013962445835339,
+      "loss": 1.2559,
+      "theoretical_loss": 3.4178264990275826,
+      "tokens_seen": 2315321344
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030135612261274273,
+      "loss": 1.2207,
+      "theoretical_loss": 3.417810958187428,
+      "tokens_seen": 2315452416
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030131600064195154,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4177954184732844,
+      "tokens_seen": 2315583488
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030127587867116035,
+      "loss": 1.2144,
+      "theoretical_loss": 3.417779879885007,
+      "tokens_seen": 2315714560
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003012357567003691,
+      "loss": 1.258,
+      "theoretical_loss": 3.4177643424224504,
+      "tokens_seen": 2315845632
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030119563472957796,
+      "loss": 1.2441,
+      "theoretical_loss": 3.4177488060854695,
+      "tokens_seen": 2315976704
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003011555127587867,
+      "loss": 1.2483,
+      "theoretical_loss": 3.4177332708739185,
+      "tokens_seen": 2316107776
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003011153907879955,
+      "loss": 1.1965,
+      "theoretical_loss": 3.417717736787653,
+      "tokens_seen": 2316238848
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003010752688172043,
+      "loss": 1.1917,
+      "theoretical_loss": 3.4177022038265275,
+      "tokens_seen": 2316369920
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003010351468464131,
+      "loss": 1.2282,
+      "theoretical_loss": 3.4176866719903964,
+      "tokens_seen": 2316500992
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030099502487562194,
+      "loss": 1.2517,
+      "theoretical_loss": 3.417671141279115,
+      "tokens_seen": 2316632064
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003009549029048307,
+      "loss": 1.206,
+      "theoretical_loss": 3.4176556116925383,
+      "tokens_seen": 2316763136
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003009147809340395,
+      "loss": 1.2006,
+      "theoretical_loss": 3.4176400832305216,
+      "tokens_seen": 2316894208
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003008746589632483,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4176245558929184,
+      "tokens_seen": 2317025280
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030083453699245705,
+      "loss": 1.175,
+      "theoretical_loss": 3.417609029679585,
+      "tokens_seen": 2317156352
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030079441502166586,
+      "loss": 1.2324,
+      "theoretical_loss": 3.4175935045903763,
+      "tokens_seen": 2317287424
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030075429305087466,
+      "loss": 1.2454,
+      "theoretical_loss": 3.417577980625147,
+      "tokens_seen": 2317418496
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030071417108008347,
+      "loss": 1.2228,
+      "theoretical_loss": 3.4175624577837516,
+      "tokens_seen": 2317549568
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003006740491092922,
+      "loss": 1.1832,
+      "theoretical_loss": 3.4175469360660466,
+      "tokens_seen": 2317680640
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003006339271385011,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4175314154718857,
+      "tokens_seen": 2317811712
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030059380516770984,
+      "loss": 1.2708,
+      "theoretical_loss": 3.417515896001125,
+      "tokens_seen": 2317942784
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003005536831969186,
+      "loss": 1.2014,
+      "theoretical_loss": 3.4175003776536195,
+      "tokens_seen": 2318073856
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030051356122612745,
+      "loss": 1.2253,
+      "theoretical_loss": 3.417484860429224,
+      "tokens_seen": 2318204928
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003004734392553362,
+      "loss": 1.3179,
+      "theoretical_loss": 3.417469344327794,
+      "tokens_seen": 2318336000
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 778698,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.186708688735962,
+      "objective/train/theoretical_loss": 3.4174538293491845,
+      "objective/train/tokens_used": 688926176,
+      "theoretical_loss": 3.4174538293491845,
+      "tokens_seen": 2318467072
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00030043331728454506,
+      "loss": 1.2308,
+      "theoretical_loss": 3.4174538293491845,
+      "tokens_seen": 2318467072
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003003931953137538,
+      "loss": 1.2529,
+      "theoretical_loss": 3.4174383154932513,
+      "tokens_seen": 2318598144
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003003530733429626,
+      "loss": 1.3065,
+      "theoretical_loss": 3.417422802759849,
+      "tokens_seen": 2318729216
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003003129513721714,
+      "loss": 1.2014,
+      "theoretical_loss": 3.417407291148834,
+      "tokens_seen": 2318860288
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003002728294013802,
+      "loss": 1.255,
+      "theoretical_loss": 3.4173917806600604,
+      "tokens_seen": 2318991360
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000300232707430589,
+      "loss": 1.2433,
+      "theoretical_loss": 3.417376271293384,
+      "tokens_seen": 2319122432
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003001925854597978,
+      "loss": 1.223,
+      "theoretical_loss": 3.417360763048661,
+      "tokens_seen": 2319253504
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003001524634890066,
+      "loss": 1.258,
+      "theoretical_loss": 3.4173452559257456,
+      "tokens_seen": 2319384576
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00030011234151821535,
+      "loss": 1.1713,
+      "theoretical_loss": 3.417329749924494,
+      "tokens_seen": 2319515648
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003000722195474242,
+      "loss": 1.2114,
+      "theoretical_loss": 3.417314245044762,
+      "tokens_seen": 2319646720
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00030003209757663296,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4172987412864044,
+      "tokens_seen": 2319777792
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002999919756058417,
+      "loss": 1.2505,
+      "theoretical_loss": 3.417283238649277,
+      "tokens_seen": 2319908864
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002999518536350506,
+      "loss": 1.2622,
+      "theoretical_loss": 3.417267737133235,
+      "tokens_seen": 2320039936
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002999117316642593,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4172522367381353,
+      "tokens_seen": 2320171008
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002998716096934682,
+      "loss": 1.2298,
+      "theoretical_loss": 3.417236737463832,
+      "tokens_seen": 2320302080
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029983148772267694,
+      "loss": 1.1789,
+      "theoretical_loss": 3.4172212393101815,
+      "tokens_seen": 2320433152
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029979136575188574,
+      "loss": 1.1908,
+      "theoretical_loss": 3.4172057422770394,
+      "tokens_seen": 2320564224
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029975124378109455,
+      "loss": 1.3086,
+      "theoretical_loss": 3.417190246364261,
+      "tokens_seen": 2320695296
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029971112181030336,
+      "loss": 1.2011,
+      "theoretical_loss": 3.417174751571703,
+      "tokens_seen": 2320826368
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002996709998395121,
+      "loss": 1.2059,
+      "theoretical_loss": 3.41715925789922,
+      "tokens_seen": 2320957440
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002996308778687209,
+      "loss": 1.2349,
+      "theoretical_loss": 3.4171437653466685,
+      "tokens_seen": 2321088512
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002995907558979297,
+      "loss": 1.2921,
+      "theoretical_loss": 3.4171282739139044,
+      "tokens_seen": 2321219584
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002995506339271385,
+      "loss": 1.2489,
+      "theoretical_loss": 3.417112783600783,
+      "tokens_seen": 2321350656
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029951051195634733,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4170972944071605,
+      "tokens_seen": 2321481728
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002994703899855561,
+      "loss": 1.2552,
+      "theoretical_loss": 3.417081806332893,
+      "tokens_seen": 2321612800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002994302680147649,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4170663193778354,
+      "tokens_seen": 2321743872
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002993901460439737,
+      "loss": 1.2267,
+      "theoretical_loss": 3.417050833541845,
+      "tokens_seen": 2321874944
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029935002407318245,
+      "loss": 1.2792,
+      "theoretical_loss": 3.4170353488247773,
+      "tokens_seen": 2322006016
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029930990210239126,
+      "loss": 1.2187,
+      "theoretical_loss": 3.417019865226488,
+      "tokens_seen": 2322137088
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029926978013160006,
+      "loss": 1.2091,
+      "theoretical_loss": 3.4170043827468333,
+      "tokens_seen": 2322268160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029922965816080887,
+      "loss": 1.2871,
+      "theoretical_loss": 3.416988901385669,
+      "tokens_seen": 2322399232
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002991895361900177,
+      "loss": 1.2956,
+      "theoretical_loss": 3.4169734211428517,
+      "tokens_seen": 2322530304
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002991494142192265,
+      "loss": 1.2208,
+      "theoretical_loss": 3.416957942018237,
+      "tokens_seen": 2322661376
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029910929224843524,
+      "loss": 1.2256,
+      "theoretical_loss": 3.4169424640116817,
+      "tokens_seen": 2322792448
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029906917027764404,
+      "loss": 1.1638,
+      "theoretical_loss": 3.416926987123041,
+      "tokens_seen": 2322923520
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029902904830685285,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4169115113521724,
+      "tokens_seen": 2323054592
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002989889263360616,
+      "loss": 1.216,
+      "theoretical_loss": 3.4168960366989305,
+      "tokens_seen": 2323185664
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029894880436527046,
+      "loss": 1.2505,
+      "theoretical_loss": 3.416880563163173,
+      "tokens_seen": 2323316736
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002989086823944792,
+      "loss": 1.191,
+      "theoretical_loss": 3.4168650907447553,
+      "tokens_seen": 2323447808
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000298868560423688,
+      "loss": 1.2468,
+      "theoretical_loss": 3.416849619443534,
+      "tokens_seen": 2323578880
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002988284384528968,
+      "loss": 1.2613,
+      "theoretical_loss": 3.4168341492593646,
+      "tokens_seen": 2323709952
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002987883164821056,
+      "loss": 1.2061,
+      "theoretical_loss": 3.416818680192105,
+      "tokens_seen": 2323841024
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002987481945113144,
+      "loss": 1.2771,
+      "theoretical_loss": 3.4168032122416108,
+      "tokens_seen": 2323972096
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002987080725405232,
+      "loss": 1.2386,
+      "theoretical_loss": 3.4167877454077376,
+      "tokens_seen": 2324103168
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000298667950569732,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4167722796903433,
+      "tokens_seen": 2324234240
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002986278285989408,
+      "loss": 1.1989,
+      "theoretical_loss": 3.4167568150892835,
+      "tokens_seen": 2324365312
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002985877066281496,
+      "loss": 1.241,
+      "theoretical_loss": 3.4167413516044145,
+      "tokens_seen": 2324496384
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029854758465735836,
+      "loss": 1.1313,
+      "theoretical_loss": 3.416725889235593,
+      "tokens_seen": 2324627456
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029850746268656717,
+      "loss": 1.2546,
+      "theoretical_loss": 3.416710427982676,
+      "tokens_seen": 2324758528
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000298467340715776,
+      "loss": 1.2139,
+      "theoretical_loss": 3.41669496784552,
+      "tokens_seen": 2324889600
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 780809,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1698287725448608,
+      "objective/train/theoretical_loss": 3.4166795088239805,
+      "objective/train/tokens_used": 695479776,
+      "theoretical_loss": 3.4166795088239805,
+      "tokens_seen": 2325020672
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002984272187449847,
+      "loss": 1.2453,
+      "theoretical_loss": 3.4166795088239805,
+      "tokens_seen": 2325020672
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002983870967741936,
+      "loss": 1.2229,
+      "theoretical_loss": 3.4166640509179156,
+      "tokens_seen": 2325151744
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029834697480340234,
+      "loss": 1.2898,
+      "theoretical_loss": 3.416648594127181,
+      "tokens_seen": 2325282816
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029830685283261114,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4166331384516333,
+      "tokens_seen": 2325413888
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029826673086181995,
+      "loss": 1.2885,
+      "theoretical_loss": 3.41661768389113,
+      "tokens_seen": 2325544960
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029822660889102876,
+      "loss": 1.2819,
+      "theoretical_loss": 3.4166022304455272,
+      "tokens_seen": 2325676032
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002981864869202375,
+      "loss": 1.2385,
+      "theoretical_loss": 3.4165867781146813,
+      "tokens_seen": 2325807104
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002981463649494463,
+      "loss": 1.2011,
+      "theoretical_loss": 3.41657132689845,
+      "tokens_seen": 2325938176
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002981062429786551,
+      "loss": 1.2362,
+      "theoretical_loss": 3.4165558767966893,
+      "tokens_seen": 2326069248
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002980661210078639,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4165404278092564,
+      "tokens_seen": 2326200320
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029802599903707273,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4165249799360082,
+      "tokens_seen": 2326331392
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002979858770662815,
+      "loss": 1.2827,
+      "theoretical_loss": 3.4165095331768014,
+      "tokens_seen": 2326462464
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029794575509549035,
+      "loss": 1.254,
+      "theoretical_loss": 3.416494087531493,
+      "tokens_seen": 2326593536
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002979056331246991,
+      "loss": 1.222,
+      "theoretical_loss": 3.4164786429999396,
+      "tokens_seen": 2326724608
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029786551115390785,
+      "loss": 1.2392,
+      "theoretical_loss": 3.4164631995819987,
+      "tokens_seen": 2326855680
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002978253891831167,
+      "loss": 1.2015,
+      "theoretical_loss": 3.4164477572775267,
+      "tokens_seen": 2326986752
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029778526721232546,
+      "loss": 1.26,
+      "theoretical_loss": 3.4164323160863814,
+      "tokens_seen": 2327117824
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029774514524153427,
+      "loss": 1.2804,
+      "theoretical_loss": 3.4164168760084195,
+      "tokens_seen": 2327248896
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002977050232707431,
+      "loss": 1.1399,
+      "theoretical_loss": 3.4164014370434974,
+      "tokens_seen": 2327379968
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002976649012999519,
+      "loss": 1.2561,
+      "theoretical_loss": 3.4163859991914727,
+      "tokens_seen": 2327511040
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029762477932916063,
+      "loss": 1.1809,
+      "theoretical_loss": 3.4163705624522027,
+      "tokens_seen": 2327642112
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029758465735836944,
+      "loss": 1.2002,
+      "theoretical_loss": 3.4163551268255445,
+      "tokens_seen": 2327773184
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029754453538757825,
+      "loss": 1.2466,
+      "theoretical_loss": 3.416339692311355,
+      "tokens_seen": 2327904256
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000297504413416787,
+      "loss": 1.3367,
+      "theoretical_loss": 3.4163242589094915,
+      "tokens_seen": 2328035328
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029746429144599586,
+      "loss": 1.2867,
+      "theoretical_loss": 3.416308826619811,
+      "tokens_seen": 2328166400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002974241694752046,
+      "loss": 1.2109,
+      "theoretical_loss": 3.4162933954421715,
+      "tokens_seen": 2328297472
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029738404750441347,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4162779653764295,
+      "tokens_seen": 2328428544
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002973439255336222,
+      "loss": 1.1812,
+      "theoretical_loss": 3.4162625364224426,
+      "tokens_seen": 2328559616
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000297303803562831,
+      "loss": 1.2267,
+      "theoretical_loss": 3.416247108580068,
+      "tokens_seen": 2328690688
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029726368159203984,
+      "loss": 1.2647,
+      "theoretical_loss": 3.416231681849163,
+      "tokens_seen": 2328821760
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002972235596212486,
+      "loss": 1.2768,
+      "theoretical_loss": 3.4162162562295855,
+      "tokens_seen": 2328952832
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002971834376504574,
+      "loss": 1.2086,
+      "theoretical_loss": 3.4162008317211923,
+      "tokens_seen": 2329083904
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002971433156796662,
+      "loss": 1.2995,
+      "theoretical_loss": 3.416185408323841,
+      "tokens_seen": 2329214976
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000297103193708875,
+      "loss": 1.1956,
+      "theoretical_loss": 3.416169986037389,
+      "tokens_seen": 2329346048
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029706307173808376,
+      "loss": 1.2078,
+      "theoretical_loss": 3.416154564861694,
+      "tokens_seen": 2329477120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029702294976729257,
+      "loss": 1.2499,
+      "theoretical_loss": 3.416139144796613,
+      "tokens_seen": 2329608192
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002969828277965014,
+      "loss": 1.2438,
+      "theoretical_loss": 3.416123725842005,
+      "tokens_seen": 2329739264
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002969427058257101,
+      "loss": 1.1962,
+      "theoretical_loss": 3.4161083079977255,
+      "tokens_seen": 2329870336
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000296902583854919,
+      "loss": 1.2316,
+      "theoretical_loss": 3.4160928912636335,
+      "tokens_seen": 2330001408
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029686246188412774,
+      "loss": 1.2015,
+      "theoretical_loss": 3.416077475639586,
+      "tokens_seen": 2330132480
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029682233991333654,
+      "loss": 1.2614,
+      "theoretical_loss": 3.416062061125441,
+      "tokens_seen": 2330263552
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029678221794254535,
+      "loss": 1.2395,
+      "theoretical_loss": 3.416046647721055,
+      "tokens_seen": 2330394624
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002967420959717541,
+      "loss": 1.2293,
+      "theoretical_loss": 3.4160312354262876,
+      "tokens_seen": 2330525696
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029670197400096296,
+      "loss": 1.2171,
+      "theoretical_loss": 3.4160158242409957,
+      "tokens_seen": 2330656768
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002966618520301717,
+      "loss": 1.2875,
+      "theoretical_loss": 3.416000414165037,
+      "tokens_seen": 2330787840
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002966217300593805,
+      "loss": 1.2157,
+      "theoretical_loss": 3.4159850051982685,
+      "tokens_seen": 2330918912
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029658160808858933,
+      "loss": 1.2491,
+      "theoretical_loss": 3.415969597340549,
+      "tokens_seen": 2331049984
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029654148611779813,
+      "loss": 1.283,
+      "theoretical_loss": 3.4159541905917363,
+      "tokens_seen": 2331181056
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002965013641470069,
+      "loss": 1.2603,
+      "theoretical_loss": 3.415938784951688,
+      "tokens_seen": 2331312128
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029646124217621575,
+      "loss": 1.2479,
+      "theoretical_loss": 3.415923380420262,
+      "tokens_seen": 2331443200
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 782999,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.249419927597046,
+      "objective/train/theoretical_loss": 3.4159079769973157,
+      "objective/train/tokens_used": 702033376,
+      "theoretical_loss": 3.4159079769973157,
+      "tokens_seen": 2331574272
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002964211202054245,
+      "loss": 1.2234,
+      "theoretical_loss": 3.4159079769973157,
+      "tokens_seen": 2331574272
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029638099823463325,
+      "loss": 1.2317,
+      "theoretical_loss": 3.4158925746827076,
+      "tokens_seen": 2331705344
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002963408762638421,
+      "loss": 1.2264,
+      "theoretical_loss": 3.4158771734762956,
+      "tokens_seen": 2331836416
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029630075429305086,
+      "loss": 1.3058,
+      "theoretical_loss": 3.4158617733779377,
+      "tokens_seen": 2331967488
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029626063232225967,
+      "loss": 1.2942,
+      "theoretical_loss": 3.415846374387492,
+      "tokens_seen": 2332098560
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002962205103514685,
+      "loss": 1.2782,
+      "theoretical_loss": 3.4158309765048163,
+      "tokens_seen": 2332229632
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002961803883806773,
+      "loss": 1.2111,
+      "theoretical_loss": 3.4158155797297685,
+      "tokens_seen": 2332360704
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029614026640988603,
+      "loss": 1.2298,
+      "theoretical_loss": 3.415800184062207,
+      "tokens_seen": 2332491776
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029610014443909484,
+      "loss": 1.2212,
+      "theoretical_loss": 3.41578478950199,
+      "tokens_seen": 2332622848
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029606002246830365,
+      "loss": 1.2929,
+      "theoretical_loss": 3.415769396048975,
+      "tokens_seen": 2332753920
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029601990049751245,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4157540037030207,
+      "tokens_seen": 2332884992
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029597977852672126,
+      "loss": 1.2013,
+      "theoretical_loss": 3.4157386124639855,
+      "tokens_seen": 2333016064
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029593965655593,
+      "loss": 1.231,
+      "theoretical_loss": 3.4157232223317275,
+      "tokens_seen": 2333147136
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029589953458513887,
+      "loss": 1.2108,
+      "theoretical_loss": 3.4157078333061044,
+      "tokens_seen": 2333278208
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002958594126143476,
+      "loss": 1.1667,
+      "theoretical_loss": 3.415692445386975,
+      "tokens_seen": 2333409280
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002958192906435564,
+      "loss": 1.234,
+      "theoretical_loss": 3.4156770585741976,
+      "tokens_seen": 2333540352
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029577916867276524,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4156616728676297,
+      "tokens_seen": 2333671424
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000295739046701974,
+      "loss": 1.2641,
+      "theoretical_loss": 3.415646288267131,
+      "tokens_seen": 2333802496
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002956989247311828,
+      "loss": 1.2949,
+      "theoretical_loss": 3.4156309047725584,
+      "tokens_seen": 2333933568
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002956588027603916,
+      "loss": 1.154,
+      "theoretical_loss": 3.4156155223837716,
+      "tokens_seen": 2334064640
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002956186807896004,
+      "loss": 1.2214,
+      "theoretical_loss": 3.4156001411006285,
+      "tokens_seen": 2334195712
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029557855881880916,
+      "loss": 1.213,
+      "theoretical_loss": 3.4155847609229877,
+      "tokens_seen": 2334326784
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029553843684801797,
+      "loss": 1.1456,
+      "theoretical_loss": 3.415569381850707,
+      "tokens_seen": 2334457856
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029549831487722677,
+      "loss": 1.2692,
+      "theoretical_loss": 3.415554003883646,
+      "tokens_seen": 2334588928
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002954581929064356,
+      "loss": 1.2215,
+      "theoretical_loss": 3.4155386270216623,
+      "tokens_seen": 2334720000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002954180709356444,
+      "loss": 1.2288,
+      "theoretical_loss": 3.4155232512646148,
+      "tokens_seen": 2334851072
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029537794896485314,
+      "loss": 1.2508,
+      "theoretical_loss": 3.415507876612362,
+      "tokens_seen": 2334982144
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.000295337826994062,
+      "loss": 1.1864,
+      "theoretical_loss": 3.4154925030647627,
+      "tokens_seen": 2335113216
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029529770502327075,
+      "loss": 1.1942,
+      "theoretical_loss": 3.415477130621676,
+      "tokens_seen": 2335244288
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002952575830524795,
+      "loss": 1.2548,
+      "theoretical_loss": 3.4154617592829593,
+      "tokens_seen": 2335375360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029521746108168836,
+      "loss": 1.2805,
+      "theoretical_loss": 3.415446389048472,
+      "tokens_seen": 2335506432
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002951773391108971,
+      "loss": 1.2331,
+      "theoretical_loss": 3.4154310199180733,
+      "tokens_seen": 2335637504
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002951372171401059,
+      "loss": 1.2611,
+      "theoretical_loss": 3.415415651891621,
+      "tokens_seen": 2335768576
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029509709516931473,
+      "loss": 1.2683,
+      "theoretical_loss": 3.415400284968974,
+      "tokens_seen": 2335899648
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029505697319852353,
+      "loss": 1.1958,
+      "theoretical_loss": 3.4153849191499916,
+      "tokens_seen": 2336030720
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002950168512277323,
+      "loss": 1.267,
+      "theoretical_loss": 3.415369554434533,
+      "tokens_seen": 2336161792
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029497672925694115,
+      "loss": 1.2007,
+      "theoretical_loss": 3.415354190822456,
+      "tokens_seen": 2336292864
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002949366072861499,
+      "loss": 1.2176,
+      "theoretical_loss": 3.4153388283136197,
+      "tokens_seen": 2336423936
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029489648531535865,
+      "loss": 1.2893,
+      "theoretical_loss": 3.415323466907883,
+      "tokens_seen": 2336555008
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002948563633445675,
+      "loss": 1.1862,
+      "theoretical_loss": 3.4153081066051056,
+      "tokens_seen": 2336686080
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029481624137377626,
+      "loss": 1.1801,
+      "theoretical_loss": 3.4152927474051453,
+      "tokens_seen": 2336817152
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002947761194029851,
+      "loss": 1.2869,
+      "theoretical_loss": 3.415277389307862,
+      "tokens_seen": 2336948224
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002947359974321939,
+      "loss": 1.2181,
+      "theoretical_loss": 3.415262032313114,
+      "tokens_seen": 2337079296
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002946958754614027,
+      "loss": 1.2777,
+      "theoretical_loss": 3.415246676420761,
+      "tokens_seen": 2337210368
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002946557534906115,
+      "loss": 1.2814,
+      "theoretical_loss": 3.415231321630662,
+      "tokens_seen": 2337341440
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029461563151982024,
+      "loss": 1.2149,
+      "theoretical_loss": 3.4152159679426752,
+      "tokens_seen": 2337472512
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029457550954902905,
+      "loss": 1.2332,
+      "theoretical_loss": 3.4152006153566603,
+      "tokens_seen": 2337603584
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029453538757823785,
+      "loss": 1.1726,
+      "theoretical_loss": 3.4151852638724765,
+      "tokens_seen": 2337734656
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029449526560744666,
+      "loss": 1.2617,
+      "theoretical_loss": 3.415169913489983,
+      "tokens_seen": 2337865728
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002944551436366554,
+      "loss": 1.186,
+      "theoretical_loss": 3.415154564209039,
+      "tokens_seen": 2337996800
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 785151,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2672348022460938,
+      "objective/train/theoretical_loss": 3.4151392160295027,
+      "objective/train/tokens_used": 708586976,
+      "theoretical_loss": 3.4151392160295027,
+      "tokens_seen": 2338127872
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029441502166586427,
+      "loss": 1.247,
+      "theoretical_loss": 3.4151392160295027,
+      "tokens_seen": 2338127872
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.000294374899695073,
+      "loss": 1.2195,
+      "theoretical_loss": 3.415123868951235,
+      "tokens_seen": 2338258944
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002943347777242818,
+      "loss": 1.1552,
+      "theoretical_loss": 3.415108522974094,
+      "tokens_seen": 2338390016
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029429465575349064,
+      "loss": 1.2133,
+      "theoretical_loss": 3.415093178097939,
+      "tokens_seen": 2338521088
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002942545337826994,
+      "loss": 1.1357,
+      "theoretical_loss": 3.4150778343226302,
+      "tokens_seen": 2338652160
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029421441181190825,
+      "loss": 1.2405,
+      "theoretical_loss": 3.415062491648026,
+      "tokens_seen": 2338783232
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.000294174289841117,
+      "loss": 1.2792,
+      "theoretical_loss": 3.4150471500739865,
+      "tokens_seen": 2338914304
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002941341678703258,
+      "loss": 1.2862,
+      "theoretical_loss": 3.4150318096003702,
+      "tokens_seen": 2339045376
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002940940458995346,
+      "loss": 1.2,
+      "theoretical_loss": 3.4150164702270374,
+      "tokens_seen": 2339176448
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029405392392874337,
+      "loss": 1.1921,
+      "theoretical_loss": 3.4150011319538467,
+      "tokens_seen": 2339307520
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029401380195795217,
+      "loss": 1.2741,
+      "theoretical_loss": 3.414985794780658,
+      "tokens_seen": 2339438592
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.000293973679987161,
+      "loss": 1.2921,
+      "theoretical_loss": 3.414970458707331,
+      "tokens_seen": 2339569664
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002939335580163698,
+      "loss": 1.2656,
+      "theoretical_loss": 3.414955123733725,
+      "tokens_seen": 2339700736
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029389343604557854,
+      "loss": 1.1808,
+      "theoretical_loss": 3.4149397898597,
+      "tokens_seen": 2339831808
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002938533140747874,
+      "loss": 1.2149,
+      "theoretical_loss": 3.4149244570851143,
+      "tokens_seen": 2339962880
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029381319210399615,
+      "loss": 1.223,
+      "theoretical_loss": 3.4149091254098285,
+      "tokens_seen": 2340093952
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002937730701332049,
+      "loss": 1.2504,
+      "theoretical_loss": 3.414893794833702,
+      "tokens_seen": 2340225024
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029373294816241376,
+      "loss": 1.2339,
+      "theoretical_loss": 3.414878465356595,
+      "tokens_seen": 2340356096
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002936928261916225,
+      "loss": 1.2899,
+      "theoretical_loss": 3.414863136978366,
+      "tokens_seen": 2340487168
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002936527042208313,
+      "loss": 1.2082,
+      "theoretical_loss": 3.414847809698876,
+      "tokens_seen": 2340618240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002936125822500401,
+      "loss": 1.3081,
+      "theoretical_loss": 3.4148324835179835,
+      "tokens_seen": 2340749312
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029357246027924893,
+      "loss": 1.2013,
+      "theoretical_loss": 3.414817158435549,
+      "tokens_seen": 2340880384
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029353233830845774,
+      "loss": 1.1622,
+      "theoretical_loss": 3.414801834451432,
+      "tokens_seen": 2341011456
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029349221633766655,
+      "loss": 1.2333,
+      "theoretical_loss": 3.414786511565492,
+      "tokens_seen": 2341142528
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002934520943668753,
+      "loss": 1.3143,
+      "theoretical_loss": 3.41477118977759,
+      "tokens_seen": 2341273600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002934119723960841,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4147558690875845,
+      "tokens_seen": 2341404672
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002933718504252929,
+      "loss": 1.1838,
+      "theoretical_loss": 3.414740549495336,
+      "tokens_seen": 2341535744
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029333172845450166,
+      "loss": 1.2633,
+      "theoretical_loss": 3.4147252310007046,
+      "tokens_seen": 2341666816
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002932916064837105,
+      "loss": 1.1824,
+      "theoretical_loss": 3.4147099136035495,
+      "tokens_seen": 2341797888
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002932514845129193,
+      "loss": 1.2678,
+      "theoretical_loss": 3.414694597303731,
+      "tokens_seen": 2341928960
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002932113625421281,
+      "loss": 1.2453,
+      "theoretical_loss": 3.4146792821011096,
+      "tokens_seen": 2342060032
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002931712405713369,
+      "loss": 1.2165,
+      "theoretical_loss": 3.4146639679955446,
+      "tokens_seen": 2342191104
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029313111860054564,
+      "loss": 1.227,
+      "theoretical_loss": 3.4146486549868964,
+      "tokens_seen": 2342322176
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029309099662975445,
+      "loss": 1.1933,
+      "theoretical_loss": 3.414633343075025,
+      "tokens_seen": 2342453248
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029305087465896325,
+      "loss": 1.2679,
+      "theoretical_loss": 3.4146180322597903,
+      "tokens_seen": 2342584320
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029301075268817206,
+      "loss": 1.2812,
+      "theoretical_loss": 3.414602722541053,
+      "tokens_seen": 2342715392
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002929706307173808,
+      "loss": 1.2451,
+      "theoretical_loss": 3.4145874139186723,
+      "tokens_seen": 2342846464
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029293050874658967,
+      "loss": 1.2773,
+      "theoretical_loss": 3.414572106392509,
+      "tokens_seen": 2342977536
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002928903867757984,
+      "loss": 1.2501,
+      "theoretical_loss": 3.414556799962423,
+      "tokens_seen": 2343108608
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029285026480500723,
+      "loss": 1.2133,
+      "theoretical_loss": 3.414541494628275,
+      "tokens_seen": 2343239680
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029281014283421604,
+      "loss": 1.2384,
+      "theoretical_loss": 3.4145261903899247,
+      "tokens_seen": 2343370752
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002927700208634248,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4145108872472325,
+      "tokens_seen": 2343501824
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029272989889263365,
+      "loss": 1.2593,
+      "theoretical_loss": 3.4144955852000587,
+      "tokens_seen": 2343632896
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002926897769218424,
+      "loss": 1.2601,
+      "theoretical_loss": 3.4144802842482633,
+      "tokens_seen": 2343763968
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002926496549510512,
+      "loss": 1.2487,
+      "theoretical_loss": 3.4144649843917074,
+      "tokens_seen": 2343895040
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029260953298026,
+      "loss": 1.2018,
+      "theoretical_loss": 3.414449685630251,
+      "tokens_seen": 2344026112
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029256941100946877,
+      "loss": 1.2083,
+      "theoretical_loss": 3.4144343879637544,
+      "tokens_seen": 2344157184
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029252928903867757,
+      "loss": 1.2349,
+      "theoretical_loss": 3.4144190913920776,
+      "tokens_seen": 2344288256
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002924891670678864,
+      "loss": 1.2229,
+      "theoretical_loss": 3.4144037959150815,
+      "tokens_seen": 2344419328
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002924490450970952,
+      "loss": 1.2353,
+      "theoretical_loss": 3.4143885015326267,
+      "tokens_seen": 2344550400
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 787449,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2501776218414307,
+      "objective/train/theoretical_loss": 3.414373208244574,
+      "objective/train/tokens_used": 715140576,
+      "theoretical_loss": 3.414373208244574,
+      "tokens_seen": 2344681472
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029240892312630394,
+      "loss": 1.2497,
+      "theoretical_loss": 3.414373208244574,
+      "tokens_seen": 2344681472
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002923688011555128,
+      "loss": 1.2879,
+      "theoretical_loss": 3.414357916050783,
+      "tokens_seen": 2344812544
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029232867918472155,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4143426249511144,
+      "tokens_seen": 2344943616
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002922885572139303,
+      "loss": 1.1992,
+      "theoretical_loss": 3.4143273349454297,
+      "tokens_seen": 2345074688
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029224843524313916,
+      "loss": 1.3511,
+      "theoretical_loss": 3.4143120460335883,
+      "tokens_seen": 2345205760
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002922083132723479,
+      "loss": 1.3047,
+      "theoretical_loss": 3.4142967582154515,
+      "tokens_seen": 2345336832
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002921681913015568,
+      "loss": 1.2283,
+      "theoretical_loss": 3.41428147149088,
+      "tokens_seen": 2345467904
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002921280693307655,
+      "loss": 1.2476,
+      "theoretical_loss": 3.4142661858597343,
+      "tokens_seen": 2345598976
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029208794735997433,
+      "loss": 1.2233,
+      "theoretical_loss": 3.414250901321875,
+      "tokens_seen": 2345730048
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029204782538918314,
+      "loss": 1.1923,
+      "theoretical_loss": 3.414235617877163,
+      "tokens_seen": 2345861120
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002920077034183919,
+      "loss": 1.2581,
+      "theoretical_loss": 3.414220335525459,
+      "tokens_seen": 2345992192
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002919675814476007,
+      "loss": 1.2513,
+      "theoretical_loss": 3.414205054266623,
+      "tokens_seen": 2346123264
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002919274594768095,
+      "loss": 1.2046,
+      "theoretical_loss": 3.4141897741005174,
+      "tokens_seen": 2346254336
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002918873375060183,
+      "loss": 1.2354,
+      "theoretical_loss": 3.4141744950270017,
+      "tokens_seen": 2346385408
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029184721553522706,
+      "loss": 1.2638,
+      "theoretical_loss": 3.4141592170459374,
+      "tokens_seen": 2346516480
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002918070935644359,
+      "loss": 1.3386,
+      "theoretical_loss": 3.414143940157185,
+      "tokens_seen": 2346647552
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002917669715936447,
+      "loss": 1.2374,
+      "theoretical_loss": 3.4141286643606055,
+      "tokens_seen": 2346778624
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002917268496228535,
+      "loss": 1.2352,
+      "theoretical_loss": 3.41411338965606,
+      "tokens_seen": 2346909696
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002916867276520623,
+      "loss": 1.189,
+      "theoretical_loss": 3.4140981160434096,
+      "tokens_seen": 2347040768
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029164660568127104,
+      "loss": 1.2886,
+      "theoretical_loss": 3.4140828435225146,
+      "tokens_seen": 2347171840
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002916064837104799,
+      "loss": 1.3099,
+      "theoretical_loss": 3.4140675720932365,
+      "tokens_seen": 2347302912
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029156636173968865,
+      "loss": 1.1824,
+      "theoretical_loss": 3.4140523017554365,
+      "tokens_seen": 2347433984
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029152623976889746,
+      "loss": 1.2668,
+      "theoretical_loss": 3.414037032508975,
+      "tokens_seen": 2347565056
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029148611779810626,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4140217643537136,
+      "tokens_seen": 2347696128
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029144599582731507,
+      "loss": 1.1929,
+      "theoretical_loss": 3.414006497289513,
+      "tokens_seen": 2347827200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002914058738565238,
+      "loss": 1.1799,
+      "theoretical_loss": 3.4139912313162353,
+      "tokens_seen": 2347958272
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029136575188573263,
+      "loss": 1.2535,
+      "theoretical_loss": 3.413975966433741,
+      "tokens_seen": 2348089344
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029132562991494144,
+      "loss": 1.2357,
+      "theoretical_loss": 3.4139607026418908,
+      "tokens_seen": 2348220416
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002912855079441502,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4139454399405462,
+      "tokens_seen": 2348351488
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029124538597335905,
+      "loss": 1.2404,
+      "theoretical_loss": 3.4139301783295686,
+      "tokens_seen": 2348482560
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002912052640025678,
+      "loss": 1.1727,
+      "theoretical_loss": 3.4139149178088193,
+      "tokens_seen": 2348613632
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002911651420317766,
+      "loss": 1.3017,
+      "theoretical_loss": 3.41389965837816,
+      "tokens_seen": 2348744704
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002911250200609854,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4138844000374506,
+      "tokens_seen": 2348875776
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029108489809019417,
+      "loss": 1.2393,
+      "theoretical_loss": 3.413869142786554,
+      "tokens_seen": 2349006848
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029104477611940297,
+      "loss": 1.2744,
+      "theoretical_loss": 3.413853886625331,
+      "tokens_seen": 2349137920
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002910046541486118,
+      "loss": 1.3237,
+      "theoretical_loss": 3.4138386315536424,
+      "tokens_seen": 2349268992
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002909645321778206,
+      "loss": 1.249,
+      "theoretical_loss": 3.4138233775713505,
+      "tokens_seen": 2349400064
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002909244102070294,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4138081246783165,
+      "tokens_seen": 2349531136
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002908842882362382,
+      "loss": 1.2867,
+      "theoretical_loss": 3.413792872874401,
+      "tokens_seen": 2349662208
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029084416626544695,
+      "loss": 1.1492,
+      "theoretical_loss": 3.4137776221594667,
+      "tokens_seen": 2349793280
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029080404429465576,
+      "loss": 1.2464,
+      "theoretical_loss": 3.4137623725333746,
+      "tokens_seen": 2349924352
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029076392232386456,
+      "loss": 1.2159,
+      "theoretical_loss": 3.4137471239959862,
+      "tokens_seen": 2350055424
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002907238003530733,
+      "loss": 1.2606,
+      "theoretical_loss": 3.4137318765471627,
+      "tokens_seen": 2350186496
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002906836783822822,
+      "loss": 1.2444,
+      "theoretical_loss": 3.4137166301867667,
+      "tokens_seen": 2350317568
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002906435564114909,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4137013849146585,
+      "tokens_seen": 2350448640
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029060343444069973,
+      "loss": 1.2074,
+      "theoretical_loss": 3.4136861407307006,
+      "tokens_seen": 2350579712
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029056331246990854,
+      "loss": 1.1962,
+      "theoretical_loss": 3.4136708976347547,
+      "tokens_seen": 2350710784
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002905231904991173,
+      "loss": 1.2188,
+      "theoretical_loss": 3.413655655626682,
+      "tokens_seen": 2350841856
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002904830685283261,
+      "loss": 1.2575,
+      "theoretical_loss": 3.413640414706345,
+      "tokens_seen": 2350972928
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002904429465575349,
+      "loss": 1.2846,
+      "theoretical_loss": 3.413625174873604,
+      "tokens_seen": 2351104000
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 789531,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3324353694915771,
+      "objective/train/theoretical_loss": 3.4136099361283225,
+      "objective/train/tokens_used": 721694176,
+      "theoretical_loss": 3.4136099361283225,
+      "tokens_seen": 2351235072
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002904028245867437,
+      "loss": 1.2895,
+      "theoretical_loss": 3.4136099361283225,
+      "tokens_seen": 2351235072
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002903627026159525,
+      "loss": 1.2636,
+      "theoretical_loss": 3.4135946984703613,
+      "tokens_seen": 2351366144
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002903225806451613,
+      "loss": 1.2928,
+      "theoretical_loss": 3.413579461899582,
+      "tokens_seen": 2351497216
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002902824586743701,
+      "loss": 1.1876,
+      "theoretical_loss": 3.413564226415847,
+      "tokens_seen": 2351628288
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029024233670357894,
+      "loss": 1.2814,
+      "theoretical_loss": 3.413548992019018,
+      "tokens_seen": 2351759360
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002902022147327877,
+      "loss": 1.1465,
+      "theoretical_loss": 3.413533758708957,
+      "tokens_seen": 2351890432
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029016209276199644,
+      "loss": 1.1945,
+      "theoretical_loss": 3.4135185264855257,
+      "tokens_seen": 2352021504
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002901219707912053,
+      "loss": 1.2469,
+      "theoretical_loss": 3.413503295348586,
+      "tokens_seen": 2352152576
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029008184882041405,
+      "loss": 1.245,
+      "theoretical_loss": 3.413488065298,
+      "tokens_seen": 2352283648
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029004172684962286,
+      "loss": 1.2173,
+      "theoretical_loss": 3.41347283633363,
+      "tokens_seen": 2352414720
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029000160487883166,
+      "loss": 1.2538,
+      "theoretical_loss": 3.4134576084553374,
+      "tokens_seen": 2352545792
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028996148290804047,
+      "loss": 1.269,
+      "theoretical_loss": 3.4134423816629846,
+      "tokens_seen": 2352676864
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002899213609372492,
+      "loss": 1.3248,
+      "theoretical_loss": 3.4134271559564335,
+      "tokens_seen": 2352807936
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028988123896645803,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4134119313355464,
+      "tokens_seen": 2352939008
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028984111699566684,
+      "loss": 1.3085,
+      "theoretical_loss": 3.4133967078001852,
+      "tokens_seen": 2353070080
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002898009950248756,
+      "loss": 1.2454,
+      "theoretical_loss": 3.413381485350212,
+      "tokens_seen": 2353201152
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028976087305408445,
+      "loss": 1.3049,
+      "theoretical_loss": 3.4133662639854894,
+      "tokens_seen": 2353332224
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002897207510832932,
+      "loss": 1.2465,
+      "theoretical_loss": 3.4133510437058794,
+      "tokens_seen": 2353463296
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028968062911250206,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4133358245112437,
+      "tokens_seen": 2353594368
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002896405071417108,
+      "loss": 1.2263,
+      "theoretical_loss": 3.413320606401445,
+      "tokens_seen": 2353725440
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028960038517091957,
+      "loss": 1.2604,
+      "theoretical_loss": 3.4133053893763456,
+      "tokens_seen": 2353856512
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002895602632001284,
+      "loss": 1.2181,
+      "theoretical_loss": 3.413290173435808,
+      "tokens_seen": 2353987584
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002895201412293372,
+      "loss": 1.3331,
+      "theoretical_loss": 3.413274958579694,
+      "tokens_seen": 2354118656
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000289480019258546,
+      "loss": 1.2341,
+      "theoretical_loss": 3.413259744807866,
+      "tokens_seen": 2354249728
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002894398972877548,
+      "loss": 1.1985,
+      "theoretical_loss": 3.413244532120187,
+      "tokens_seen": 2354380800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002893997753169636,
+      "loss": 1.3209,
+      "theoretical_loss": 3.4132293205165185,
+      "tokens_seen": 2354511872
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028935965334617235,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4132141099967237,
+      "tokens_seen": 2354642944
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028931953137538115,
+      "loss": 1.1891,
+      "theoretical_loss": 3.4131989005606647,
+      "tokens_seen": 2354774016
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028927940940458996,
+      "loss": 1.3146,
+      "theoretical_loss": 3.4131836922082037,
+      "tokens_seen": 2354905088
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002892392874337987,
+      "loss": 1.2156,
+      "theoretical_loss": 3.4131684849392037,
+      "tokens_seen": 2355036160
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002891991654630076,
+      "loss": 1.2307,
+      "theoretical_loss": 3.413153278753527,
+      "tokens_seen": 2355167232
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002891590434922163,
+      "loss": 1.2869,
+      "theoretical_loss": 3.413138073651036,
+      "tokens_seen": 2355298304
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002891189215214252,
+      "loss": 1.2725,
+      "theoretical_loss": 3.4131228696315934,
+      "tokens_seen": 2355429376
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028907879955063394,
+      "loss": 1.2544,
+      "theoretical_loss": 3.4131076666950615,
+      "tokens_seen": 2355560448
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002890386775798427,
+      "loss": 1.2359,
+      "theoretical_loss": 3.4130924648413035,
+      "tokens_seen": 2355691520
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028899855560905155,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4130772640701816,
+      "tokens_seen": 2355822592
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002889584336382603,
+      "loss": 1.2616,
+      "theoretical_loss": 3.413062064381559,
+      "tokens_seen": 2355953664
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002889183116674691,
+      "loss": 1.2918,
+      "theoretical_loss": 3.4130468657752977,
+      "tokens_seen": 2356084736
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002888781896966779,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4130316682512607,
+      "tokens_seen": 2356215808
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002888380677258867,
+      "loss": 1.2142,
+      "theoretical_loss": 3.4130164718093106,
+      "tokens_seen": 2356346880
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002887979457550955,
+      "loss": 1.3161,
+      "theoretical_loss": 3.413001276449311,
+      "tokens_seen": 2356477952
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028875782378430433,
+      "loss": 1.224,
+      "theoretical_loss": 3.4129860821711233,
+      "tokens_seen": 2356609024
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002887177018135131,
+      "loss": 1.194,
+      "theoretical_loss": 3.4129708889746113,
+      "tokens_seen": 2356740096
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028867757984272184,
+      "loss": 1.2846,
+      "theoretical_loss": 3.4129556968596377,
+      "tokens_seen": 2356871168
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002886374578719307,
+      "loss": 1.2154,
+      "theoretical_loss": 3.4129405058260653,
+      "tokens_seen": 2357002240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028859733590113945,
+      "loss": 1.1742,
+      "theoretical_loss": 3.412925315873757,
+      "tokens_seen": 2357133312
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028855721393034826,
+      "loss": 1.203,
+      "theoretical_loss": 3.4129101270025757,
+      "tokens_seen": 2357264384
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028851709195955706,
+      "loss": 1.2172,
+      "theoretical_loss": 3.412894939212384,
+      "tokens_seen": 2357395456
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028847696998876587,
+      "loss": 1.2075,
+      "theoretical_loss": 3.4128797525030454,
+      "tokens_seen": 2357526528
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002884368480179747,
+      "loss": 1.2577,
+      "theoretical_loss": 3.412864566874423,
+      "tokens_seen": 2357657600
+    },
+    {
+      "epoch": 0.43,
+      "objective/train/docs_used": 791745,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2201902866363525,
+      "objective/train/theoretical_loss": 3.412849382326379,
+      "objective/train/tokens_used": 728247776,
+      "theoretical_loss": 3.412849382326379,
+      "tokens_seen": 2357788672
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028839672604718343,
+      "loss": 1.3035,
+      "theoretical_loss": 3.412849382326379,
+      "tokens_seen": 2357788672
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028835660407639224,
+      "loss": 1.223,
+      "theoretical_loss": 3.4128341988587767,
+      "tokens_seen": 2357919744
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028831648210560104,
+      "loss": 1.2724,
+      "theoretical_loss": 3.41281901647148,
+      "tokens_seen": 2358050816
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028827636013480985,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4128038351643513,
+      "tokens_seen": 2358181888
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002882362381640186,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4127886549372537,
+      "tokens_seen": 2358312960
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028819611619322746,
+      "loss": 1.2496,
+      "theoretical_loss": 3.4127734757900505,
+      "tokens_seen": 2358444032
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002881559942224362,
+      "loss": 1.2617,
+      "theoretical_loss": 3.412758297722605,
+      "tokens_seen": 2358575104
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028811587225164496,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4127431207347803,
+      "tokens_seen": 2358706176
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002880757502808538,
+      "loss": 1.1786,
+      "theoretical_loss": 3.412727944826439,
+      "tokens_seen": 2358837248
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002880356283100626,
+      "loss": 1.2329,
+      "theoretical_loss": 3.4127127699974458,
+      "tokens_seen": 2358968320
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002879955063392714,
+      "loss": 1.1836,
+      "theoretical_loss": 3.4126975962476624,
+      "tokens_seen": 2359099392
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002879553843684802,
+      "loss": 1.2417,
+      "theoretical_loss": 3.412682423576953,
+      "tokens_seen": 2359230464
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000287915262397689,
+      "loss": 1.263,
+      "theoretical_loss": 3.4126672519851806,
+      "tokens_seen": 2359361536
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028787514042689775,
+      "loss": 1.2075,
+      "theoretical_loss": 3.4126520814722086,
+      "tokens_seen": 2359492608
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028783501845610655,
+      "loss": 1.2777,
+      "theoretical_loss": 3.4126369120379003,
+      "tokens_seen": 2359623680
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028779489648531536,
+      "loss": 1.2267,
+      "theoretical_loss": 3.412621743682119,
+      "tokens_seen": 2359754752
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028775477451452417,
+      "loss": 1.2766,
+      "theoretical_loss": 3.412606576404729,
+      "tokens_seen": 2359885824
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000287714652543733,
+      "loss": 1.2314,
+      "theoretical_loss": 3.412591410205593,
+      "tokens_seen": 2360016896
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002876745305729417,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4125762450845736,
+      "tokens_seen": 2360147968
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002876344086021506,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4125610810415363,
+      "tokens_seen": 2360279040
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028759428663135934,
+      "loss": 1.2813,
+      "theoretical_loss": 3.4125459180763427,
+      "tokens_seen": 2360410112
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002875541646605681,
+      "loss": 1.2084,
+      "theoretical_loss": 3.4125307561888576,
+      "tokens_seen": 2360541184
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028751404268977695,
+      "loss": 1.2452,
+      "theoretical_loss": 3.412515595378944,
+      "tokens_seen": 2360672256
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002874739207189857,
+      "loss": 1.2624,
+      "theoretical_loss": 3.4125004356464657,
+      "tokens_seen": 2360803328
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002874337987481945,
+      "loss": 1.2092,
+      "theoretical_loss": 3.412485276991286,
+      "tokens_seen": 2360934400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002873936767774033,
+      "loss": 1.2309,
+      "theoretical_loss": 3.412470119413269,
+      "tokens_seen": 2361065472
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002873535548066121,
+      "loss": 1.2464,
+      "theoretical_loss": 3.4124549629122782,
+      "tokens_seen": 2361196544
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002873134328358209,
+      "loss": 1.2482,
+      "theoretical_loss": 3.412439807488177,
+      "tokens_seen": 2361327616
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002872733108650297,
+      "loss": 1.2257,
+      "theoretical_loss": 3.4124246531408295,
+      "tokens_seen": 2361458688
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002872331888942385,
+      "loss": 1.2039,
+      "theoretical_loss": 3.4124094998700993,
+      "tokens_seen": 2361589760
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002871930669234473,
+      "loss": 1.2685,
+      "theoretical_loss": 3.4123943476758507,
+      "tokens_seen": 2361720832
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002871529449526561,
+      "loss": 1.186,
+      "theoretical_loss": 3.4123791965579464,
+      "tokens_seen": 2361851904
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028711282298186485,
+      "loss": 1.2551,
+      "theoretical_loss": 3.4123640465162506,
+      "tokens_seen": 2361982976
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002870727010110737,
+      "loss": 1.2249,
+      "theoretical_loss": 3.4123488975506273,
+      "tokens_seen": 2362114048
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028703257904028246,
+      "loss": 1.254,
+      "theoretical_loss": 3.4123337496609407,
+      "tokens_seen": 2362245120
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028699245706949127,
+      "loss": 1.222,
+      "theoretical_loss": 3.4123186028470545,
+      "tokens_seen": 2362376192
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002869523350987001,
+      "loss": 1.2921,
+      "theoretical_loss": 3.412303457108832,
+      "tokens_seen": 2362507264
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028691221312790883,
+      "loss": 1.2555,
+      "theoretical_loss": 3.412288312446138,
+      "tokens_seen": 2362638336
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028687209115711763,
+      "loss": 1.2366,
+      "theoretical_loss": 3.412273168858836,
+      "tokens_seen": 2362769408
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028683196918632644,
+      "loss": 1.2257,
+      "theoretical_loss": 3.41225802634679,
+      "tokens_seen": 2362900480
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028679184721553525,
+      "loss": 1.2001,
+      "theoretical_loss": 3.412242884909864,
+      "tokens_seen": 2363031552
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000286751725244744,
+      "loss": 1.2868,
+      "theoretical_loss": 3.4122277445479225,
+      "tokens_seen": 2363162624
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028671160327395286,
+      "loss": 1.2705,
+      "theoretical_loss": 3.412212605260829,
+      "tokens_seen": 2363293696
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002866714813031616,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4121974670484474,
+      "tokens_seen": 2363424768
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028663135933237036,
+      "loss": 1.2112,
+      "theoretical_loss": 3.4121823299106424,
+      "tokens_seen": 2363555840
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002865912373615792,
+      "loss": 1.213,
+      "theoretical_loss": 3.412167193847278,
+      "tokens_seen": 2363686912
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000286551115390788,
+      "loss": 1.2492,
+      "theoretical_loss": 3.4121520588582186,
+      "tokens_seen": 2363817984
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028651099341999684,
+      "loss": 1.2687,
+      "theoretical_loss": 3.4121369249433275,
+      "tokens_seen": 2363949056
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002864708714492056,
+      "loss": 1.2303,
+      "theoretical_loss": 3.41212179210247,
+      "tokens_seen": 2364080128
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002864307494784144,
+      "loss": 1.2202,
+      "theoretical_loss": 3.4121066603355095,
+      "tokens_seen": 2364211200
+    },
+    {
+      "epoch": 0.43,
+      "objective/train/docs_used": 793971,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.284865140914917,
+      "objective/train/theoretical_loss": 3.4120915296423107,
+      "objective/train/tokens_used": 734801376,
+      "theoretical_loss": 3.4120915296423107,
+      "tokens_seen": 2364342272
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002863906275076232,
+      "loss": 1.2819,
+      "theoretical_loss": 3.4120915296423107,
+      "tokens_seen": 2364342272
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028635050553683195,
+      "loss": 1.2066,
+      "theoretical_loss": 3.4120764000227375,
+      "tokens_seen": 2364473344
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028631038356604076,
+      "loss": 1.2524,
+      "theoretical_loss": 3.412061271476655,
+      "tokens_seen": 2364604416
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028627026159524957,
+      "loss": 1.1992,
+      "theoretical_loss": 3.4120461440039267,
+      "tokens_seen": 2364735488
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002862301396244584,
+      "loss": 1.196,
+      "theoretical_loss": 3.4120310176044173,
+      "tokens_seen": 2364866560
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002861900176536671,
+      "loss": 1.1969,
+      "theoretical_loss": 3.412015892277991,
+      "tokens_seen": 2364997632
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000286149895682876,
+      "loss": 1.2936,
+      "theoretical_loss": 3.4120007680245124,
+      "tokens_seen": 2365128704
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028610977371208474,
+      "loss": 1.2585,
+      "theoretical_loss": 3.411985644843846,
+      "tokens_seen": 2365259776
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002860696517412935,
+      "loss": 1.2518,
+      "theoretical_loss": 3.411970522735856,
+      "tokens_seen": 2365390848
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028602952977050235,
+      "loss": 1.2191,
+      "theoretical_loss": 3.4119554017004075,
+      "tokens_seen": 2365521920
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002859894077997111,
+      "loss": 1.2201,
+      "theoretical_loss": 3.411940281737364,
+      "tokens_seen": 2365652992
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028594928582891996,
+      "loss": 1.2174,
+      "theoretical_loss": 3.411925162846591,
+      "tokens_seen": 2365784064
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002859091638581287,
+      "loss": 1.3363,
+      "theoretical_loss": 3.4119100450279523,
+      "tokens_seen": 2365915136
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002858690418873375,
+      "loss": 1.2761,
+      "theoretical_loss": 3.411894928281313,
+      "tokens_seen": 2366046208
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028582891991654633,
+      "loss": 1.2777,
+      "theoretical_loss": 3.4118798126065375,
+      "tokens_seen": 2366177280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002857887979457551,
+      "loss": 1.2248,
+      "theoretical_loss": 3.41186469800349,
+      "tokens_seen": 2366308352
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002857486759749639,
+      "loss": 1.2887,
+      "theoretical_loss": 3.4118495844720362,
+      "tokens_seen": 2366439424
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002857085540041727,
+      "loss": 1.2893,
+      "theoretical_loss": 3.41183447201204,
+      "tokens_seen": 2366570496
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002856684320333815,
+      "loss": 1.2964,
+      "theoretical_loss": 3.411819360623366,
+      "tokens_seen": 2366701568
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028562831006259025,
+      "loss": 1.2727,
+      "theoretical_loss": 3.41180425030588,
+      "tokens_seen": 2366832640
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002855881880917991,
+      "loss": 1.2672,
+      "theoretical_loss": 3.411789141059445,
+      "tokens_seen": 2366963712
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028554806612100786,
+      "loss": 1.2368,
+      "theoretical_loss": 3.4117740328839274,
+      "tokens_seen": 2367094784
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028550794415021667,
+      "loss": 1.2451,
+      "theoretical_loss": 3.4117589257791914,
+      "tokens_seen": 2367225856
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002854678221794255,
+      "loss": 1.2257,
+      "theoretical_loss": 3.4117438197451015,
+      "tokens_seen": 2367356928
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028542770020863423,
+      "loss": 1.2701,
+      "theoretical_loss": 3.411728714781523,
+      "tokens_seen": 2367488000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028538757823784303,
+      "loss": 1.2204,
+      "theoretical_loss": 3.41171361088832,
+      "tokens_seen": 2367619072
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028534745626705184,
+      "loss": 1.1969,
+      "theoretical_loss": 3.411698508065359,
+      "tokens_seen": 2367750144
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028530733429626065,
+      "loss": 1.2913,
+      "theoretical_loss": 3.411683406312503,
+      "tokens_seen": 2367881216
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028526721232546945,
+      "loss": 1.2575,
+      "theoretical_loss": 3.411668305629618,
+      "tokens_seen": 2368012288
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028522709035467826,
+      "loss": 1.2435,
+      "theoretical_loss": 3.4116532060165694,
+      "tokens_seen": 2368143360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000285186968383887,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4116381074732214,
+      "tokens_seen": 2368274432
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002851468464130958,
+      "loss": 1.2327,
+      "theoretical_loss": 3.411623009999439,
+      "tokens_seen": 2368405504
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002851067244423046,
+      "loss": 1.2415,
+      "theoretical_loss": 3.411607913595088,
+      "tokens_seen": 2368536576
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002850666024715134,
+      "loss": 1.2333,
+      "theoretical_loss": 3.411592818260033,
+      "tokens_seen": 2368667648
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028502648050072224,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4115777239941387,
+      "tokens_seen": 2368798720
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000284986358529931,
+      "loss": 1.2436,
+      "theoretical_loss": 3.411562630797271,
+      "tokens_seen": 2368929792
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002849462365591398,
+      "loss": 1.224,
+      "theoretical_loss": 3.411547538669294,
+      "tokens_seen": 2369060864
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002849061145883486,
+      "loss": 1.2258,
+      "theoretical_loss": 3.411532447610074,
+      "tokens_seen": 2369191936
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028486599261755735,
+      "loss": 1.263,
+      "theoretical_loss": 3.4115173576194757,
+      "tokens_seen": 2369323008
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028482587064676616,
+      "loss": 1.2605,
+      "theoretical_loss": 3.4115022686973644,
+      "tokens_seen": 2369454080
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028478574867597497,
+      "loss": 1.2341,
+      "theoretical_loss": 3.411487180843605,
+      "tokens_seen": 2369585152
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028474562670518377,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4114720940580634,
+      "tokens_seen": 2369716224
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002847055047343925,
+      "loss": 1.1556,
+      "theoretical_loss": 3.411457008340604,
+      "tokens_seen": 2369847296
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002846653827636014,
+      "loss": 1.1863,
+      "theoretical_loss": 3.4114419236910933,
+      "tokens_seen": 2369978368
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028462526079281014,
+      "loss": 1.2083,
+      "theoretical_loss": 3.411426840109396,
+      "tokens_seen": 2370109440
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028458513882201894,
+      "loss": 1.262,
+      "theoretical_loss": 3.411411757595377,
+      "tokens_seen": 2370240512
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028454501685122775,
+      "loss": 1.269,
+      "theoretical_loss": 3.4113966761489025,
+      "tokens_seen": 2370371584
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002845048948804365,
+      "loss": 1.2092,
+      "theoretical_loss": 3.4113815957698375,
+      "tokens_seen": 2370502656
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028446477290964536,
+      "loss": 1.2972,
+      "theoretical_loss": 3.411366516458047,
+      "tokens_seen": 2370633728
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002844246509388541,
+      "loss": 1.2371,
+      "theoretical_loss": 3.411351438213398,
+      "tokens_seen": 2370764800
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 796080,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3398199081420898,
+      "objective/train/theoretical_loss": 3.4113363610357537,
+      "objective/train/tokens_used": 741354976,
+      "theoretical_loss": 3.4113363610357537,
+      "tokens_seen": 2370895872
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002843845289680629,
+      "loss": 1.3029,
+      "theoretical_loss": 3.4113363610357537,
+      "tokens_seen": 2370895872
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028434440699727173,
+      "loss": 1.2652,
+      "theoretical_loss": 3.411321284924982,
+      "tokens_seen": 2371026944
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002843042850264805,
+      "loss": 1.2441,
+      "theoretical_loss": 3.4113062098809466,
+      "tokens_seen": 2371158016
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002842641630556893,
+      "loss": 1.2535,
+      "theoretical_loss": 3.411291135903514,
+      "tokens_seen": 2371289088
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002842240410848981,
+      "loss": 1.2726,
+      "theoretical_loss": 3.41127606299255,
+      "tokens_seen": 2371420160
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002841839191141069,
+      "loss": 1.2621,
+      "theoretical_loss": 3.411260991147919,
+      "tokens_seen": 2371551232
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028414379714331565,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4112459203694874,
+      "tokens_seen": 2371682304
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002841036751725245,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4112308506571214,
+      "tokens_seen": 2371813376
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028406355320173326,
+      "loss": 1.1907,
+      "theoretical_loss": 3.411215782010686,
+      "tokens_seen": 2371944448
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000284023431230942,
+      "loss": 1.2782,
+      "theoretical_loss": 3.4112007144300467,
+      "tokens_seen": 2372075520
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002839833092601509,
+      "loss": 1.2417,
+      "theoretical_loss": 3.4111856479150697,
+      "tokens_seen": 2372206592
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028394318728935963,
+      "loss": 1.1642,
+      "theoretical_loss": 3.411170582465621,
+      "tokens_seen": 2372337664
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002839030653185685,
+      "loss": 1.1918,
+      "theoretical_loss": 3.4111555180815656,
+      "tokens_seen": 2372468736
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028386294334777724,
+      "loss": 1.2046,
+      "theoretical_loss": 3.4111404547627697,
+      "tokens_seen": 2372599808
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028382282137698605,
+      "loss": 1.1787,
+      "theoretical_loss": 3.4111253925090996,
+      "tokens_seen": 2372730880
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028378269940619485,
+      "loss": 1.2239,
+      "theoretical_loss": 3.4111103313204203,
+      "tokens_seen": 2372861952
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028374257743540366,
+      "loss": 1.2223,
+      "theoretical_loss": 3.411095271196598,
+      "tokens_seen": 2372993024
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002837024554646124,
+      "loss": 1.2014,
+      "theoretical_loss": 3.411080212137499,
+      "tokens_seen": 2373124096
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002836623334938212,
+      "loss": 1.2385,
+      "theoretical_loss": 3.411065154142989,
+      "tokens_seen": 2373255168
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028362221152303,
+      "loss": 1.2199,
+      "theoretical_loss": 3.411050097212933,
+      "tokens_seen": 2373386240
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002835820895522388,
+      "loss": 1.2545,
+      "theoretical_loss": 3.4110350413471986,
+      "tokens_seen": 2373517312
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028354196758144764,
+      "loss": 1.2293,
+      "theoretical_loss": 3.411019986545651,
+      "tokens_seen": 2373648384
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002835018456106564,
+      "loss": 1.2256,
+      "theoretical_loss": 3.411004932808156,
+      "tokens_seen": 2373779456
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002834617236398652,
+      "loss": 1.2828,
+      "theoretical_loss": 3.41098988013458,
+      "tokens_seen": 2373910528
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000283421601669074,
+      "loss": 1.2352,
+      "theoretical_loss": 3.410974828524789,
+      "tokens_seen": 2374041600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028338147969828275,
+      "loss": 1.2479,
+      "theoretical_loss": 3.410959777978649,
+      "tokens_seen": 2374172672
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002833413577274916,
+      "loss": 1.2124,
+      "theoretical_loss": 3.4109447284960264,
+      "tokens_seen": 2374303744
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028330123575670037,
+      "loss": 1.3277,
+      "theoretical_loss": 3.4109296800767868,
+      "tokens_seen": 2374434816
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028326111378590917,
+      "loss": 1.2511,
+      "theoretical_loss": 3.4109146327207966,
+      "tokens_seen": 2374565888
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000283220991815118,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4108995864279223,
+      "tokens_seen": 2374696960
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002831808698443268,
+      "loss": 1.2035,
+      "theoretical_loss": 3.41088454119803,
+      "tokens_seen": 2374828032
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028314074787353554,
+      "loss": 1.227,
+      "theoretical_loss": 3.4108694970309856,
+      "tokens_seen": 2374959104
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028310062590274434,
+      "loss": 1.2385,
+      "theoretical_loss": 3.410854453926656,
+      "tokens_seen": 2375090176
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028306050393195315,
+      "loss": 1.2298,
+      "theoretical_loss": 3.410839411884907,
+      "tokens_seen": 2375221248
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002830203819611619,
+      "loss": 1.2309,
+      "theoretical_loss": 3.4108243709056048,
+      "tokens_seen": 2375352320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028298025999037076,
+      "loss": 1.2437,
+      "theoretical_loss": 3.4108093309886156,
+      "tokens_seen": 2375483392
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002829401380195795,
+      "loss": 1.2281,
+      "theoretical_loss": 3.4107942921338066,
+      "tokens_seen": 2375614464
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002829000160487883,
+      "loss": 1.1937,
+      "theoretical_loss": 3.4107792543410436,
+      "tokens_seen": 2375745536
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028285989407799713,
+      "loss": 1.2456,
+      "theoretical_loss": 3.4107642176101933,
+      "tokens_seen": 2375876608
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002828197721072059,
+      "loss": 1.2459,
+      "theoretical_loss": 3.4107491819411218,
+      "tokens_seen": 2376007680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002827796501364147,
+      "loss": 1.1814,
+      "theoretical_loss": 3.410734147333695,
+      "tokens_seen": 2376138752
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002827395281656235,
+      "loss": 1.2539,
+      "theoretical_loss": 3.4107191137877813,
+      "tokens_seen": 2376269824
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002826994061948323,
+      "loss": 1.2279,
+      "theoretical_loss": 3.410704081303245,
+      "tokens_seen": 2376400896
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002826592842240411,
+      "loss": 1.2304,
+      "theoretical_loss": 3.410689049879954,
+      "tokens_seen": 2376531968
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002826191622532499,
+      "loss": 1.207,
+      "theoretical_loss": 3.4106740195177747,
+      "tokens_seen": 2376663040
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028257904028245866,
+      "loss": 1.227,
+      "theoretical_loss": 3.4106589902165734,
+      "tokens_seen": 2376794112
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028253891831166747,
+      "loss": 1.2887,
+      "theoretical_loss": 3.4106439619762163,
+      "tokens_seen": 2376925184
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002824987963408763,
+      "loss": 1.266,
+      "theoretical_loss": 3.410628934796571,
+      "tokens_seen": 2377056256
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028245867437008503,
+      "loss": 1.1851,
+      "theoretical_loss": 3.4106139086775036,
+      "tokens_seen": 2377187328
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002824185523992939,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4105988836188805,
+      "tokens_seen": 2377318400
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 798224,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.246376395225525,
+      "objective/train/theoretical_loss": 3.410583859620569,
+      "objective/train/tokens_used": 747908576,
+      "theoretical_loss": 3.410583859620569,
+      "tokens_seen": 2377449472
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028237843042850264,
+      "loss": 1.2306,
+      "theoretical_loss": 3.410583859620569,
+      "tokens_seen": 2377449472
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028233830845771145,
+      "loss": 1.2962,
+      "theoretical_loss": 3.410568836682435,
+      "tokens_seen": 2377580544
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028229818648692025,
+      "loss": 1.2152,
+      "theoretical_loss": 3.4105538148043464,
+      "tokens_seen": 2377711616
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028225806451612906,
+      "loss": 1.2366,
+      "theoretical_loss": 3.410538793986169,
+      "tokens_seen": 2377842688
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002822179425453378,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4105237742277703,
+      "tokens_seen": 2377973760
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002821778205745466,
+      "loss": 1.1673,
+      "theoretical_loss": 3.4105087555290163,
+      "tokens_seen": 2378104832
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002821376986037554,
+      "loss": 1.2596,
+      "theoretical_loss": 3.410493737889775,
+      "tokens_seen": 2378235904
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028209757663296423,
+      "loss": 1.3051,
+      "theoretical_loss": 3.4104787213099117,
+      "tokens_seen": 2378366976
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028205745466217304,
+      "loss": 1.2803,
+      "theoretical_loss": 3.410463705789295,
+      "tokens_seen": 2378498048
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002820173326913818,
+      "loss": 1.178,
+      "theoretical_loss": 3.4104486913277903,
+      "tokens_seen": 2378629120
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028197721072059065,
+      "loss": 1.2156,
+      "theoretical_loss": 3.4104336779252655,
+      "tokens_seen": 2378760192
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002819370887497994,
+      "loss": 1.1909,
+      "theoretical_loss": 3.4104186655815876,
+      "tokens_seen": 2378891264
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028189696677900815,
+      "loss": 1.2611,
+      "theoretical_loss": 3.410403654296623,
+      "tokens_seen": 2379022336
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000281856844808217,
+      "loss": 1.1905,
+      "theoretical_loss": 3.410388644070239,
+      "tokens_seen": 2379153408
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028181672283742577,
+      "loss": 1.2665,
+      "theoretical_loss": 3.410373634902303,
+      "tokens_seen": 2379284480
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028177660086663457,
+      "loss": 1.2892,
+      "theoretical_loss": 3.4103586267926813,
+      "tokens_seen": 2379415552
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002817364788958434,
+      "loss": 1.2668,
+      "theoretical_loss": 3.4103436197412416,
+      "tokens_seen": 2379546624
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002816963569250522,
+      "loss": 1.2905,
+      "theoretical_loss": 3.4103286137478506,
+      "tokens_seen": 2379677696
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028165623495426094,
+      "loss": 1.2204,
+      "theoretical_loss": 3.410313608812376,
+      "tokens_seen": 2379808768
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028161611298346974,
+      "loss": 1.2584,
+      "theoretical_loss": 3.410298604934684,
+      "tokens_seen": 2379939840
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028157599101267855,
+      "loss": 1.1841,
+      "theoretical_loss": 3.410283602114643,
+      "tokens_seen": 2380070912
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002815358690418873,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4102686003521194,
+      "tokens_seen": 2380201984
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028149574707109616,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4102535996469805,
+      "tokens_seen": 2380333056
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002814556251003049,
+      "loss": 1.2363,
+      "theoretical_loss": 3.4102385999990936,
+      "tokens_seen": 2380464128
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002814155031295138,
+      "loss": 1.2584,
+      "theoretical_loss": 3.410223601408326,
+      "tokens_seen": 2380595200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002813753811587225,
+      "loss": 1.2013,
+      "theoretical_loss": 3.410208603874545,
+      "tokens_seen": 2380726272
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002813352591879313,
+      "loss": 1.2924,
+      "theoretical_loss": 3.410193607397618,
+      "tokens_seen": 2380857344
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028129513721714014,
+      "loss": 1.2989,
+      "theoretical_loss": 3.4101786119774125,
+      "tokens_seen": 2380988416
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002812550152463489,
+      "loss": 1.2357,
+      "theoretical_loss": 3.4101636176137955,
+      "tokens_seen": 2381119488
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002812148932755577,
+      "loss": 1.2052,
+      "theoretical_loss": 3.410148624306635,
+      "tokens_seen": 2381250560
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002811747713047665,
+      "loss": 1.259,
+      "theoretical_loss": 3.410133632055797,
+      "tokens_seen": 2381381632
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002811346493339753,
+      "loss": 1.3079,
+      "theoretical_loss": 3.4101186408611506,
+      "tokens_seen": 2381512704
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028109452736318406,
+      "loss": 1.2028,
+      "theoretical_loss": 3.410103650722563,
+      "tokens_seen": 2381643776
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028105440539239287,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4100886616399007,
+      "tokens_seen": 2381774848
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002810142834216017,
+      "loss": 1.2544,
+      "theoretical_loss": 3.4100736736130317,
+      "tokens_seen": 2381905920
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028097416145081043,
+      "loss": 1.2453,
+      "theoretical_loss": 3.410058686641824,
+      "tokens_seen": 2382036992
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002809340394800193,
+      "loss": 1.2156,
+      "theoretical_loss": 3.410043700726145,
+      "tokens_seen": 2382168064
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028089391750922804,
+      "loss": 1.2721,
+      "theoretical_loss": 3.4100287158658618,
+      "tokens_seen": 2382299136
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002808537955384369,
+      "loss": 1.2684,
+      "theoretical_loss": 3.4100137320608424,
+      "tokens_seen": 2382430208
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028081367356764565,
+      "loss": 1.2197,
+      "theoretical_loss": 3.4099987493109545,
+      "tokens_seen": 2382561280
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028077355159685446,
+      "loss": 1.2263,
+      "theoretical_loss": 3.4099837676160654,
+      "tokens_seen": 2382692352
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028073342962606327,
+      "loss": 1.2608,
+      "theoretical_loss": 3.409968786976043,
+      "tokens_seen": 2382823424
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000280693307655272,
+      "loss": 1.2454,
+      "theoretical_loss": 3.409953807390755,
+      "tokens_seen": 2382954496
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002806531856844808,
+      "loss": 1.2868,
+      "theoretical_loss": 3.4099388288600694,
+      "tokens_seen": 2383085568
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028061306371368963,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4099238513838537,
+      "tokens_seen": 2383216640
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028057294174289844,
+      "loss": 1.2012,
+      "theoretical_loss": 3.4099088749619755,
+      "tokens_seen": 2383347712
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002805328197721072,
+      "loss": 1.1956,
+      "theoretical_loss": 3.4098938995943024,
+      "tokens_seen": 2383478784
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028049269780131605,
+      "loss": 1.2828,
+      "theoretical_loss": 3.409878925280703,
+      "tokens_seen": 2383609856
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002804525758305248,
+      "loss": 1.2254,
+      "theoretical_loss": 3.4098639520210448,
+      "tokens_seen": 2383740928
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028041245385973355,
+      "loss": 1.2209,
+      "theoretical_loss": 3.409848979815196,
+      "tokens_seen": 2383872000
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 800454,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2488914728164673,
+      "objective/train/theoretical_loss": 3.409834008663023,
+      "objective/train/tokens_used": 754462176,
+      "theoretical_loss": 3.409834008663023,
+      "tokens_seen": 2384003072
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002803723318889424,
+      "loss": 1.2012,
+      "theoretical_loss": 3.409834008663023,
+      "tokens_seen": 2384003072
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028033220991815117,
+      "loss": 1.2011,
+      "theoretical_loss": 3.409819038564396,
+      "tokens_seen": 2384134144
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028029208794735997,
+      "loss": 1.2912,
+      "theoretical_loss": 3.4098040695191814,
+      "tokens_seen": 2384265216
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002802519659765688,
+      "loss": 1.198,
+      "theoretical_loss": 3.4097891015272475,
+      "tokens_seen": 2384396288
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002802118440057776,
+      "loss": 1.2823,
+      "theoretical_loss": 3.4097741345884627,
+      "tokens_seen": 2384527360
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002801717220349864,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4097591687026942,
+      "tokens_seen": 2384658432
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028013160006419514,
+      "loss": 1.2467,
+      "theoretical_loss": 3.4097442038698107,
+      "tokens_seen": 2384789504
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028009147809340395,
+      "loss": 1.2642,
+      "theoretical_loss": 3.40972924008968,
+      "tokens_seen": 2384920576
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028005135612261276,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4097142773621703,
+      "tokens_seen": 2385051648
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028001123415182156,
+      "loss": 1.2107,
+      "theoretical_loss": 3.4096993156871496,
+      "tokens_seen": 2385182720
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002799711121810303,
+      "loss": 1.2324,
+      "theoretical_loss": 3.409684355064486,
+      "tokens_seen": 2385313792
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002799309902102392,
+      "loss": 1.2622,
+      "theoretical_loss": 3.4096693954940482,
+      "tokens_seen": 2385444864
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002798908682394479,
+      "loss": 1.1526,
+      "theoretical_loss": 3.409654436975704,
+      "tokens_seen": 2385575936
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002798507462686567,
+      "loss": 1.2305,
+      "theoretical_loss": 3.4096394795093214,
+      "tokens_seen": 2385707008
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027981062429786554,
+      "loss": 1.2642,
+      "theoretical_loss": 3.409624523094769,
+      "tokens_seen": 2385838080
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002797705023270743,
+      "loss": 1.2482,
+      "theoretical_loss": 3.4096095677319145,
+      "tokens_seen": 2385969152
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002797303803562831,
+      "loss": 1.2751,
+      "theoretical_loss": 3.4095946134206265,
+      "tokens_seen": 2386100224
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002796902583854919,
+      "loss": 1.2068,
+      "theoretical_loss": 3.4095796601607735,
+      "tokens_seen": 2386231296
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002796501364147007,
+      "loss": 1.201,
+      "theoretical_loss": 3.4095647079522236,
+      "tokens_seen": 2386362368
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027961001444390946,
+      "loss": 1.2344,
+      "theoretical_loss": 3.409549756794845,
+      "tokens_seen": 2386493440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027956989247311827,
+      "loss": 1.2391,
+      "theoretical_loss": 3.409534806688507,
+      "tokens_seen": 2386624512
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002795297705023271,
+      "loss": 1.1989,
+      "theoretical_loss": 3.4095198576330765,
+      "tokens_seen": 2386755584
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002794896485315359,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4095049096284233,
+      "tokens_seen": 2386886656
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002794495265607447,
+      "loss": 1.3103,
+      "theoretical_loss": 3.409489962674415,
+      "tokens_seen": 2387017728
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027940940458995344,
+      "loss": 1.2501,
+      "theoretical_loss": 3.4094750167709202,
+      "tokens_seen": 2387148800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002793692826191623,
+      "loss": 1.2504,
+      "theoretical_loss": 3.409460071917808,
+      "tokens_seen": 2387279872
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027932916064837105,
+      "loss": 1.2366,
+      "theoretical_loss": 3.409445128114946,
+      "tokens_seen": 2387410944
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002792890386775798,
+      "loss": 1.2718,
+      "theoretical_loss": 3.4094301853622033,
+      "tokens_seen": 2387542016
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027924891670678866,
+      "loss": 1.2186,
+      "theoretical_loss": 3.4094152436594483,
+      "tokens_seen": 2387673088
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002792087947359974,
+      "loss": 1.1647,
+      "theoretical_loss": 3.4094003030065494,
+      "tokens_seen": 2387804160
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002791686727652062,
+      "loss": 1.2897,
+      "theoretical_loss": 3.4093853634033757,
+      "tokens_seen": 2387935232
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027912855079441503,
+      "loss": 1.2877,
+      "theoretical_loss": 3.4093704248497954,
+      "tokens_seen": 2388066304
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027908842882362384,
+      "loss": 1.2773,
+      "theoretical_loss": 3.4093554873456773,
+      "tokens_seen": 2388197376
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002790483068528326,
+      "loss": 1.2605,
+      "theoretical_loss": 3.4093405508908905,
+      "tokens_seen": 2388328448
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027900818488204145,
+      "loss": 1.2601,
+      "theoretical_loss": 3.4093256154853027,
+      "tokens_seen": 2388459520
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002789680629112502,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4093106811287837,
+      "tokens_seen": 2388590592
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027892794094045895,
+      "loss": 1.214,
+      "theoretical_loss": 3.4092957478212016,
+      "tokens_seen": 2388721664
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002788878189696678,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4092808155624255,
+      "tokens_seen": 2388852736
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027884769699887657,
+      "loss": 1.2087,
+      "theoretical_loss": 3.409265884352324,
+      "tokens_seen": 2388983808
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002788075750280854,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4092509541907656,
+      "tokens_seen": 2389114880
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002787674530572942,
+      "loss": 1.304,
+      "theoretical_loss": 3.4092360250776204,
+      "tokens_seen": 2389245952
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.000278727331086503,
+      "loss": 1.2693,
+      "theoretical_loss": 3.4092210970127557,
+      "tokens_seen": 2389377024
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002786872091157118,
+      "loss": 1.2476,
+      "theoretical_loss": 3.4092061699960414,
+      "tokens_seen": 2389508096
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027864708714492054,
+      "loss": 1.2208,
+      "theoretical_loss": 3.4091912440273457,
+      "tokens_seen": 2389639168
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027860696517412935,
+      "loss": 1.2766,
+      "theoretical_loss": 3.4091763191065385,
+      "tokens_seen": 2389770240
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027856684320333816,
+      "loss": 1.2519,
+      "theoretical_loss": 3.409161395233488,
+      "tokens_seen": 2389901312
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027852672123254696,
+      "loss": 1.2484,
+      "theoretical_loss": 3.4091464724080636,
+      "tokens_seen": 2390032384
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002784865992617557,
+      "loss": 1.2075,
+      "theoretical_loss": 3.409131550630134,
+      "tokens_seen": 2390163456
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002784464772909646,
+      "loss": 1.2112,
+      "theoretical_loss": 3.409116629899568,
+      "tokens_seen": 2390294528
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002784063553201733,
+      "loss": 1.3238,
+      "theoretical_loss": 3.4091017102162353,
+      "tokens_seen": 2390425600
+    },
+    {
+      "epoch": 0.45,
+      "objective/train/docs_used": 802547,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2158358097076416,
+      "objective/train/theoretical_loss": 3.4090867915800045,
+      "objective/train/tokens_used": 761015776,
+      "theoretical_loss": 3.4090867915800045,
+      "tokens_seen": 2390556672
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002783662333493821,
+      "loss": 1.2375,
+      "theoretical_loss": 3.4090867915800045,
+      "tokens_seen": 2390556672
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027832611137859094,
+      "loss": 1.2521,
+      "theoretical_loss": 3.409071873990745,
+      "tokens_seen": 2390687744
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002782859894077997,
+      "loss": 1.2728,
+      "theoretical_loss": 3.409056957448326,
+      "tokens_seen": 2390818816
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027824586743700855,
+      "loss": 1.2266,
+      "theoretical_loss": 3.409042041952617,
+      "tokens_seen": 2390949888
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002782057454662173,
+      "loss": 1.2314,
+      "theoretical_loss": 3.4090271275034856,
+      "tokens_seen": 2391080960
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002781656234954261,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4090122141008026,
+      "tokens_seen": 2391212032
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002781255015246349,
+      "loss": 1.2719,
+      "theoretical_loss": 3.408997301744437,
+      "tokens_seen": 2391343104
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027808537955384367,
+      "loss": 1.2407,
+      "theoretical_loss": 3.4089823904342573,
+      "tokens_seen": 2391474176
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002780452575830525,
+      "loss": 1.2407,
+      "theoretical_loss": 3.408967480170133,
+      "tokens_seen": 2391605248
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002780051356122613,
+      "loss": 1.3088,
+      "theoretical_loss": 3.408952570951934,
+      "tokens_seen": 2391736320
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002779650136414701,
+      "loss": 1.2026,
+      "theoretical_loss": 3.408937662779529,
+      "tokens_seen": 2391867392
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027792489167067884,
+      "loss": 1.262,
+      "theoretical_loss": 3.408922755652788,
+      "tokens_seen": 2391998464
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002778847696998877,
+      "loss": 1.206,
+      "theoretical_loss": 3.4089078495715794,
+      "tokens_seen": 2392129536
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027784464772909645,
+      "loss": 1.2788,
+      "theoretical_loss": 3.408892944535773,
+      "tokens_seen": 2392260608
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002778045257583052,
+      "loss": 1.2847,
+      "theoretical_loss": 3.408878040545239,
+      "tokens_seen": 2392391680
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027776440378751406,
+      "loss": 1.2744,
+      "theoretical_loss": 3.4088631375998455,
+      "tokens_seen": 2392522752
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002777242818167228,
+      "loss": 1.2941,
+      "theoretical_loss": 3.408848235699463,
+      "tokens_seen": 2392653824
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002776841598459316,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4088333348439606,
+      "tokens_seen": 2392784896
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027764403787514043,
+      "loss": 1.3114,
+      "theoretical_loss": 3.4088184350332074,
+      "tokens_seen": 2392915968
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027760391590434924,
+      "loss": 1.1612,
+      "theoretical_loss": 3.4088035362670737,
+      "tokens_seen": 2393047040
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027756379393355804,
+      "loss": 1.2363,
+      "theoretical_loss": 3.408788638545429,
+      "tokens_seen": 2393178112
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027752367196276685,
+      "loss": 1.2318,
+      "theoretical_loss": 3.408773741868142,
+      "tokens_seen": 2393309184
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002774835499919756,
+      "loss": 1.187,
+      "theoretical_loss": 3.408758846235083,
+      "tokens_seen": 2393440256
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002774434280211844,
+      "loss": 1.2928,
+      "theoretical_loss": 3.4087439516461213,
+      "tokens_seen": 2393571328
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002774033060503932,
+      "loss": 1.2294,
+      "theoretical_loss": 3.408729058101127,
+      "tokens_seen": 2393702400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027736318407960196,
+      "loss": 1.2685,
+      "theoretical_loss": 3.4087141655999695,
+      "tokens_seen": 2393833472
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002773230621088108,
+      "loss": 1.2977,
+      "theoretical_loss": 3.4086992741425184,
+      "tokens_seen": 2393964544
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002772829401380196,
+      "loss": 1.2174,
+      "theoretical_loss": 3.4086843837286436,
+      "tokens_seen": 2394095616
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002772428181672284,
+      "loss": 1.3279,
+      "theoretical_loss": 3.408669494358215,
+      "tokens_seen": 2394226688
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002772026961964372,
+      "loss": 1.2267,
+      "theoretical_loss": 3.4086546060311016,
+      "tokens_seen": 2394357760
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027716257422564594,
+      "loss": 1.2439,
+      "theoretical_loss": 3.4086397187471738,
+      "tokens_seen": 2394488832
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027712245225485475,
+      "loss": 1.2651,
+      "theoretical_loss": 3.4086248325063018,
+      "tokens_seen": 2394619904
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027708233028406355,
+      "loss": 1.2024,
+      "theoretical_loss": 3.408609947308354,
+      "tokens_seen": 2394750976
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027704220831327236,
+      "loss": 1.2833,
+      "theoretical_loss": 3.4085950631532023,
+      "tokens_seen": 2394882048
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027700208634248117,
+      "loss": 1.2258,
+      "theoretical_loss": 3.4085801800407145,
+      "tokens_seen": 2395013120
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027696196437169,
+      "loss": 1.159,
+      "theoretical_loss": 3.408565297970762,
+      "tokens_seen": 2395144192
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002769218424008987,
+      "loss": 1.2342,
+      "theoretical_loss": 3.408550416943214,
+      "tokens_seen": 2395275264
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027688172043010753,
+      "loss": 1.248,
+      "theoretical_loss": 3.408535536957941,
+      "tokens_seen": 2395406336
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027684159845931634,
+      "loss": 1.2222,
+      "theoretical_loss": 3.4085206580148126,
+      "tokens_seen": 2395537408
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002768014764885251,
+      "loss": 1.3271,
+      "theoretical_loss": 3.4085057801136984,
+      "tokens_seen": 2395668480
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027676135451773395,
+      "loss": 1.2435,
+      "theoretical_loss": 3.408490903254469,
+      "tokens_seen": 2395799552
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002767212325469427,
+      "loss": 1.267,
+      "theoretical_loss": 3.4084760274369943,
+      "tokens_seen": 2395930624
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002766811105761515,
+      "loss": 1.2662,
+      "theoretical_loss": 3.4084611526611446,
+      "tokens_seen": 2396061696
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002766409886053603,
+      "loss": 1.2159,
+      "theoretical_loss": 3.4084462789267898,
+      "tokens_seen": 2396192768
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027660086663456907,
+      "loss": 1.3477,
+      "theoretical_loss": 3.4084314062337997,
+      "tokens_seen": 2396323840
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002765607446637779,
+      "loss": 1.2735,
+      "theoretical_loss": 3.4084165345820447,
+      "tokens_seen": 2396454912
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002765206226929867,
+      "loss": 1.3411,
+      "theoretical_loss": 3.4084016639713948,
+      "tokens_seen": 2396585984
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002764805007221955,
+      "loss": 1.2574,
+      "theoretical_loss": 3.4083867944017205,
+      "tokens_seen": 2396717056
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027644037875140424,
+      "loss": 1.2205,
+      "theoretical_loss": 3.408371925872892,
+      "tokens_seen": 2396848128
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002764002567806131,
+      "loss": 1.232,
+      "theoretical_loss": 3.4083570583847793,
+      "tokens_seen": 2396979200
+    },
+    {
+      "epoch": 0.45,
+      "objective/train/docs_used": 804843,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2514108419418335,
+      "objective/train/theoretical_loss": 3.4083421919372525,
+      "objective/train/tokens_used": 767569376,
+      "theoretical_loss": 3.4083421919372525,
+      "tokens_seen": 2397110272
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027636013480982185,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4083421919372525,
+      "tokens_seen": 2397110272
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027632001283903066,
+      "loss": 1.2074,
+      "theoretical_loss": 3.4083273265301823,
+      "tokens_seen": 2397241344
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027627989086823946,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4083124621634386,
+      "tokens_seen": 2397372416
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002762397688974482,
+      "loss": 1.3225,
+      "theoretical_loss": 3.408297598836892,
+      "tokens_seen": 2397503488
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002761996469266571,
+      "loss": 1.2017,
+      "theoretical_loss": 3.4082827365504134,
+      "tokens_seen": 2397634560
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027615952495586583,
+      "loss": 1.2881,
+      "theoretical_loss": 3.4082678753038715,
+      "tokens_seen": 2397765632
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027611940298507464,
+      "loss": 1.1997,
+      "theoretical_loss": 3.408253015097139,
+      "tokens_seen": 2397896704
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027607928101428344,
+      "loss": 1.2388,
+      "theoretical_loss": 3.408238155930084,
+      "tokens_seen": 2398027776
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027603915904349225,
+      "loss": 1.2331,
+      "theoretical_loss": 3.408223297802578,
+      "tokens_seen": 2398158848
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.000275999037072701,
+      "loss": 1.222,
+      "theoretical_loss": 3.4082084407144917,
+      "tokens_seen": 2398289920
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002759589151019098,
+      "loss": 1.2459,
+      "theoretical_loss": 3.4081935846656957,
+      "tokens_seen": 2398420992
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002759187931311186,
+      "loss": 1.1911,
+      "theoretical_loss": 3.4081787296560595,
+      "tokens_seen": 2398552064
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027587867116032736,
+      "loss": 1.2478,
+      "theoretical_loss": 3.408163875685455,
+      "tokens_seen": 2398683136
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002758385491895362,
+      "loss": 1.3087,
+      "theoretical_loss": 3.4081490227537516,
+      "tokens_seen": 2398814208
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.000275798427218745,
+      "loss": 1.2328,
+      "theoretical_loss": 3.40813417086082,
+      "tokens_seen": 2398945280
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027575830524795384,
+      "loss": 1.2878,
+      "theoretical_loss": 3.4081193200065316,
+      "tokens_seen": 2399076352
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002757181832771626,
+      "loss": 1.2511,
+      "theoretical_loss": 3.4081044701907564,
+      "tokens_seen": 2399207424
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027567806130637134,
+      "loss": 1.324,
+      "theoretical_loss": 3.408089621413365,
+      "tokens_seen": 2399338496
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002756379393355802,
+      "loss": 1.1934,
+      "theoretical_loss": 3.4080747736742287,
+      "tokens_seen": 2399469568
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027559781736478895,
+      "loss": 1.1819,
+      "theoretical_loss": 3.4080599269732175,
+      "tokens_seen": 2399600640
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027555769539399776,
+      "loss": 1.2872,
+      "theoretical_loss": 3.4080450813102026,
+      "tokens_seen": 2399731712
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027551757342320657,
+      "loss": 1.2312,
+      "theoretical_loss": 3.408030236685054,
+      "tokens_seen": 2399862784
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002754774514524154,
+      "loss": 1.2471,
+      "theoretical_loss": 3.408015393097643,
+      "tokens_seen": 2399993856
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002754373294816241,
+      "loss": 1.1896,
+      "theoretical_loss": 3.408000550547841,
+      "tokens_seen": 2400124928
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027539720751083293,
+      "loss": 1.2941,
+      "theoretical_loss": 3.4079857090355175,
+      "tokens_seen": 2400256000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027535708554004174,
+      "loss": 1.2139,
+      "theoretical_loss": 3.407970868560544,
+      "tokens_seen": 2400387072
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002753169635692505,
+      "loss": 1.1835,
+      "theoretical_loss": 3.4079560291227917,
+      "tokens_seen": 2400518144
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027527684159845935,
+      "loss": 1.2363,
+      "theoretical_loss": 3.407941190722131,
+      "tokens_seen": 2400649216
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002752367196276681,
+      "loss": 1.249,
+      "theoretical_loss": 3.4079263533584334,
+      "tokens_seen": 2400780288
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002751965976568769,
+      "loss": 1.2307,
+      "theoretical_loss": 3.407911517031569,
+      "tokens_seen": 2400911360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002751564756860857,
+      "loss": 1.2032,
+      "theoretical_loss": 3.4078966817414087,
+      "tokens_seen": 2401042432
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027511635371529447,
+      "loss": 1.1964,
+      "theoretical_loss": 3.4078818474878245,
+      "tokens_seen": 2401173504
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027507623174450333,
+      "loss": 1.2385,
+      "theoretical_loss": 3.4078670142706864,
+      "tokens_seen": 2401304576
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002750361097737121,
+      "loss": 1.2865,
+      "theoretical_loss": 3.407852182089866,
+      "tokens_seen": 2401435648
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002749959878029209,
+      "loss": 1.3314,
+      "theoretical_loss": 3.407837350945234,
+      "tokens_seen": 2401566720
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002749558658321297,
+      "loss": 1.2127,
+      "theoretical_loss": 3.4078225208366617,
+      "tokens_seen": 2401697792
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002749157438613385,
+      "loss": 1.3594,
+      "theoretical_loss": 3.40780769176402,
+      "tokens_seen": 2401828864
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027487562189054725,
+      "loss": 1.2046,
+      "theoretical_loss": 3.40779286372718,
+      "tokens_seen": 2401959936
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027483549991975606,
+      "loss": 1.2176,
+      "theoretical_loss": 3.4077780367260133,
+      "tokens_seen": 2402091008
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027479537794896486,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4077632107603906,
+      "tokens_seen": 2402222080
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002747552559781736,
+      "loss": 1.2212,
+      "theoretical_loss": 3.4077483858301827,
+      "tokens_seen": 2402353152
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002747151340073825,
+      "loss": 1.2915,
+      "theoretical_loss": 3.407733561935262,
+      "tokens_seen": 2402484224
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027467501203659123,
+      "loss": 1.2842,
+      "theoretical_loss": 3.407718739075498,
+      "tokens_seen": 2402615296
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027463489006580003,
+      "loss": 1.2688,
+      "theoretical_loss": 3.407703917250764,
+      "tokens_seen": 2402746368
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027459476809500884,
+      "loss": 1.3079,
+      "theoretical_loss": 3.4076890964609294,
+      "tokens_seen": 2402877440
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002745546461242176,
+      "loss": 1.1836,
+      "theoretical_loss": 3.4076742767058668,
+      "tokens_seen": 2403008512
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002745145241534264,
+      "loss": 1.1897,
+      "theoretical_loss": 3.4076594579854467,
+      "tokens_seen": 2403139584
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002744744021826352,
+      "loss": 1.2228,
+      "theoretical_loss": 3.4076446402995413,
+      "tokens_seen": 2403270656
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000274434280211844,
+      "loss": 1.181,
+      "theoretical_loss": 3.407629823648021,
+      "tokens_seen": 2403401728
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002743941582410528,
+      "loss": 1.2445,
+      "theoretical_loss": 3.4076150080307577,
+      "tokens_seen": 2403532800
+    },
+    {
+      "epoch": 0.46,
+      "objective/train/docs_used": 806890,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2349939346313477,
+      "objective/train/theoretical_loss": 3.4076001934476228,
+      "objective/train/tokens_used": 774122976,
+      "theoretical_loss": 3.4076001934476228,
+      "tokens_seen": 2403663872
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002743540362702616,
+      "loss": 1.2293,
+      "theoretical_loss": 3.4076001934476228,
+      "tokens_seen": 2403663872
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002743139142994704,
+      "loss": 1.1782,
+      "theoretical_loss": 3.407585379898488,
+      "tokens_seen": 2403794944
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027427379232867924,
+      "loss": 1.2461,
+      "theoretical_loss": 3.4075705673832237,
+      "tokens_seen": 2403926016
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000274233670357888,
+      "loss": 1.2251,
+      "theoretical_loss": 3.4075557559017025,
+      "tokens_seen": 2404057088
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027419354838709674,
+      "loss": 1.246,
+      "theoretical_loss": 3.4075409454537953,
+      "tokens_seen": 2404188160
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002741534264163056,
+      "loss": 1.2017,
+      "theoretical_loss": 3.4075261360393743,
+      "tokens_seen": 2404319232
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027411330444551435,
+      "loss": 1.2054,
+      "theoretical_loss": 3.40751132765831,
+      "tokens_seen": 2404450304
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027407318247472316,
+      "loss": 1.2057,
+      "theoretical_loss": 3.407496520310475,
+      "tokens_seen": 2404581376
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027403306050393197,
+      "loss": 1.1922,
+      "theoretical_loss": 3.40748171399574,
+      "tokens_seen": 2404712448
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002739929385331408,
+      "loss": 1.2035,
+      "theoretical_loss": 3.4074669087139773,
+      "tokens_seen": 2404843520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002739528165623495,
+      "loss": 1.1943,
+      "theoretical_loss": 3.407452104465058,
+      "tokens_seen": 2404974592
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027391269459155833,
+      "loss": 1.206,
+      "theoretical_loss": 3.4074373012488546,
+      "tokens_seen": 2405105664
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027387257262076714,
+      "loss": 1.1895,
+      "theoretical_loss": 3.407422499065238,
+      "tokens_seen": 2405236736
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027383245064997594,
+      "loss": 1.2326,
+      "theoretical_loss": 3.40740769791408,
+      "tokens_seen": 2405367808
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027379232867918475,
+      "loss": 1.1878,
+      "theoretical_loss": 3.4073928977952526,
+      "tokens_seen": 2405498880
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002737522067083935,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4073780987086275,
+      "tokens_seen": 2405629952
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027371208473760236,
+      "loss": 1.1906,
+      "theoretical_loss": 3.407363300654076,
+      "tokens_seen": 2405761024
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002736719627668111,
+      "loss": 1.1534,
+      "theoretical_loss": 3.407348503631471,
+      "tokens_seen": 2405892096
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027363184079601987,
+      "loss": 1.2139,
+      "theoretical_loss": 3.4073337076406833,
+      "tokens_seen": 2406023168
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027359171882522873,
+      "loss": 1.2444,
+      "theoretical_loss": 3.407318912681585,
+      "tokens_seen": 2406154240
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002735515968544375,
+      "loss": 1.2058,
+      "theoretical_loss": 3.4073041187540483,
+      "tokens_seen": 2406285312
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002735114748836463,
+      "loss": 1.3028,
+      "theoretical_loss": 3.4072893258579446,
+      "tokens_seen": 2406416384
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002734713529128551,
+      "loss": 1.1733,
+      "theoretical_loss": 3.407274533993146,
+      "tokens_seen": 2406547456
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002734312309420639,
+      "loss": 1.2934,
+      "theoretical_loss": 3.4072597431595244,
+      "tokens_seen": 2406678528
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027339110897127265,
+      "loss": 1.2288,
+      "theoretical_loss": 3.407244953356952,
+      "tokens_seen": 2406809600
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027335098700048146,
+      "loss": 1.1837,
+      "theoretical_loss": 3.407230164585301,
+      "tokens_seen": 2406940672
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027331086502969026,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4072153768444426,
+      "tokens_seen": 2407071744
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000273270743058899,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4072005901342495,
+      "tokens_seen": 2407202816
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002732306210881079,
+      "loss": 1.2432,
+      "theoretical_loss": 3.4071858044545933,
+      "tokens_seen": 2407333888
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027319049911731663,
+      "loss": 1.1963,
+      "theoretical_loss": 3.407171019805346,
+      "tokens_seen": 2407464960
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002731503771465255,
+      "loss": 1.2112,
+      "theoretical_loss": 3.4071562361863807,
+      "tokens_seen": 2407596032
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027311025517573424,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4071414535975686,
+      "tokens_seen": 2407727104
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000273070133204943,
+      "loss": 1.2735,
+      "theoretical_loss": 3.407126672038782,
+      "tokens_seen": 2407858176
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027303001123415185,
+      "loss": 1.1797,
+      "theoretical_loss": 3.407111891509893,
+      "tokens_seen": 2407989248
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002729898892633606,
+      "loss": 1.2017,
+      "theoretical_loss": 3.4070971120107743,
+      "tokens_seen": 2408120320
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002729497672925694,
+      "loss": 1.2726,
+      "theoretical_loss": 3.407082333541297,
+      "tokens_seen": 2408251392
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002729096453217782,
+      "loss": 1.338,
+      "theoretical_loss": 3.407067556101335,
+      "tokens_seen": 2408382464
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000272869523350987,
+      "loss": 1.2039,
+      "theoretical_loss": 3.407052779690759,
+      "tokens_seen": 2408513536
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002728294013801958,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4070380043094417,
+      "tokens_seen": 2408644608
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027278927940940464,
+      "loss": 1.1794,
+      "theoretical_loss": 3.407023229957256,
+      "tokens_seen": 2408775680
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002727491574386134,
+      "loss": 1.24,
+      "theoretical_loss": 3.407008456634074,
+      "tokens_seen": 2408906752
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027270903546782214,
+      "loss": 1.2727,
+      "theoretical_loss": 3.406993684339767,
+      "tokens_seen": 2409037824
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000272668913497031,
+      "loss": 1.1755,
+      "theoretical_loss": 3.406978913074209,
+      "tokens_seen": 2409168896
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027262879152623975,
+      "loss": 1.2234,
+      "theoretical_loss": 3.4069641428372717,
+      "tokens_seen": 2409299968
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002725886695554486,
+      "loss": 1.2225,
+      "theoretical_loss": 3.406949373628827,
+      "tokens_seen": 2409431040
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027254854758465737,
+      "loss": 1.2421,
+      "theoretical_loss": 3.406934605448748,
+      "tokens_seen": 2409562112
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027250842561386617,
+      "loss": 1.2336,
+      "theoretical_loss": 3.4069198382969073,
+      "tokens_seen": 2409693184
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000272468303643075,
+      "loss": 1.2366,
+      "theoretical_loss": 3.4069050721731764,
+      "tokens_seen": 2409824256
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027242818167228373,
+      "loss": 1.2887,
+      "theoretical_loss": 3.406890307077429,
+      "tokens_seen": 2409955328
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027238805970149254,
+      "loss": 1.2416,
+      "theoretical_loss": 3.4068755430095368,
+      "tokens_seen": 2410086400
+    },
+    {
+      "epoch": 0.46,
+      "objective/train/docs_used": 808982,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1325750350952148,
+      "objective/train/theoretical_loss": 3.4068607799693726,
+      "objective/train/tokens_used": 780676576,
+      "theoretical_loss": 3.4068607799693726,
+      "tokens_seen": 2410217472
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027234793773070134,
+      "loss": 1.1479,
+      "theoretical_loss": 3.4068607799693726,
+      "tokens_seen": 2410217472
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027230781575991015,
+      "loss": 1.2302,
+      "theoretical_loss": 3.406846017956809,
+      "tokens_seen": 2410348544
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002722676937891189,
+      "loss": 1.2932,
+      "theoretical_loss": 3.406831256971719,
+      "tokens_seen": 2410479616
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027222757181832776,
+      "loss": 1.2171,
+      "theoretical_loss": 3.406816497013974,
+      "tokens_seen": 2410610688
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002721874498475365,
+      "loss": 1.1999,
+      "theoretical_loss": 3.4068017380834483,
+      "tokens_seen": 2410741760
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027214732787674527,
+      "loss": 1.2968,
+      "theoretical_loss": 3.406786980180013,
+      "tokens_seen": 2410872832
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027210720590595413,
+      "loss": 1.2441,
+      "theoretical_loss": 3.406772223303542,
+      "tokens_seen": 2411003904
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002720670839351629,
+      "loss": 1.221,
+      "theoretical_loss": 3.406757467453908,
+      "tokens_seen": 2411134976
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002720269619643717,
+      "loss": 1.2695,
+      "theoretical_loss": 3.4067427126309826,
+      "tokens_seen": 2411266048
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002719868399935805,
+      "loss": 1.2418,
+      "theoretical_loss": 3.4067279588346393,
+      "tokens_seen": 2411397120
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002719467180227893,
+      "loss": 1.2613,
+      "theoretical_loss": 3.406713206064751,
+      "tokens_seen": 2411528192
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002719065960519981,
+      "loss": 1.3202,
+      "theoretical_loss": 3.40669845432119,
+      "tokens_seen": 2411659264
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027186647408120686,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4066837036038295,
+      "tokens_seen": 2411790336
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027182635211041566,
+      "loss": 1.266,
+      "theoretical_loss": 3.4066689539125425,
+      "tokens_seen": 2411921408
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027178623013962447,
+      "loss": 1.2238,
+      "theoretical_loss": 3.4066542052472015,
+      "tokens_seen": 2412052480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002717461081688333,
+      "loss": 1.2967,
+      "theoretical_loss": 3.40663945760768,
+      "tokens_seen": 2412183552
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027170598619804203,
+      "loss": 1.2761,
+      "theoretical_loss": 3.40662471099385,
+      "tokens_seen": 2412314624
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002716658642272509,
+      "loss": 1.2308,
+      "theoretical_loss": 3.406609965405585,
+      "tokens_seen": 2412445696
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027162574225645964,
+      "loss": 1.2078,
+      "theoretical_loss": 3.4065952208427577,
+      "tokens_seen": 2412576768
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002715856202856684,
+      "loss": 1.2845,
+      "theoretical_loss": 3.4065804773052415,
+      "tokens_seen": 2412707840
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027154549831487725,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4065657347929093,
+      "tokens_seen": 2412838912
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000271505376344086,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4065509933056335,
+      "tokens_seen": 2412969984
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002714652543732948,
+      "loss": 1.3213,
+      "theoretical_loss": 3.406536252843288,
+      "tokens_seen": 2413101056
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002714251324025036,
+      "loss": 1.2275,
+      "theoretical_loss": 3.4065215134057456,
+      "tokens_seen": 2413232128
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002713850104317124,
+      "loss": 1.206,
+      "theoretical_loss": 3.406506774992879,
+      "tokens_seen": 2413363200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002713448884609212,
+      "loss": 1.2211,
+      "theoretical_loss": 3.406492037604562,
+      "tokens_seen": 2413494272
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027130476649013004,
+      "loss": 1.2746,
+      "theoretical_loss": 3.4064773012406673,
+      "tokens_seen": 2413625344
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002712646445193388,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4064625659010677,
+      "tokens_seen": 2413756416
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002712245225485476,
+      "loss": 1.2614,
+      "theoretical_loss": 3.4064478315856372,
+      "tokens_seen": 2413887488
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002711844005777564,
+      "loss": 1.2033,
+      "theoretical_loss": 3.4064330982942486,
+      "tokens_seen": 2414018560
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027114427860696515,
+      "loss": 1.2019,
+      "theoretical_loss": 3.4064183660267755,
+      "tokens_seen": 2414149632
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000271104156636174,
+      "loss": 1.3275,
+      "theoretical_loss": 3.40640363478309,
+      "tokens_seen": 2414280704
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027106403466538277,
+      "loss": 1.2714,
+      "theoretical_loss": 3.406388904563067,
+      "tokens_seen": 2414411776
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027102391269459157,
+      "loss": 1.2617,
+      "theoretical_loss": 3.4063741753665786,
+      "tokens_seen": 2414542848
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002709837907238004,
+      "loss": 1.2823,
+      "theoretical_loss": 3.406359447193499,
+      "tokens_seen": 2414673920
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027094366875300913,
+      "loss": 1.3388,
+      "theoretical_loss": 3.4063447200437005,
+      "tokens_seen": 2414804992
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027090354678221794,
+      "loss": 1.255,
+      "theoretical_loss": 3.406329993917057,
+      "tokens_seen": 2414936064
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027086342481142674,
+      "loss": 1.202,
+      "theoretical_loss": 3.4063152688134424,
+      "tokens_seen": 2415067136
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027082330284063555,
+      "loss": 1.2349,
+      "theoretical_loss": 3.4063005447327295,
+      "tokens_seen": 2415198208
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002707831808698443,
+      "loss": 1.2175,
+      "theoretical_loss": 3.4062858216747918,
+      "tokens_seen": 2415329280
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027074305889905316,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4062710996395027,
+      "tokens_seen": 2415460352
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002707029369282619,
+      "loss": 1.2237,
+      "theoretical_loss": 3.406256378626736,
+      "tokens_seen": 2415591424
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027066281495747067,
+      "loss": 1.2714,
+      "theoretical_loss": 3.406241658636365,
+      "tokens_seen": 2415722496
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002706226929866795,
+      "loss": 1.1927,
+      "theoretical_loss": 3.4062269396682634,
+      "tokens_seen": 2415853568
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002705825710158883,
+      "loss": 1.1686,
+      "theoretical_loss": 3.4062122217223045,
+      "tokens_seen": 2415984640
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027054244904509714,
+      "loss": 1.2596,
+      "theoretical_loss": 3.406197504798362,
+      "tokens_seen": 2416115712
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002705023270743059,
+      "loss": 1.1984,
+      "theoretical_loss": 3.406182788896309,
+      "tokens_seen": 2416246784
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002704622051035147,
+      "loss": 1.3171,
+      "theoretical_loss": 3.40616807401602,
+      "tokens_seen": 2416377856
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002704220831327235,
+      "loss": 1.2084,
+      "theoretical_loss": 3.406153360157368,
+      "tokens_seen": 2416508928
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027038196116193226,
+      "loss": 1.2499,
+      "theoretical_loss": 3.406138647320227,
+      "tokens_seen": 2416640000
+    },
+    {
+      "epoch": 0.46,
+      "objective/train/docs_used": 811333,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2398498058319092,
+      "objective/train/theoretical_loss": 3.406123935504471,
+      "objective/train/tokens_used": 787230176,
+      "theoretical_loss": 3.406123935504471,
+      "tokens_seen": 2416771072
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027034183919114106,
+      "loss": 1.2432,
+      "theoretical_loss": 3.406123935504471,
+      "tokens_seen": 2416771072
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027030171722034987,
+      "loss": 1.2181,
+      "theoretical_loss": 3.4061092247099727,
+      "tokens_seen": 2416902144
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002702615952495587,
+      "loss": 1.2169,
+      "theoretical_loss": 3.4060945149366066,
+      "tokens_seen": 2417033216
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027022147327876743,
+      "loss": 1.2225,
+      "theoretical_loss": 3.406079806184246,
+      "tokens_seen": 2417164288
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002701813513079763,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4060650984527654,
+      "tokens_seen": 2417295360
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00027014122933718504,
+      "loss": 1.2179,
+      "theoretical_loss": 3.406050391742038,
+      "tokens_seen": 2417426432
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002701011073663938,
+      "loss": 1.182,
+      "theoretical_loss": 3.4060356860519376,
+      "tokens_seen": 2417557504
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00027006098539560265,
+      "loss": 1.3326,
+      "theoretical_loss": 3.406020981382338,
+      "tokens_seen": 2417688576
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002700208634248114,
+      "loss": 1.1903,
+      "theoretical_loss": 3.4060062777331135,
+      "tokens_seen": 2417819648
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026998074145402027,
+      "loss": 1.2089,
+      "theoretical_loss": 3.4059915751041374,
+      "tokens_seen": 2417950720
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.000269940619483229,
+      "loss": 1.2529,
+      "theoretical_loss": 3.4059768734952844,
+      "tokens_seen": 2418081792
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002699004975124378,
+      "loss": 1.195,
+      "theoretical_loss": 3.4059621729064276,
+      "tokens_seen": 2418212864
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026986037554164663,
+      "loss": 1.2114,
+      "theoretical_loss": 3.4059474733374415,
+      "tokens_seen": 2418343936
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002698202535708554,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4059327747882,
+      "tokens_seen": 2418475008
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002697801316000642,
+      "loss": 1.2153,
+      "theoretical_loss": 3.4059180772585766,
+      "tokens_seen": 2418606080
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.000269740009629273,
+      "loss": 1.3286,
+      "theoretical_loss": 3.405903380748446,
+      "tokens_seen": 2418737152
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002696998876584818,
+      "loss": 1.3026,
+      "theoretical_loss": 3.4058886852576817,
+      "tokens_seen": 2418868224
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026965976568769055,
+      "loss": 1.1805,
+      "theoretical_loss": 3.405873990786158,
+      "tokens_seen": 2418999296
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002696196437168994,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4058592973337496,
+      "tokens_seen": 2419130368
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026957952174610817,
+      "loss": 1.2719,
+      "theoretical_loss": 3.4058446049003295,
+      "tokens_seen": 2419261440
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026953939977531697,
+      "loss": 1.2146,
+      "theoretical_loss": 3.405829913485772,
+      "tokens_seen": 2419392512
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002694992778045258,
+      "loss": 1.2144,
+      "theoretical_loss": 3.4058152230899523,
+      "tokens_seen": 2419523584
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026945915583373453,
+      "loss": 1.2652,
+      "theoretical_loss": 3.4058005337127435,
+      "tokens_seen": 2419654656
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026941903386294334,
+      "loss": 1.2536,
+      "theoretical_loss": 3.4057858453540204,
+      "tokens_seen": 2419785728
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026937891189215214,
+      "loss": 1.267,
+      "theoretical_loss": 3.405771158013657,
+      "tokens_seen": 2419916800
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026933878992136095,
+      "loss": 1.3031,
+      "theoretical_loss": 3.405756471691527,
+      "tokens_seen": 2420047872
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026929866795056976,
+      "loss": 1.1804,
+      "theoretical_loss": 3.4057417863875057,
+      "tokens_seen": 2420178944
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026925854597977856,
+      "loss": 1.255,
+      "theoretical_loss": 3.405727102101467,
+      "tokens_seen": 2420310016
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002692184240089873,
+      "loss": 1.2088,
+      "theoretical_loss": 3.405712418833285,
+      "tokens_seen": 2420441088
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002691783020381961,
+      "loss": 1.1598,
+      "theoretical_loss": 3.405697736582834,
+      "tokens_seen": 2420572160
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002691381800674049,
+      "loss": 1.2106,
+      "theoretical_loss": 3.4056830553499884,
+      "tokens_seen": 2420703232
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002690980580966137,
+      "loss": 1.2046,
+      "theoretical_loss": 3.4056683751346224,
+      "tokens_seen": 2420834304
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026905793612582254,
+      "loss": 1.2259,
+      "theoretical_loss": 3.405653695936611,
+      "tokens_seen": 2420965376
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002690178141550313,
+      "loss": 1.1777,
+      "theoretical_loss": 3.4056390177558287,
+      "tokens_seen": 2421096448
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002689776921842401,
+      "loss": 1.247,
+      "theoretical_loss": 3.4056243405921487,
+      "tokens_seen": 2421227520
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002689375702134489,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4056096644454468,
+      "tokens_seen": 2421358592
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026889744824265766,
+      "loss": 1.2233,
+      "theoretical_loss": 3.405594989315597,
+      "tokens_seen": 2421489664
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026885732627186646,
+      "loss": 1.2423,
+      "theoretical_loss": 3.4055803152024735,
+      "tokens_seen": 2421620736
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026881720430107527,
+      "loss": 1.292,
+      "theoretical_loss": 3.4055656421059517,
+      "tokens_seen": 2421751808
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002687770823302841,
+      "loss": 1.223,
+      "theoretical_loss": 3.405550970025905,
+      "tokens_seen": 2421882880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002687369603594929,
+      "loss": 1.2652,
+      "theoretical_loss": 3.405536298962209,
+      "tokens_seen": 2422013952
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002686968383887017,
+      "loss": 1.2247,
+      "theoretical_loss": 3.4055216289147374,
+      "tokens_seen": 2422145024
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026865671641791044,
+      "loss": 1.2833,
+      "theoretical_loss": 3.4055069598833656,
+      "tokens_seen": 2422276096
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026861659444711925,
+      "loss": 1.2379,
+      "theoretical_loss": 3.4054922918679678,
+      "tokens_seen": 2422407168
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026857647247632805,
+      "loss": 1.2063,
+      "theoretical_loss": 3.4054776248684187,
+      "tokens_seen": 2422538240
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002685363505055368,
+      "loss": 1.2202,
+      "theoretical_loss": 3.4054629588845935,
+      "tokens_seen": 2422669312
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026849622853474566,
+      "loss": 1.2711,
+      "theoretical_loss": 3.405448293916366,
+      "tokens_seen": 2422800384
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002684561065639544,
+      "loss": 1.2527,
+      "theoretical_loss": 3.405433629963612,
+      "tokens_seen": 2422931456
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002684159845931632,
+      "loss": 1.2452,
+      "theoretical_loss": 3.4054189670262054,
+      "tokens_seen": 2423062528
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026837586262237203,
+      "loss": 1.2662,
+      "theoretical_loss": 3.405404305104021,
+      "tokens_seen": 2423193600
+    },
+    {
+      "epoch": 0.47,
+      "objective/train/docs_used": 813659,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2853989601135254,
+      "objective/train/theoretical_loss": 3.405389644196934,
+      "objective/train/tokens_used": 793783776,
+      "theoretical_loss": 3.405389644196934,
+      "tokens_seen": 2423324672
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002683357406515808,
+      "loss": 1.2203,
+      "theoretical_loss": 3.405389644196934,
+      "tokens_seen": 2423324672
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002682956186807896,
+      "loss": 1.2307,
+      "theoretical_loss": 3.405374984304819,
+      "tokens_seen": 2423455744
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002682554967099984,
+      "loss": 1.2153,
+      "theoretical_loss": 3.4053603254275515,
+      "tokens_seen": 2423586816
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002682153747392072,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4053456675650056,
+      "tokens_seen": 2423717888
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026817525276841595,
+      "loss": 1.236,
+      "theoretical_loss": 3.405331010717056,
+      "tokens_seen": 2423848960
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002681351307976248,
+      "loss": 1.3106,
+      "theoretical_loss": 3.4053163548835785,
+      "tokens_seen": 2423980032
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026809500882683357,
+      "loss": 1.2049,
+      "theoretical_loss": 3.405301700064447,
+      "tokens_seen": 2424111104
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002680548868560424,
+      "loss": 1.2371,
+      "theoretical_loss": 3.4052870462595375,
+      "tokens_seen": 2424242176
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002680147648852512,
+      "loss": 1.2691,
+      "theoretical_loss": 3.405272393468724,
+      "tokens_seen": 2424373248
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026797464291445993,
+      "loss": 1.2251,
+      "theoretical_loss": 3.4052577416918823,
+      "tokens_seen": 2424504320
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002679345209436688,
+      "loss": 1.225,
+      "theoretical_loss": 3.405243090928887,
+      "tokens_seen": 2424635392
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026789439897287754,
+      "loss": 1.2402,
+      "theoretical_loss": 3.405228441179613,
+      "tokens_seen": 2424766464
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026785427700208635,
+      "loss": 1.301,
+      "theoretical_loss": 3.405213792443936,
+      "tokens_seen": 2424897536
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026781415503129516,
+      "loss": 1.2839,
+      "theoretical_loss": 3.4051991447217307,
+      "tokens_seen": 2425028608
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026777403306050396,
+      "loss": 1.2465,
+      "theoretical_loss": 3.405184498012872,
+      "tokens_seen": 2425159680
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002677339110897127,
+      "loss": 1.2026,
+      "theoretical_loss": 3.4051698523172353,
+      "tokens_seen": 2425290752
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002676937891189215,
+      "loss": 1.2184,
+      "theoretical_loss": 3.4051552076346954,
+      "tokens_seen": 2425421824
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002676536671481303,
+      "loss": 1.2847,
+      "theoretical_loss": 3.405140563965128,
+      "tokens_seen": 2425552896
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002676135451773391,
+      "loss": 1.262,
+      "theoretical_loss": 3.405125921308408,
+      "tokens_seen": 2425683968
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026757342320654794,
+      "loss": 1.254,
+      "theoretical_loss": 3.405111279664411,
+      "tokens_seen": 2425815040
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002675333012357567,
+      "loss": 1.1879,
+      "theoretical_loss": 3.4050966390330117,
+      "tokens_seen": 2425946112
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026749317926496555,
+      "loss": 1.2159,
+      "theoretical_loss": 3.4050819994140857,
+      "tokens_seen": 2426077184
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002674530572941743,
+      "loss": 1.3154,
+      "theoretical_loss": 3.405067360807508,
+      "tokens_seen": 2426208256
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026741293532338306,
+      "loss": 1.2259,
+      "theoretical_loss": 3.405052723213154,
+      "tokens_seen": 2426339328
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002673728133525919,
+      "loss": 1.2102,
+      "theoretical_loss": 3.4050380866308996,
+      "tokens_seen": 2426470400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026733269138180067,
+      "loss": 1.1954,
+      "theoretical_loss": 3.4050234510606194,
+      "tokens_seen": 2426601472
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002672925694110095,
+      "loss": 1.2899,
+      "theoretical_loss": 3.405008816502189,
+      "tokens_seen": 2426732544
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002672524474402183,
+      "loss": 1.2236,
+      "theoretical_loss": 3.404994182955484,
+      "tokens_seen": 2426863616
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002672123254694271,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4049795504203795,
+      "tokens_seen": 2426994688
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026717220349863584,
+      "loss": 1.2209,
+      "theoretical_loss": 3.4049649188967512,
+      "tokens_seen": 2427125760
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026713208152784465,
+      "loss": 1.2531,
+      "theoretical_loss": 3.404950288384475,
+      "tokens_seen": 2427256832
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026709195955705345,
+      "loss": 1.2932,
+      "theoretical_loss": 3.404935658883425,
+      "tokens_seen": 2427387904
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002670518375862622,
+      "loss": 1.1674,
+      "theoretical_loss": 3.4049210303934783,
+      "tokens_seen": 2427518976
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026701171561547106,
+      "loss": 1.3381,
+      "theoretical_loss": 3.4049064029145093,
+      "tokens_seen": 2427650048
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002669715936446798,
+      "loss": 1.2184,
+      "theoretical_loss": 3.404891776446394,
+      "tokens_seen": 2427781120
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002669314716738886,
+      "loss": 1.3517,
+      "theoretical_loss": 3.404877150989008,
+      "tokens_seen": 2427912192
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026689134970309743,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4048625265422268,
+      "tokens_seen": 2428043264
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002668512277323062,
+      "loss": 1.2516,
+      "theoretical_loss": 3.404847903105926,
+      "tokens_seen": 2428174336
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026681110576151504,
+      "loss": 1.2224,
+      "theoretical_loss": 3.4048332806799815,
+      "tokens_seen": 2428305408
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002667709837907238,
+      "loss": 1.3066,
+      "theoretical_loss": 3.4048186592642686,
+      "tokens_seen": 2428436480
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002667308618199326,
+      "loss": 1.2209,
+      "theoretical_loss": 3.4048040388586633,
+      "tokens_seen": 2428567552
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002666907398491414,
+      "loss": 1.2084,
+      "theoretical_loss": 3.404789419463041,
+      "tokens_seen": 2428698624
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002666506178783502,
+      "loss": 1.2512,
+      "theoretical_loss": 3.4047748010772776,
+      "tokens_seen": 2428829696
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026661049590755896,
+      "loss": 1.2612,
+      "theoretical_loss": 3.404760183701249,
+      "tokens_seen": 2428960768
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002665703739367678,
+      "loss": 1.2056,
+      "theoretical_loss": 3.4047455673348304,
+      "tokens_seen": 2429091840
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002665302519659766,
+      "loss": 1.25,
+      "theoretical_loss": 3.404730951977898,
+      "tokens_seen": 2429222912
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026649012999518533,
+      "loss": 1.2843,
+      "theoretical_loss": 3.404716337630328,
+      "tokens_seen": 2429353984
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002664500080243942,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4047017242919955,
+      "tokens_seen": 2429485056
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026640988605360294,
+      "loss": 1.2481,
+      "theoretical_loss": 3.4046871119627773,
+      "tokens_seen": 2429616128
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026636976408281175,
+      "loss": 1.2713,
+      "theoretical_loss": 3.4046725006425484,
+      "tokens_seen": 2429747200
+    },
+    {
+      "epoch": 0.47,
+      "objective/train/docs_used": 815865,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2329447269439697,
+      "objective/train/theoretical_loss": 3.4046578903311846,
+      "objective/train/tokens_used": 800337376,
+      "theoretical_loss": 3.4046578903311846,
+      "tokens_seen": 2429878272
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026632964211202055,
+      "loss": 1.2444,
+      "theoretical_loss": 3.4046578903311846,
+      "tokens_seen": 2429878272
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026628952014122936,
+      "loss": 1.2951,
+      "theoretical_loss": 3.4046432810285627,
+      "tokens_seen": 2430009344
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002662493981704381,
+      "loss": 1.2452,
+      "theoretical_loss": 3.404628672734558,
+      "tokens_seen": 2430140416
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002662092761996469,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4046140654490467,
+      "tokens_seen": 2430271488
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002661691542288557,
+      "loss": 1.2546,
+      "theoretical_loss": 3.4045994591719047,
+      "tokens_seen": 2430402560
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026612903225806453,
+      "loss": 1.2667,
+      "theoretical_loss": 3.4045848539030077,
+      "tokens_seen": 2430533632
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026608891028727334,
+      "loss": 1.2007,
+      "theoretical_loss": 3.4045702496422328,
+      "tokens_seen": 2430664704
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002660487883164821,
+      "loss": 1.2795,
+      "theoretical_loss": 3.404555646389455,
+      "tokens_seen": 2430795776
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026600866634569095,
+      "loss": 1.2257,
+      "theoretical_loss": 3.4045410441445507,
+      "tokens_seen": 2430926848
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002659685443748997,
+      "loss": 1.2959,
+      "theoretical_loss": 3.404526442907396,
+      "tokens_seen": 2431057920
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026592842240410846,
+      "loss": 1.2375,
+      "theoretical_loss": 3.4045118426778673,
+      "tokens_seen": 2431188992
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002658883004333173,
+      "loss": 1.2679,
+      "theoretical_loss": 3.40449724345584,
+      "tokens_seen": 2431320064
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026584817846252607,
+      "loss": 1.2678,
+      "theoretical_loss": 3.404482645241191,
+      "tokens_seen": 2431451136
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002658080564917349,
+      "loss": 1.2559,
+      "theoretical_loss": 3.4044680480337965,
+      "tokens_seen": 2431582208
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002657679345209437,
+      "loss": 1.2352,
+      "theoretical_loss": 3.4044534518335325,
+      "tokens_seen": 2431713280
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002657278125501525,
+      "loss": 1.2797,
+      "theoretical_loss": 3.404438856640275,
+      "tokens_seen": 2431844352
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026568769057936124,
+      "loss": 1.1889,
+      "theoretical_loss": 3.4044242624539,
+      "tokens_seen": 2431975424
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026564756860857005,
+      "loss": 1.2594,
+      "theoretical_loss": 3.404409669274285,
+      "tokens_seen": 2432106496
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026560744663777885,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4043950771013054,
+      "tokens_seen": 2432237568
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026556732466698766,
+      "loss": 1.2634,
+      "theoretical_loss": 3.4043804859348374,
+      "tokens_seen": 2432368640
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026552720269619646,
+      "loss": 1.1913,
+      "theoretical_loss": 3.404365895774758,
+      "tokens_seen": 2432499712
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002654870807254052,
+      "loss": 1.2465,
+      "theoretical_loss": 3.4043513066209425,
+      "tokens_seen": 2432630784
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002654469587546141,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4043367184732682,
+      "tokens_seen": 2432761856
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026540683678382283,
+      "loss": 1.2776,
+      "theoretical_loss": 3.4043221313316114,
+      "tokens_seen": 2432892928
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002653667148130316,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4043075451958487,
+      "tokens_seen": 2433024000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026532659284224044,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4042929600658556,
+      "tokens_seen": 2433155072
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002652864708714492,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4042783759415096,
+      "tokens_seen": 2433286144
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.000265246348900658,
+      "loss": 1.2091,
+      "theoretical_loss": 3.404263792822687,
+      "tokens_seen": 2433417216
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002652062269298668,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4042492107092635,
+      "tokens_seen": 2433548288
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002651661049590756,
+      "loss": 1.2347,
+      "theoretical_loss": 3.404234629601117,
+      "tokens_seen": 2433679360
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026512598298828436,
+      "loss": 1.235,
+      "theoretical_loss": 3.4042200494981225,
+      "tokens_seen": 2433810432
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026508586101749317,
+      "loss": 1.2072,
+      "theoretical_loss": 3.404205470400158,
+      "tokens_seen": 2433941504
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000265045739046702,
+      "loss": 1.1756,
+      "theoretical_loss": 3.404190892307099,
+      "tokens_seen": 2434072576
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026500561707591073,
+      "loss": 1.2477,
+      "theoretical_loss": 3.404176315218823,
+      "tokens_seen": 2434203648
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002649654951051196,
+      "loss": 1.2747,
+      "theoretical_loss": 3.4041617391352066,
+      "tokens_seen": 2434334720
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026492537313432834,
+      "loss": 1.2553,
+      "theoretical_loss": 3.404147164056126,
+      "tokens_seen": 2434465792
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002648852511635372,
+      "loss": 1.2614,
+      "theoretical_loss": 3.404132589981458,
+      "tokens_seen": 2434596864
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026484512919274595,
+      "loss": 1.2974,
+      "theoretical_loss": 3.404118016911079,
+      "tokens_seen": 2434727936
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026480500722195476,
+      "loss": 1.246,
+      "theoretical_loss": 3.4041034448448664,
+      "tokens_seen": 2434859008
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026476488525116357,
+      "loss": 1.2846,
+      "theoretical_loss": 3.4040888737826966,
+      "tokens_seen": 2434990080
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002647247632803723,
+      "loss": 1.25,
+      "theoretical_loss": 3.4040743037244465,
+      "tokens_seen": 2435121152
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002646846413095811,
+      "loss": 1.2208,
+      "theoretical_loss": 3.404059734669993,
+      "tokens_seen": 2435252224
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026464451933878993,
+      "loss": 1.2081,
+      "theoretical_loss": 3.4040451666192126,
+      "tokens_seen": 2435383296
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026460439736799874,
+      "loss": 1.255,
+      "theoretical_loss": 3.404030599571982,
+      "tokens_seen": 2435514368
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002645642753972075,
+      "loss": 1.1981,
+      "theoretical_loss": 3.404016033528179,
+      "tokens_seen": 2435645440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026452415342641635,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4040014684876794,
+      "tokens_seen": 2435776512
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002644840314556251,
+      "loss": 1.2319,
+      "theoretical_loss": 3.4039869044503606,
+      "tokens_seen": 2435907584
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026444390948483385,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4039723414160994,
+      "tokens_seen": 2436038656
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002644037875140427,
+      "loss": 1.2032,
+      "theoretical_loss": 3.4039577793847733,
+      "tokens_seen": 2436169728
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026436366554325147,
+      "loss": 1.2266,
+      "theoretical_loss": 3.4039432183562583,
+      "tokens_seen": 2436300800
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 817860,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2630696296691895,
+      "objective/train/theoretical_loss": 3.4039286583304325,
+      "objective/train/tokens_used": 806890976,
+      "theoretical_loss": 3.4039286583304325,
+      "tokens_seen": 2436431872
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026432354357246033,
+      "loss": 1.2382,
+      "theoretical_loss": 3.4039286583304325,
+      "tokens_seen": 2436431872
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002642834216016691,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4039140993071717,
+      "tokens_seen": 2436562944
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002642432996308779,
+      "loss": 1.3432,
+      "theoretical_loss": 3.4038995412863544,
+      "tokens_seen": 2436694016
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002642031776600867,
+      "loss": 1.2868,
+      "theoretical_loss": 3.4038849842678562,
+      "tokens_seen": 2436825088
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026416305568929544,
+      "loss": 1.2269,
+      "theoretical_loss": 3.403870428251555,
+      "tokens_seen": 2436956160
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026412293371850425,
+      "loss": 1.2603,
+      "theoretical_loss": 3.403855873237328,
+      "tokens_seen": 2437087232
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026408281174771306,
+      "loss": 1.1981,
+      "theoretical_loss": 3.4038413192250516,
+      "tokens_seen": 2437218304
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026404268977692186,
+      "loss": 1.2416,
+      "theoretical_loss": 3.4038267662146033,
+      "tokens_seen": 2437349376
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002640025678061306,
+      "loss": 1.2447,
+      "theoretical_loss": 3.403812214205861,
+      "tokens_seen": 2437480448
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002639624458353395,
+      "loss": 1.223,
+      "theoretical_loss": 3.403797663198701,
+      "tokens_seen": 2437611520
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026392232386454823,
+      "loss": 1.2186,
+      "theoretical_loss": 3.4037831131930005,
+      "tokens_seen": 2437742592
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000263882201893757,
+      "loss": 1.2033,
+      "theoretical_loss": 3.403768564188637,
+      "tokens_seen": 2437873664
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026384207992296584,
+      "loss": 1.256,
+      "theoretical_loss": 3.403754016185488,
+      "tokens_seen": 2438004736
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002638019579521746,
+      "loss": 1.2056,
+      "theoretical_loss": 3.4037394691834306,
+      "tokens_seen": 2438135808
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002637618359813834,
+      "loss": 1.1987,
+      "theoretical_loss": 3.4037249231823417,
+      "tokens_seen": 2438266880
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002637217140105922,
+      "loss": 1.2609,
+      "theoretical_loss": 3.403710378182099,
+      "tokens_seen": 2438397952
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000263681592039801,
+      "loss": 1.2225,
+      "theoretical_loss": 3.4036958341825807,
+      "tokens_seen": 2438529024
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002636414700690098,
+      "loss": 1.2353,
+      "theoretical_loss": 3.4036812911836622,
+      "tokens_seen": 2438660096
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026360134809821857,
+      "loss": 1.2672,
+      "theoretical_loss": 3.4036667491852226,
+      "tokens_seen": 2438791168
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002635612261274274,
+      "loss": 1.2266,
+      "theoretical_loss": 3.4036522081871388,
+      "tokens_seen": 2438922240
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002635211041566362,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4036376681892873,
+      "tokens_seen": 2439053312
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000263480982185845,
+      "loss": 1.2596,
+      "theoretical_loss": 3.403623129191547,
+      "tokens_seen": 2439184384
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026344086021505374,
+      "loss": 1.3409,
+      "theoretical_loss": 3.4036085911937946,
+      "tokens_seen": 2439315456
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002634007382442626,
+      "loss": 1.3078,
+      "theoretical_loss": 3.4035940541959073,
+      "tokens_seen": 2439446528
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026336061627347135,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4035795181977635,
+      "tokens_seen": 2439577600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026332049430268016,
+      "loss": 1.1731,
+      "theoretical_loss": 3.4035649831992405,
+      "tokens_seen": 2439708672
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026328037233188897,
+      "loss": 1.245,
+      "theoretical_loss": 3.403550449200215,
+      "tokens_seen": 2439839744
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002632402503610977,
+      "loss": 1.2432,
+      "theoretical_loss": 3.4035359162005654,
+      "tokens_seen": 2439970816
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002632001283903065,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4035213842001695,
+      "tokens_seen": 2440101888
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026316000641951533,
+      "loss": 1.1906,
+      "theoretical_loss": 3.403506853198904,
+      "tokens_seen": 2440232960
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026311988444872414,
+      "loss": 1.2651,
+      "theoretical_loss": 3.403492323196647,
+      "tokens_seen": 2440364032
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002630797624779329,
+      "loss": 1.1996,
+      "theoretical_loss": 3.403477794193277,
+      "tokens_seen": 2440495104
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026303964050714175,
+      "loss": 1.1973,
+      "theoretical_loss": 3.4034632661886706,
+      "tokens_seen": 2440626176
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002629995185363505,
+      "loss": 1.2464,
+      "theoretical_loss": 3.4034487391827057,
+      "tokens_seen": 2440757248
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002629593965655593,
+      "loss": 1.2215,
+      "theoretical_loss": 3.4034342131752604,
+      "tokens_seen": 2440888320
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002629192745947681,
+      "loss": 1.2359,
+      "theoretical_loss": 3.403419688166212,
+      "tokens_seen": 2441019392
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026287915262397687,
+      "loss": 1.2253,
+      "theoretical_loss": 3.4034051641554384,
+      "tokens_seen": 2441150464
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026283903065318573,
+      "loss": 1.2904,
+      "theoretical_loss": 3.403390641142818,
+      "tokens_seen": 2441281536
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002627989086823945,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4033761191282275,
+      "tokens_seen": 2441412608
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002627587867116033,
+      "loss": 1.2705,
+      "theoretical_loss": 3.4033615981115455,
+      "tokens_seen": 2441543680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002627186647408121,
+      "loss": 1.2092,
+      "theoretical_loss": 3.40334707809265,
+      "tokens_seen": 2441674752
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026267854277002084,
+      "loss": 1.2645,
+      "theoretical_loss": 3.4033325590714183,
+      "tokens_seen": 2441805824
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026263842079922965,
+      "loss": 1.1727,
+      "theoretical_loss": 3.403318041047729,
+      "tokens_seen": 2441936896
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026259829882843846,
+      "loss": 1.2563,
+      "theoretical_loss": 3.4033035240214593,
+      "tokens_seen": 2442067968
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026255817685764726,
+      "loss": 1.1733,
+      "theoretical_loss": 3.4032890079924876,
+      "tokens_seen": 2442199040
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000262518054886856,
+      "loss": 1.2612,
+      "theoretical_loss": 3.4032744929606915,
+      "tokens_seen": 2442330112
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002624779329160649,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4032599789259494,
+      "tokens_seen": 2442461184
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026243781094527363,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4032454658881393,
+      "tokens_seen": 2442592256
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002623976889744824,
+      "loss": 1.2321,
+      "theoretical_loss": 3.4032309538471393,
+      "tokens_seen": 2442723328
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026235756700369124,
+      "loss": 1.2349,
+      "theoretical_loss": 3.403216442802827,
+      "tokens_seen": 2442854400
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 820144,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2281115055084229,
+      "objective/train/theoretical_loss": 3.4032019327550804,
+      "objective/train/tokens_used": 813444576,
+      "theoretical_loss": 3.4032019327550804,
+      "tokens_seen": 2442985472
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002623174450329,
+      "loss": 1.252,
+      "theoretical_loss": 3.4032019327550804,
+      "tokens_seen": 2442985472
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026227732306210885,
+      "loss": 1.1949,
+      "theoretical_loss": 3.403187423703778,
+      "tokens_seen": 2443116544
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002622372010913176,
+      "loss": 1.2518,
+      "theoretical_loss": 3.403172915648798,
+      "tokens_seen": 2443247616
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002621970791205264,
+      "loss": 1.1947,
+      "theoretical_loss": 3.403158408590018,
+      "tokens_seen": 2443378688
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002621569571497352,
+      "loss": 1.2445,
+      "theoretical_loss": 3.403143902527317,
+      "tokens_seen": 2443509760
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026211683517894397,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4031293974605727,
+      "tokens_seen": 2443640832
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002620767132081528,
+      "loss": 1.2511,
+      "theoretical_loss": 3.403114893389663,
+      "tokens_seen": 2443771904
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002620365912373616,
+      "loss": 1.3016,
+      "theoretical_loss": 3.4031003903144663,
+      "tokens_seen": 2443902976
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002619964692665704,
+      "loss": 1.1982,
+      "theoretical_loss": 3.4030858882348616,
+      "tokens_seen": 2444034048
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026195634729577914,
+      "loss": 1.2308,
+      "theoretical_loss": 3.4030713871507263,
+      "tokens_seen": 2444165120
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000261916225324988,
+      "loss": 1.2144,
+      "theoretical_loss": 3.4030568870619384,
+      "tokens_seen": 2444296192
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026187610335419675,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4030423879683775,
+      "tokens_seen": 2444427264
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026183598138340556,
+      "loss": 1.2669,
+      "theoretical_loss": 3.4030278898699207,
+      "tokens_seen": 2444558336
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026179585941261437,
+      "loss": 1.2441,
+      "theoretical_loss": 3.403013392766447,
+      "tokens_seen": 2444689408
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002617557374418231,
+      "loss": 1.1968,
+      "theoretical_loss": 3.4029988966578344,
+      "tokens_seen": 2444820480
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000261715615471032,
+      "loss": 1.1914,
+      "theoretical_loss": 3.4029844015439616,
+      "tokens_seen": 2444951552
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026167549350024073,
+      "loss": 1.2796,
+      "theoretical_loss": 3.4029699074247075,
+      "tokens_seen": 2445082624
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026163537152944954,
+      "loss": 1.2293,
+      "theoretical_loss": 3.4029554142999494,
+      "tokens_seen": 2445213696
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026159524955865834,
+      "loss": 1.2384,
+      "theoretical_loss": 3.4029409221695666,
+      "tokens_seen": 2445344768
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026155512758786715,
+      "loss": 1.2418,
+      "theoretical_loss": 3.402926431033437,
+      "tokens_seen": 2445475840
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002615150056170759,
+      "loss": 1.1992,
+      "theoretical_loss": 3.4029119408914394,
+      "tokens_seen": 2445606912
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002614748836462847,
+      "loss": 1.2405,
+      "theoretical_loss": 3.4028974517434527,
+      "tokens_seen": 2445737984
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002614347616754935,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4028829635893545,
+      "tokens_seen": 2445869056
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026139463970470227,
+      "loss": 1.2444,
+      "theoretical_loss": 3.4028684764290245,
+      "tokens_seen": 2446000128
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026135451773391113,
+      "loss": 1.2622,
+      "theoretical_loss": 3.4028539902623405,
+      "tokens_seen": 2446131200
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002613143957631199,
+      "loss": 1.2152,
+      "theoretical_loss": 3.4028395050891813,
+      "tokens_seen": 2446262272
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002612742737923287,
+      "loss": 1.209,
+      "theoretical_loss": 3.4028250209094257,
+      "tokens_seen": 2446393344
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002612341518215375,
+      "loss": 1.3077,
+      "theoretical_loss": 3.402810537722952,
+      "tokens_seen": 2446524416
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026119402985074624,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4027960555296395,
+      "tokens_seen": 2446655488
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026115390787995505,
+      "loss": 1.2546,
+      "theoretical_loss": 3.4027815743293663,
+      "tokens_seen": 2446786560
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026111378590916386,
+      "loss": 1.1876,
+      "theoretical_loss": 3.4027670941220114,
+      "tokens_seen": 2446917632
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026107366393837266,
+      "loss": 1.1993,
+      "theoretical_loss": 3.4027526149074534,
+      "tokens_seen": 2447048704
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026103354196758147,
+      "loss": 1.2336,
+      "theoretical_loss": 3.402738136685571,
+      "tokens_seen": 2447179776
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002609934199967903,
+      "loss": 1.2791,
+      "theoretical_loss": 3.4027236594562433,
+      "tokens_seen": 2447310848
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026095329802599903,
+      "loss": 1.237,
+      "theoretical_loss": 3.402709183219349,
+      "tokens_seen": 2447441920
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026091317605520783,
+      "loss": 1.2987,
+      "theoretical_loss": 3.4026947079747667,
+      "tokens_seen": 2447572992
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026087305408441664,
+      "loss": 1.1425,
+      "theoretical_loss": 3.4026802337223754,
+      "tokens_seen": 2447704064
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002608329321136254,
+      "loss": 1.2517,
+      "theoretical_loss": 3.402665760462054,
+      "tokens_seen": 2447835136
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026079281014283425,
+      "loss": 1.2479,
+      "theoretical_loss": 3.402651288193681,
+      "tokens_seen": 2447966208
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000260752688172043,
+      "loss": 1.2769,
+      "theoretical_loss": 3.402636816917136,
+      "tokens_seen": 2448097280
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002607125662012518,
+      "loss": 1.2881,
+      "theoretical_loss": 3.402622346632297,
+      "tokens_seen": 2448228352
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002606724442304606,
+      "loss": 1.2067,
+      "theoretical_loss": 3.402607877339044,
+      "tokens_seen": 2448359424
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026063232225966937,
+      "loss": 1.2374,
+      "theoretical_loss": 3.402593409037255,
+      "tokens_seen": 2448490496
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002605922002888782,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4025789417268104,
+      "tokens_seen": 2448621568
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000260552078318087,
+      "loss": 1.2773,
+      "theoretical_loss": 3.4025644754075874,
+      "tokens_seen": 2448752640
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002605119563472958,
+      "loss": 1.254,
+      "theoretical_loss": 3.4025500100794663,
+      "tokens_seen": 2448883712
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002604718343765046,
+      "loss": 1.1692,
+      "theoretical_loss": 3.402535545742326,
+      "tokens_seen": 2449014784
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002604317124057134,
+      "loss": 1.2201,
+      "theoretical_loss": 3.402521082396045,
+      "tokens_seen": 2449145856
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026039159043492215,
+      "loss": 1.2485,
+      "theoretical_loss": 3.4025066200405027,
+      "tokens_seen": 2449276928
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026035146846413096,
+      "loss": 1.2545,
+      "theoretical_loss": 3.402492158675579,
+      "tokens_seen": 2449408000
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 822421,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2847944498062134,
+      "objective/train/theoretical_loss": 3.4024776983011513,
+      "objective/train/tokens_used": 819998176,
+      "theoretical_loss": 3.4024776983011513,
+      "tokens_seen": 2449539072
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026031134649333977,
+      "loss": 1.2194,
+      "theoretical_loss": 3.4024776983011513,
+      "tokens_seen": 2449539072
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002602712245225485,
+      "loss": 1.2607,
+      "theoretical_loss": 3.4024632389171003,
+      "tokens_seen": 2449670144
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002602311025517574,
+      "loss": 1.295,
+      "theoretical_loss": 3.402448780523305,
+      "tokens_seen": 2449801216
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026019098058096613,
+      "loss": 1.1482,
+      "theoretical_loss": 3.4024343231196434,
+      "tokens_seen": 2449932288
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026015085861017494,
+      "loss": 1.2941,
+      "theoretical_loss": 3.4024198667059964,
+      "tokens_seen": 2450063360
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026011073663938374,
+      "loss": 1.2045,
+      "theoretical_loss": 3.4024054112822424,
+      "tokens_seen": 2450194432
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00026007061466859255,
+      "loss": 1.2781,
+      "theoretical_loss": 3.4023909568482607,
+      "tokens_seen": 2450325504
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002600304926978013,
+      "loss": 1.2402,
+      "theoretical_loss": 3.40237650340393,
+      "tokens_seen": 2450456576
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002599903707270101,
+      "loss": 1.2721,
+      "theoretical_loss": 3.402362050949131,
+      "tokens_seen": 2450587648
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002599502487562189,
+      "loss": 1.2564,
+      "theoretical_loss": 3.4023475994837415,
+      "tokens_seen": 2450718720
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025991012678542767,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4023331490076423,
+      "tokens_seen": 2450849792
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002598700048146365,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4023186995207118,
+      "tokens_seen": 2450980864
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002598298828438453,
+      "loss": 1.219,
+      "theoretical_loss": 3.40230425102283,
+      "tokens_seen": 2451111936
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025978976087305414,
+      "loss": 1.1935,
+      "theoretical_loss": 3.4022898035138756,
+      "tokens_seen": 2451243008
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002597496389022629,
+      "loss": 1.2928,
+      "theoretical_loss": 3.4022753569937283,
+      "tokens_seen": 2451374080
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025970951693147164,
+      "loss": 1.2702,
+      "theoretical_loss": 3.4022609114622684,
+      "tokens_seen": 2451505152
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002596693949606805,
+      "loss": 1.1866,
+      "theoretical_loss": 3.402246466919374,
+      "tokens_seen": 2451636224
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025962927298988926,
+      "loss": 1.2369,
+      "theoretical_loss": 3.4022320233649257,
+      "tokens_seen": 2451767296
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025958915101909806,
+      "loss": 1.3034,
+      "theoretical_loss": 3.402217580798802,
+      "tokens_seen": 2451898368
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025954902904830687,
+      "loss": 1.2252,
+      "theoretical_loss": 3.402203139220884,
+      "tokens_seen": 2452029440
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002595089070775157,
+      "loss": 1.1834,
+      "theoretical_loss": 3.4021886986310497,
+      "tokens_seen": 2452160512
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025946878510672443,
+      "loss": 1.3122,
+      "theoretical_loss": 3.4021742590291795,
+      "tokens_seen": 2452291584
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025942866313593323,
+      "loss": 1.2919,
+      "theoretical_loss": 3.4021598204151524,
+      "tokens_seen": 2452422656
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025938854116514204,
+      "loss": 1.212,
+      "theoretical_loss": 3.402145382788849,
+      "tokens_seen": 2452553728
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002593484191943508,
+      "loss": 1.2363,
+      "theoretical_loss": 3.402130946150148,
+      "tokens_seen": 2452684800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025930829722355965,
+      "loss": 1.1992,
+      "theoretical_loss": 3.4021165104989297,
+      "tokens_seen": 2452815872
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002592681752527684,
+      "loss": 1.2709,
+      "theoretical_loss": 3.4021020758350735,
+      "tokens_seen": 2452946944
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025922805328197727,
+      "loss": 1.1839,
+      "theoretical_loss": 3.402087642158459,
+      "tokens_seen": 2453078016
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.000259187931311186,
+      "loss": 1.2213,
+      "theoretical_loss": 3.4020732094689663,
+      "tokens_seen": 2453209088
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025914780934039477,
+      "loss": 1.2458,
+      "theoretical_loss": 3.402058777766475,
+      "tokens_seen": 2453340160
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025910768736960363,
+      "loss": 1.259,
+      "theoretical_loss": 3.4020443470508646,
+      "tokens_seen": 2453471232
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002590675653988124,
+      "loss": 1.3223,
+      "theoretical_loss": 3.4020299173220154,
+      "tokens_seen": 2453602304
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002590274434280212,
+      "loss": 1.2532,
+      "theoretical_loss": 3.4020154885798064,
+      "tokens_seen": 2453733376
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025898732145723,
+      "loss": 1.216,
+      "theoretical_loss": 3.4020010608241185,
+      "tokens_seen": 2453864448
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002589471994864388,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4019866340548313,
+      "tokens_seen": 2453995520
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025890707751564755,
+      "loss": 1.2287,
+      "theoretical_loss": 3.401972208271824,
+      "tokens_seen": 2454126592
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025886695554485636,
+      "loss": 1.3324,
+      "theoretical_loss": 3.401957783474977,
+      "tokens_seen": 2454257664
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025882683357406517,
+      "loss": 1.2406,
+      "theoretical_loss": 3.40194335966417,
+      "tokens_seen": 2454388736
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002587867116032739,
+      "loss": 1.2669,
+      "theoretical_loss": 3.4019289368392833,
+      "tokens_seen": 2454519808
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002587465896324828,
+      "loss": 1.2946,
+      "theoretical_loss": 3.401914515000197,
+      "tokens_seen": 2454650880
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025870646766169153,
+      "loss": 1.245,
+      "theoretical_loss": 3.40190009414679,
+      "tokens_seen": 2454781952
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025866634569090034,
+      "loss": 1.2624,
+      "theoretical_loss": 3.4018856742789434,
+      "tokens_seen": 2454913024
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025862622372010914,
+      "loss": 1.2384,
+      "theoretical_loss": 3.401871255396537,
+      "tokens_seen": 2455044096
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025858610174931795,
+      "loss": 1.1969,
+      "theoretical_loss": 3.4018568374994507,
+      "tokens_seen": 2455175168
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025854597977852676,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4018424205875646,
+      "tokens_seen": 2455306240
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002585058578077355,
+      "loss": 1.3518,
+      "theoretical_loss": 3.401828004660759,
+      "tokens_seen": 2455437312
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002584657358369443,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4018135897189135,
+      "tokens_seen": 2455568384
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002584256138661531,
+      "loss": 1.2407,
+      "theoretical_loss": 3.4017991757619086,
+      "tokens_seen": 2455699456
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002583854918953619,
+      "loss": 1.3065,
+      "theoretical_loss": 3.4017847627896245,
+      "tokens_seen": 2455830528
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002583453699245707,
+      "loss": 1.303,
+      "theoretical_loss": 3.4017703508019412,
+      "tokens_seen": 2455961600
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 824774,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2580161094665527,
+      "objective/train/theoretical_loss": 3.401755939798739,
+      "objective/train/tokens_used": 826551776,
+      "theoretical_loss": 3.401755939798739,
+      "tokens_seen": 2456092672
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025830524795377954,
+      "loss": 1.2444,
+      "theoretical_loss": 3.401755939798739,
+      "tokens_seen": 2456092672
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002582651259829883,
+      "loss": 1.2374,
+      "theoretical_loss": 3.401741529779898,
+      "tokens_seen": 2456223744
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025822500401219704,
+      "loss": 1.2644,
+      "theoretical_loss": 3.401727120745299,
+      "tokens_seen": 2456354816
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002581848820414059,
+      "loss": 1.2513,
+      "theoretical_loss": 3.4017127126948212,
+      "tokens_seen": 2456485888
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025814476007061466,
+      "loss": 1.2221,
+      "theoretical_loss": 3.4016983056283454,
+      "tokens_seen": 2456616960
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025810463809982346,
+      "loss": 1.2384,
+      "theoretical_loss": 3.4016838995457523,
+      "tokens_seen": 2456748032
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025806451612903227,
+      "loss": 1.2108,
+      "theoretical_loss": 3.401669494446922,
+      "tokens_seen": 2456879104
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002580243941582411,
+      "loss": 1.2101,
+      "theoretical_loss": 3.401655090331735,
+      "tokens_seen": 2457010176
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002579842721874498,
+      "loss": 1.228,
+      "theoretical_loss": 3.4016406872000706,
+      "tokens_seen": 2457141248
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025794415021665863,
+      "loss": 1.3059,
+      "theoretical_loss": 3.4016262850518104,
+      "tokens_seen": 2457272320
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025790402824586744,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4016118838868343,
+      "tokens_seen": 2457403392
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025786390627507625,
+      "loss": 1.2795,
+      "theoretical_loss": 3.4015974837050234,
+      "tokens_seen": 2457534464
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025782378430428505,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4015830845062567,
+      "tokens_seen": 2457665536
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002577836623334938,
+      "loss": 1.2473,
+      "theoretical_loss": 3.401568686290416,
+      "tokens_seen": 2457796608
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025774354036270266,
+      "loss": 1.2588,
+      "theoretical_loss": 3.4015542890573816,
+      "tokens_seen": 2457927680
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002577034183919114,
+      "loss": 1.2984,
+      "theoretical_loss": 3.4015398928070333,
+      "tokens_seen": 2458058752
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025766329642112017,
+      "loss": 1.2726,
+      "theoretical_loss": 3.401525497539252,
+      "tokens_seen": 2458189824
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025762317445032903,
+      "loss": 1.2361,
+      "theoretical_loss": 3.4015111032539185,
+      "tokens_seen": 2458320896
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002575830524795378,
+      "loss": 1.2677,
+      "theoretical_loss": 3.4014967099509135,
+      "tokens_seen": 2458451968
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002575429305087466,
+      "loss": 1.2005,
+      "theoretical_loss": 3.401482317630117,
+      "tokens_seen": 2458583040
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002575028085379554,
+      "loss": 1.2876,
+      "theoretical_loss": 3.40146792629141,
+      "tokens_seen": 2458714112
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002574626865671642,
+      "loss": 1.2425,
+      "theoretical_loss": 3.4014535359346727,
+      "tokens_seen": 2458845184
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025742256459637295,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4014391465597864,
+      "tokens_seen": 2458976256
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025738244262558176,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4014247581666313,
+      "tokens_seen": 2459107328
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025734232065479057,
+      "loss": 1.265,
+      "theoretical_loss": 3.4014103707550887,
+      "tokens_seen": 2459238400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002573021986839993,
+      "loss": 1.27,
+      "theoretical_loss": 3.4013959843250383,
+      "tokens_seen": 2459369472
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002572620767132082,
+      "loss": 1.2907,
+      "theoretical_loss": 3.401381598876362,
+      "tokens_seen": 2459500544
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025722195474241693,
+      "loss": 1.2281,
+      "theoretical_loss": 3.40136721440894,
+      "tokens_seen": 2459631616
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002571818327716258,
+      "loss": 1.2457,
+      "theoretical_loss": 3.4013528309226526,
+      "tokens_seen": 2459762688
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025714171080083454,
+      "loss": 1.1933,
+      "theoretical_loss": 3.4013384484173814,
+      "tokens_seen": 2459893760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025710158883004335,
+      "loss": 1.1663,
+      "theoretical_loss": 3.401324066893007,
+      "tokens_seen": 2460024832
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025706146685925216,
+      "loss": 1.1954,
+      "theoretical_loss": 3.4013096863494097,
+      "tokens_seen": 2460155904
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002570213448884609,
+      "loss": 1.2599,
+      "theoretical_loss": 3.401295306786471,
+      "tokens_seen": 2460286976
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002569812229176697,
+      "loss": 1.3014,
+      "theoretical_loss": 3.4012809282040717,
+      "tokens_seen": 2460418048
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002569411009468785,
+      "loss": 1.2406,
+      "theoretical_loss": 3.4012665506020925,
+      "tokens_seen": 2460549120
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002569009789760873,
+      "loss": 1.2154,
+      "theoretical_loss": 3.401252173980415,
+      "tokens_seen": 2460680192
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002568608570052961,
+      "loss": 1.2697,
+      "theoretical_loss": 3.401237798338919,
+      "tokens_seen": 2460811264
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025682073503450494,
+      "loss": 1.1766,
+      "theoretical_loss": 3.4012234236774863,
+      "tokens_seen": 2460942336
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002567806130637137,
+      "loss": 1.3142,
+      "theoretical_loss": 3.4012090499959977,
+      "tokens_seen": 2461073408
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025674049109292244,
+      "loss": 1.28,
+      "theoretical_loss": 3.401194677294334,
+      "tokens_seen": 2461204480
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002567003691221313,
+      "loss": 1.2986,
+      "theoretical_loss": 3.4011803055723764,
+      "tokens_seen": 2461335552
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025666024715134006,
+      "loss": 1.3171,
+      "theoretical_loss": 3.4011659348300056,
+      "tokens_seen": 2461466624
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002566201251805489,
+      "loss": 1.2857,
+      "theoretical_loss": 3.401151565067104,
+      "tokens_seen": 2461597696
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025658000320975767,
+      "loss": 1.1772,
+      "theoretical_loss": 3.4011371962835506,
+      "tokens_seen": 2461728768
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002565398812389665,
+      "loss": 1.2422,
+      "theoretical_loss": 3.4011228284792283,
+      "tokens_seen": 2461859840
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002564997592681753,
+      "loss": 1.2995,
+      "theoretical_loss": 3.401108461654017,
+      "tokens_seen": 2461990912
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025645963729738403,
+      "loss": 1.1604,
+      "theoretical_loss": 3.401094095807799,
+      "tokens_seen": 2462121984
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025641951532659284,
+      "loss": 1.2314,
+      "theoretical_loss": 3.4010797309404546,
+      "tokens_seen": 2462253056
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025637939335580165,
+      "loss": 1.2481,
+      "theoretical_loss": 3.4010653670518654,
+      "tokens_seen": 2462384128
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025633927138501045,
+      "loss": 1.2679,
+      "theoretical_loss": 3.4010510041419124,
+      "tokens_seen": 2462515200
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 827146,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2985577583312988,
+      "objective/train/theoretical_loss": 3.401036642210477,
+      "objective/train/tokens_used": 833105376,
+      "theoretical_loss": 3.401036642210477,
+      "tokens_seen": 2462646272
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002562991494142192,
+      "loss": 1.2873,
+      "theoretical_loss": 3.401036642210477,
+      "tokens_seen": 2462646272
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025625902744342806,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4010222812574407,
+      "tokens_seen": 2462777344
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002562189054726368,
+      "loss": 1.2672,
+      "theoretical_loss": 3.4010079212826843,
+      "tokens_seen": 2462908416
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025617878350184557,
+      "loss": 1.3003,
+      "theoretical_loss": 3.4009935622860894,
+      "tokens_seen": 2463039488
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025613866153105443,
+      "loss": 1.1986,
+      "theoretical_loss": 3.4009792042675366,
+      "tokens_seen": 2463170560
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002560985395602632,
+      "loss": 1.2031,
+      "theoretical_loss": 3.4009648472269083,
+      "tokens_seen": 2463301632
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.000256058417589472,
+      "loss": 1.2274,
+      "theoretical_loss": 3.4009504911640858,
+      "tokens_seen": 2463432704
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002560182956186808,
+      "loss": 1.2972,
+      "theoretical_loss": 3.4009361360789496,
+      "tokens_seen": 2463563776
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002559781736478896,
+      "loss": 1.1741,
+      "theoretical_loss": 3.4009217819713817,
+      "tokens_seen": 2463694848
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002559380516770984,
+      "loss": 1.1706,
+      "theoretical_loss": 3.4009074288412635,
+      "tokens_seen": 2463825920
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025589792970630716,
+      "loss": 1.2622,
+      "theoretical_loss": 3.4008930766884764,
+      "tokens_seen": 2463956992
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025585780773551597,
+      "loss": 1.1846,
+      "theoretical_loss": 3.400878725512902,
+      "tokens_seen": 2464088064
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025581768576472477,
+      "loss": 1.2223,
+      "theoretical_loss": 3.400864375314421,
+      "tokens_seen": 2464219136
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002557775637939336,
+      "loss": 1.268,
+      "theoretical_loss": 3.4008500260929164,
+      "tokens_seen": 2464350208
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025573744182314233,
+      "loss": 1.1889,
+      "theoretical_loss": 3.4008356778482685,
+      "tokens_seen": 2464481280
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002556973198523512,
+      "loss": 1.1845,
+      "theoretical_loss": 3.400821330580359,
+      "tokens_seen": 2464612352
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025565719788155994,
+      "loss": 1.2714,
+      "theoretical_loss": 3.40080698428907,
+      "tokens_seen": 2464743424
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002556170759107687,
+      "loss": 1.2162,
+      "theoretical_loss": 3.400792638974283,
+      "tokens_seen": 2464874496
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025557695393997755,
+      "loss": 1.297,
+      "theoretical_loss": 3.400778294635879,
+      "tokens_seen": 2465005568
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002555368319691863,
+      "loss": 1.2709,
+      "theoretical_loss": 3.40076395127374,
+      "tokens_seen": 2465136640
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002554967099983951,
+      "loss": 1.3178,
+      "theoretical_loss": 3.400749608887748,
+      "tokens_seen": 2465267712
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002554565880276039,
+      "loss": 1.202,
+      "theoretical_loss": 3.400735267477784,
+      "tokens_seen": 2465398784
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002554164660568127,
+      "loss": 1.2351,
+      "theoretical_loss": 3.4007209270437304,
+      "tokens_seen": 2465529856
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025537634408602153,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4007065875854683,
+      "tokens_seen": 2465660928
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025533622211523034,
+      "loss": 1.1662,
+      "theoretical_loss": 3.40069224910288,
+      "tokens_seen": 2465792000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002552961001444391,
+      "loss": 1.227,
+      "theoretical_loss": 3.4006779115958468,
+      "tokens_seen": 2465923072
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002552559781736479,
+      "loss": 1.2683,
+      "theoretical_loss": 3.4006635750642507,
+      "tokens_seen": 2466054144
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002552158562028567,
+      "loss": 1.1932,
+      "theoretical_loss": 3.400649239507973,
+      "tokens_seen": 2466185216
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025517573423206546,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4006349049268967,
+      "tokens_seen": 2466316288
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002551356122612743,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4006205713209026,
+      "tokens_seen": 2466447360
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025509549029048307,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4006062386898726,
+      "tokens_seen": 2466578432
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002550553683196919,
+      "loss": 1.2003,
+      "theoretical_loss": 3.400591907033689,
+      "tokens_seen": 2466709504
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002550152463489007,
+      "loss": 1.2321,
+      "theoretical_loss": 3.4005775763522337,
+      "tokens_seen": 2466840576
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025497512437810943,
+      "loss": 1.275,
+      "theoretical_loss": 3.400563246645388,
+      "tokens_seen": 2466971648
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025493500240731824,
+      "loss": 1.2465,
+      "theoretical_loss": 3.4005489179130346,
+      "tokens_seen": 2467102720
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025489488043652705,
+      "loss": 1.191,
+      "theoretical_loss": 3.400534590155055,
+      "tokens_seen": 2467233792
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025485475846573585,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4005202633713316,
+      "tokens_seen": 2467364864
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002548146364949446,
+      "loss": 1.3092,
+      "theoretical_loss": 3.4005059375617464,
+      "tokens_seen": 2467495936
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025477451452415346,
+      "loss": 1.2629,
+      "theoretical_loss": 3.4004916127261806,
+      "tokens_seen": 2467627008
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002547343925533622,
+      "loss": 1.2448,
+      "theoretical_loss": 3.4004772888645167,
+      "tokens_seen": 2467758080
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000254694270582571,
+      "loss": 1.2078,
+      "theoretical_loss": 3.4004629659766374,
+      "tokens_seen": 2467889152
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025465414861177983,
+      "loss": 1.2397,
+      "theoretical_loss": 3.400448644062424,
+      "tokens_seen": 2468020224
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002546140266409886,
+      "loss": 1.2179,
+      "theoretical_loss": 3.400434323121759,
+      "tokens_seen": 2468151296
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025457390467019744,
+      "loss": 1.2665,
+      "theoretical_loss": 3.400420003154524,
+      "tokens_seen": 2468282368
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002545337826994062,
+      "loss": 1.2727,
+      "theoretical_loss": 3.400405684160602,
+      "tokens_seen": 2468413440
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000254493660728615,
+      "loss": 1.2712,
+      "theoretical_loss": 3.400391366139875,
+      "tokens_seen": 2468544512
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002544535387578238,
+      "loss": 1.1484,
+      "theoretical_loss": 3.4003770490922243,
+      "tokens_seen": 2468675584
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025441341678703256,
+      "loss": 1.294,
+      "theoretical_loss": 3.400362733017533,
+      "tokens_seen": 2468806656
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025437329481624136,
+      "loss": 1.274,
+      "theoretical_loss": 3.4003484179156827,
+      "tokens_seen": 2468937728
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025433317284545017,
+      "loss": 1.1945,
+      "theoretical_loss": 3.400334103786556,
+      "tokens_seen": 2469068800
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 829005,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1221880912780762,
+      "objective/train/theoretical_loss": 3.4003197906300353,
+      "objective/train/tokens_used": 839658976,
+      "theoretical_loss": 3.4003197906300353,
+      "tokens_seen": 2469199872
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000254293050874659,
+      "loss": 1.1447,
+      "theoretical_loss": 3.4003197906300353,
+      "tokens_seen": 2469199872
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025425292890386773,
+      "loss": 1.2787,
+      "theoretical_loss": 3.4003054784460027,
+      "tokens_seen": 2469330944
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002542128069330766,
+      "loss": 1.3089,
+      "theoretical_loss": 3.4002911672343408,
+      "tokens_seen": 2469462016
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025417268496228534,
+      "loss": 1.2807,
+      "theoretical_loss": 3.4002768569949318,
+      "tokens_seen": 2469593088
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002541325629914941,
+      "loss": 1.1987,
+      "theoretical_loss": 3.4002625477276576,
+      "tokens_seen": 2469724160
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025409244102070295,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4002482394324005,
+      "tokens_seen": 2469855232
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002540523190499117,
+      "loss": 1.2194,
+      "theoretical_loss": 3.4002339321090442,
+      "tokens_seen": 2469986304
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025401219707912057,
+      "loss": 1.2086,
+      "theoretical_loss": 3.4002196257574697,
+      "tokens_seen": 2470117376
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002539720751083293,
+      "loss": 1.2498,
+      "theoretical_loss": 3.40020532037756,
+      "tokens_seen": 2470248448
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002539319531375381,
+      "loss": 1.2365,
+      "theoretical_loss": 3.400191015969198,
+      "tokens_seen": 2470379520
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025389183116674693,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4001767125322653,
+      "tokens_seen": 2470510592
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025385170919595574,
+      "loss": 1.2015,
+      "theoretical_loss": 3.4001624100666445,
+      "tokens_seen": 2470641664
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002538115872251645,
+      "loss": 1.2104,
+      "theoretical_loss": 3.4001481085722185,
+      "tokens_seen": 2470772736
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002537714652543733,
+      "loss": 1.2712,
+      "theoretical_loss": 3.40013380804887,
+      "tokens_seen": 2470903808
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002537313432835821,
+      "loss": 1.1984,
+      "theoretical_loss": 3.4001195084964815,
+      "tokens_seen": 2471034880
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025369122131279086,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4001052099149347,
+      "tokens_seen": 2471165952
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002536510993419997,
+      "loss": 1.3082,
+      "theoretical_loss": 3.400090912304113,
+      "tokens_seen": 2471297024
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025361097737120847,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4000766156638993,
+      "tokens_seen": 2471428096
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002535708554004173,
+      "loss": 1.2709,
+      "theoretical_loss": 3.4000623199941757,
+      "tokens_seen": 2471559168
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002535307334296261,
+      "loss": 1.2067,
+      "theoretical_loss": 3.400048025294825,
+      "tokens_seen": 2471690240
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025349061145883483,
+      "loss": 1.1791,
+      "theoretical_loss": 3.4000337315657294,
+      "tokens_seen": 2471821312
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002534504894880437,
+      "loss": 1.3252,
+      "theoretical_loss": 3.4000194388067726,
+      "tokens_seen": 2471952384
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025341036751725244,
+      "loss": 1.2636,
+      "theoretical_loss": 3.4000051470178367,
+      "tokens_seen": 2472083456
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025337024554646125,
+      "loss": 1.2944,
+      "theoretical_loss": 3.399990856198804,
+      "tokens_seen": 2472214528
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025333012357567006,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3999765663495585,
+      "tokens_seen": 2472345600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025329000160487886,
+      "loss": 1.1798,
+      "theoretical_loss": 3.399962277469982,
+      "tokens_seen": 2472476672
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002532498796340876,
+      "loss": 1.199,
+      "theoretical_loss": 3.3999479895599576,
+      "tokens_seen": 2472607744
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002532097576632964,
+      "loss": 1.2536,
+      "theoretical_loss": 3.399933702619368,
+      "tokens_seen": 2472738816
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025316963569250523,
+      "loss": 1.2662,
+      "theoretical_loss": 3.399919416648096,
+      "tokens_seen": 2472869888
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000253129513721714,
+      "loss": 1.246,
+      "theoretical_loss": 3.3999051316460247,
+      "tokens_seen": 2473000960
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025308939175092284,
+      "loss": 1.3082,
+      "theoretical_loss": 3.3998908476130367,
+      "tokens_seen": 2473132032
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002530492697801316,
+      "loss": 1.2739,
+      "theoretical_loss": 3.3998765645490154,
+      "tokens_seen": 2473263104
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002530091478093404,
+      "loss": 1.1926,
+      "theoretical_loss": 3.399862282453843,
+      "tokens_seen": 2473394176
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002529690258385492,
+      "loss": 1.2857,
+      "theoretical_loss": 3.3998480013274035,
+      "tokens_seen": 2473525248
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025292890386775796,
+      "loss": 1.2402,
+      "theoretical_loss": 3.399833721169579,
+      "tokens_seen": 2473656320
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025288878189696676,
+      "loss": 1.2172,
+      "theoretical_loss": 3.3998194419802523,
+      "tokens_seen": 2473787392
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025284865992617557,
+      "loss": 1.209,
+      "theoretical_loss": 3.399805163759307,
+      "tokens_seen": 2473918464
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002528085379553844,
+      "loss": 1.2083,
+      "theoretical_loss": 3.399790886506626,
+      "tokens_seen": 2474049536
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002527684159845932,
+      "loss": 1.2565,
+      "theoretical_loss": 3.399776610222092,
+      "tokens_seen": 2474180608
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000252728294013802,
+      "loss": 1.2456,
+      "theoretical_loss": 3.399762334905589,
+      "tokens_seen": 2474311680
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025268817204301074,
+      "loss": 1.2597,
+      "theoretical_loss": 3.3997480605569987,
+      "tokens_seen": 2474442752
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025264805007221955,
+      "loss": 1.1948,
+      "theoretical_loss": 3.3997337871762054,
+      "tokens_seen": 2474573824
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025260792810142835,
+      "loss": 1.2285,
+      "theoretical_loss": 3.3997195147630914,
+      "tokens_seen": 2474704896
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002525678061306371,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3997052433175403,
+      "tokens_seen": 2474835968
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025252768415984597,
+      "loss": 1.2439,
+      "theoretical_loss": 3.3996909728394353,
+      "tokens_seen": 2474967040
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002524875621890547,
+      "loss": 1.2502,
+      "theoretical_loss": 3.3996767033286597,
+      "tokens_seen": 2475098112
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002524474402182635,
+      "loss": 1.2076,
+      "theoretical_loss": 3.399662434785096,
+      "tokens_seen": 2475229184
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025240731824747233,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3996481672086283,
+      "tokens_seen": 2475360256
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025236719627668114,
+      "loss": 1.1401,
+      "theoretical_loss": 3.399633900599139,
+      "tokens_seen": 2475491328
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002523270743058899,
+      "loss": 1.1864,
+      "theoretical_loss": 3.399619634956512,
+      "tokens_seen": 2475622400
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 831408,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2651941776275635,
+      "objective/train/theoretical_loss": 3.399605370280631,
+      "objective/train/tokens_used": 846212576,
+      "theoretical_loss": 3.399605370280631,
+      "tokens_seen": 2475753472
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002522869523350987,
+      "loss": 1.2583,
+      "theoretical_loss": 3.399605370280631,
+      "tokens_seen": 2475753472
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002522468303643075,
+      "loss": 1.2681,
+      "theoretical_loss": 3.399591106571378,
+      "tokens_seen": 2475884544
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002522067083935163,
+      "loss": 1.2607,
+      "theoretical_loss": 3.399576843828637,
+      "tokens_seen": 2476015616
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002521665864227251,
+      "loss": 1.2497,
+      "theoretical_loss": 3.3995625820522917,
+      "tokens_seen": 2476146688
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025212646445193387,
+      "loss": 1.2077,
+      "theoretical_loss": 3.399548321242225,
+      "tokens_seen": 2476277760
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025208634248114273,
+      "loss": 1.2855,
+      "theoretical_loss": 3.3995340613983203,
+      "tokens_seen": 2476408832
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002520462205103515,
+      "loss": 1.2625,
+      "theoretical_loss": 3.3995198025204614,
+      "tokens_seen": 2476539904
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025200609853956023,
+      "loss": 1.1954,
+      "theoretical_loss": 3.3995055446085316,
+      "tokens_seen": 2476670976
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002519659765687691,
+      "loss": 1.2691,
+      "theoretical_loss": 3.399491287662414,
+      "tokens_seen": 2476802048
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025192585459797784,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3994770316819927,
+      "tokens_seen": 2476933120
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025188573262718665,
+      "loss": 1.2536,
+      "theoretical_loss": 3.399462776667151,
+      "tokens_seen": 2477064192
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025184561065639546,
+      "loss": 1.2021,
+      "theoretical_loss": 3.3994485226177713,
+      "tokens_seen": 2477195264
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025180548868560426,
+      "loss": 1.251,
+      "theoretical_loss": 3.3994342695337387,
+      "tokens_seen": 2477326336
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000251765366714813,
+      "loss": 1.1885,
+      "theoretical_loss": 3.3994200174149363,
+      "tokens_seen": 2477457408
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002517252447440218,
+      "loss": 1.2018,
+      "theoretical_loss": 3.399405766261247,
+      "tokens_seen": 2477588480
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025168512277323063,
+      "loss": 1.2612,
+      "theoretical_loss": 3.399391516072555,
+      "tokens_seen": 2477719552
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002516450008024394,
+      "loss": 1.2727,
+      "theoretical_loss": 3.399377266848744,
+      "tokens_seen": 2477850624
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025160487883164824,
+      "loss": 1.2161,
+      "theoretical_loss": 3.399363018589697,
+      "tokens_seen": 2477981696
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000251564756860857,
+      "loss": 1.1921,
+      "theoretical_loss": 3.3993487712952986,
+      "tokens_seen": 2478112768
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025152463489006585,
+      "loss": 1.2428,
+      "theoretical_loss": 3.3993345249654316,
+      "tokens_seen": 2478243840
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002514845129192746,
+      "loss": 1.1967,
+      "theoretical_loss": 3.39932027959998,
+      "tokens_seen": 2478374912
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025144439094848336,
+      "loss": 1.2678,
+      "theoretical_loss": 3.3993060351988276,
+      "tokens_seen": 2478505984
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002514042689776922,
+      "loss": 1.2362,
+      "theoretical_loss": 3.399291791761858,
+      "tokens_seen": 2478637056
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025136414700690097,
+      "loss": 1.1686,
+      "theoretical_loss": 3.3992775492889553,
+      "tokens_seen": 2478768128
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002513240250361098,
+      "loss": 1.293,
+      "theoretical_loss": 3.399263307780003,
+      "tokens_seen": 2478899200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002512839030653186,
+      "loss": 1.2739,
+      "theoretical_loss": 3.399249067234885,
+      "tokens_seen": 2479030272
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002512437810945274,
+      "loss": 1.1696,
+      "theoretical_loss": 3.3992348276534847,
+      "tokens_seen": 2479161344
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025120365912373614,
+      "loss": 1.253,
+      "theoretical_loss": 3.3992205890356866,
+      "tokens_seen": 2479292416
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025116353715294495,
+      "loss": 1.2026,
+      "theoretical_loss": 3.399206351381374,
+      "tokens_seen": 2479423488
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025112341518215375,
+      "loss": 1.25,
+      "theoretical_loss": 3.399192114690431,
+      "tokens_seen": 2479554560
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002510832932113625,
+      "loss": 1.2353,
+      "theoretical_loss": 3.3991778789627416,
+      "tokens_seen": 2479685632
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025104317124057137,
+      "loss": 1.1931,
+      "theoretical_loss": 3.3991636441981896,
+      "tokens_seen": 2479816704
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002510030492697801,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3991494103966593,
+      "tokens_seen": 2479947776
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000250962927298989,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3991351775580334,
+      "tokens_seen": 2480078848
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025092280532819773,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3991209456821974,
+      "tokens_seen": 2480209920
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002508826833574065,
+      "loss": 1.2083,
+      "theoretical_loss": 3.399106714769035,
+      "tokens_seen": 2480340992
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025084256138661534,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3990924848184294,
+      "tokens_seen": 2480472064
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002508024394158241,
+      "loss": 1.1819,
+      "theoretical_loss": 3.399078255830265,
+      "tokens_seen": 2480603136
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002507623174450329,
+      "loss": 1.1552,
+      "theoretical_loss": 3.399064027804426,
+      "tokens_seen": 2480734208
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002507221954742417,
+      "loss": 1.2209,
+      "theoretical_loss": 3.3990498007407965,
+      "tokens_seen": 2480865280
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002506820735034505,
+      "loss": 1.2462,
+      "theoretical_loss": 3.399035574639261,
+      "tokens_seen": 2480996352
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025064195153265927,
+      "loss": 1.229,
+      "theoretical_loss": 3.3990213494997024,
+      "tokens_seen": 2481127424
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025060182956186813,
+      "loss": 1.2152,
+      "theoretical_loss": 3.399007125322006,
+      "tokens_seen": 2481258496
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002505617075910769,
+      "loss": 1.2231,
+      "theoretical_loss": 3.398992902106056,
+      "tokens_seen": 2481389568
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025052158562028563,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3989786798517354,
+      "tokens_seen": 2481520640
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002504814636494945,
+      "loss": 1.3127,
+      "theoretical_loss": 3.39896445855893,
+      "tokens_seen": 2481651712
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025044134167870324,
+      "loss": 1.3375,
+      "theoretical_loss": 3.3989502382275223,
+      "tokens_seen": 2481782784
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025040121970791205,
+      "loss": 1.2666,
+      "theoretical_loss": 3.398936018857398,
+      "tokens_seen": 2481913856
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025036109773712086,
+      "loss": 1.1821,
+      "theoretical_loss": 3.39892180044844,
+      "tokens_seen": 2482044928
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025032097576632966,
+      "loss": 1.2776,
+      "theoretical_loss": 3.398907583000534,
+      "tokens_seen": 2482176000
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 833535,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2504632472991943,
+      "objective/train/theoretical_loss": 3.3988933665135637,
+      "objective/train/tokens_used": 852766176,
+      "theoretical_loss": 3.3988933665135637,
+      "tokens_seen": 2482307072
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025028085379553847,
+      "loss": 1.1643,
+      "theoretical_loss": 3.3988933665135637,
+      "tokens_seen": 2482307072
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002502407318247472,
+      "loss": 1.1952,
+      "theoretical_loss": 3.398879150987413,
+      "tokens_seen": 2482438144
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025020060985395603,
+      "loss": 1.2207,
+      "theoretical_loss": 3.3988649364219663,
+      "tokens_seen": 2482569216
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025016048788316483,
+      "loss": 1.2944,
+      "theoretical_loss": 3.3988507228171088,
+      "tokens_seen": 2482700288
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025012036591237364,
+      "loss": 1.1947,
+      "theoretical_loss": 3.398836510172724,
+      "tokens_seen": 2482831360
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002500802439415824,
+      "loss": 1.1998,
+      "theoretical_loss": 3.3988222984886964,
+      "tokens_seen": 2482962432
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025004012197079125,
+      "loss": 1.2508,
+      "theoretical_loss": 3.398808087764911,
+      "tokens_seen": 2483093504
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00025,
+      "loss": 1.3014,
+      "theoretical_loss": 3.3987938780012517,
+      "tokens_seen": 2483224576
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002499598780292088,
+      "loss": 1.2763,
+      "theoretical_loss": 3.3987796691976033,
+      "tokens_seen": 2483355648
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002499197560584176,
+      "loss": 1.2846,
+      "theoretical_loss": 3.39876546135385,
+      "tokens_seen": 2483486720
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002498796340876264,
+      "loss": 1.2559,
+      "theoretical_loss": 3.398751254469876,
+      "tokens_seen": 2483617792
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002498395121168352,
+      "loss": 1.2306,
+      "theoretical_loss": 3.398737048545567,
+      "tokens_seen": 2483748864
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000249799390146044,
+      "loss": 1.3161,
+      "theoretical_loss": 3.3987228435808063,
+      "tokens_seen": 2483879936
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002497592681752528,
+      "loss": 1.1705,
+      "theoretical_loss": 3.398708639575479,
+      "tokens_seen": 2484011008
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024971914620446154,
+      "loss": 1.253,
+      "theoretical_loss": 3.3986944365294693,
+      "tokens_seen": 2484142080
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024967902423367035,
+      "loss": 1.1957,
+      "theoretical_loss": 3.3986802344426623,
+      "tokens_seen": 2484273152
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024963890226287915,
+      "loss": 1.3069,
+      "theoretical_loss": 3.3986660333149423,
+      "tokens_seen": 2484404224
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024959878029208796,
+      "loss": 1.2133,
+      "theoretical_loss": 3.3986518331461943,
+      "tokens_seen": 2484535296
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024955865832129677,
+      "loss": 1.2624,
+      "theoretical_loss": 3.3986376339363025,
+      "tokens_seen": 2484666368
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002495185363505055,
+      "loss": 1.2665,
+      "theoretical_loss": 3.3986234356851517,
+      "tokens_seen": 2484797440
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002494784143797143,
+      "loss": 1.258,
+      "theoretical_loss": 3.398609238392627,
+      "tokens_seen": 2484928512
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024943829240892313,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3985950420586124,
+      "tokens_seen": 2485059584
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024939817043813194,
+      "loss": 1.1887,
+      "theoretical_loss": 3.398580846682993,
+      "tokens_seen": 2485190656
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024935804846734074,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3985666522656537,
+      "tokens_seen": 2485321728
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024931792649654955,
+      "loss": 1.1903,
+      "theoretical_loss": 3.3985524588064795,
+      "tokens_seen": 2485452800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002492778045257583,
+      "loss": 1.0948,
+      "theoretical_loss": 3.3985382663053545,
+      "tokens_seen": 2485583872
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002492376825549671,
+      "loss": 1.251,
+      "theoretical_loss": 3.3985240747621637,
+      "tokens_seen": 2485714944
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002491975605841759,
+      "loss": 1.1181,
+      "theoretical_loss": 3.3985098841767925,
+      "tokens_seen": 2485846016
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024915743861338467,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3984956945491254,
+      "tokens_seen": 2485977088
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024911731664259347,
+      "loss": 1.2596,
+      "theoretical_loss": 3.3984815058790465,
+      "tokens_seen": 2486108160
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002490771946718023,
+      "loss": 1.278,
+      "theoretical_loss": 3.398467318166442,
+      "tokens_seen": 2486239232
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002490370727010111,
+      "loss": 1.2557,
+      "theoretical_loss": 3.3984531314111965,
+      "tokens_seen": 2486370304
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002489969507302199,
+      "loss": 1.2637,
+      "theoretical_loss": 3.3984389456131945,
+      "tokens_seen": 2486501376
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024895682875942864,
+      "loss": 1.2305,
+      "theoretical_loss": 3.398424760772321,
+      "tokens_seen": 2486632448
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024891670678863745,
+      "loss": 1.2743,
+      "theoretical_loss": 3.3984105768884607,
+      "tokens_seen": 2486763520
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024887658481784626,
+      "loss": 1.1741,
+      "theoretical_loss": 3.3983963939615,
+      "tokens_seen": 2486894592
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024883646284705506,
+      "loss": 1.3028,
+      "theoretical_loss": 3.398382211991322,
+      "tokens_seen": 2487025664
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024879634087626387,
+      "loss": 1.2443,
+      "theoretical_loss": 3.3983680309778133,
+      "tokens_seen": 2487156736
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002487562189054726,
+      "loss": 1.197,
+      "theoretical_loss": 3.398353850920858,
+      "tokens_seen": 2487287808
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024871609693468143,
+      "loss": 1.1725,
+      "theoretical_loss": 3.3983396718203416,
+      "tokens_seen": 2487418880
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024867597496389023,
+      "loss": 1.274,
+      "theoretical_loss": 3.3983254936761487,
+      "tokens_seen": 2487549952
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024863585299309904,
+      "loss": 1.2526,
+      "theoretical_loss": 3.398311316488165,
+      "tokens_seen": 2487681024
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002485957310223078,
+      "loss": 1.247,
+      "theoretical_loss": 3.398297140256276,
+      "tokens_seen": 2487812096
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002485556090515166,
+      "loss": 1.2276,
+      "theoretical_loss": 3.398282964980366,
+      "tokens_seen": 2487943168
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002485154870807254,
+      "loss": 1.2019,
+      "theoretical_loss": 3.39826879066032,
+      "tokens_seen": 2488074240
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002484753651099342,
+      "loss": 1.2108,
+      "theoretical_loss": 3.398254617296024,
+      "tokens_seen": 2488205312
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000248435243139143,
+      "loss": 1.2272,
+      "theoretical_loss": 3.398240444887363,
+      "tokens_seen": 2488336384
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002483951211683518,
+      "loss": 1.2332,
+      "theoretical_loss": 3.398226273434222,
+      "tokens_seen": 2488467456
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002483549991975606,
+      "loss": 1.2203,
+      "theoretical_loss": 3.3982121029364865,
+      "tokens_seen": 2488598528
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002483148772267694,
+      "loss": 1.2443,
+      "theoretical_loss": 3.3981979333940413,
+      "tokens_seen": 2488729600
+    },
+    {
+      "epoch": 0.51,
+      "objective/train/docs_used": 835752,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.260571002960205,
+      "objective/train/theoretical_loss": 3.398183764806772,
+      "objective/train/tokens_used": 859319776,
+      "theoretical_loss": 3.398183764806772,
+      "tokens_seen": 2488860672
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002482747552559782,
+      "loss": 1.2031,
+      "theoretical_loss": 3.398183764806772,
+      "tokens_seen": 2488860672
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000248234633285187,
+      "loss": 1.2076,
+      "theoretical_loss": 3.3981695971745647,
+      "tokens_seen": 2488991744
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024819451131439575,
+      "loss": 1.1976,
+      "theoretical_loss": 3.398155430497303,
+      "tokens_seen": 2489122816
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024815438934360455,
+      "loss": 1.2435,
+      "theoretical_loss": 3.398141264774874,
+      "tokens_seen": 2489253888
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024811426737281336,
+      "loss": 1.1761,
+      "theoretical_loss": 3.3981271000071622,
+      "tokens_seen": 2489384960
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024807414540202217,
+      "loss": 1.2795,
+      "theoretical_loss": 3.3981129361940527,
+      "tokens_seen": 2489516032
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002480340234312309,
+      "loss": 1.2158,
+      "theoretical_loss": 3.3980987733354313,
+      "tokens_seen": 2489647104
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002479939014604397,
+      "loss": 1.288,
+      "theoretical_loss": 3.398084611431184,
+      "tokens_seen": 2489778176
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024795377948964853,
+      "loss": 1.2949,
+      "theoretical_loss": 3.398070450481195,
+      "tokens_seen": 2489909248
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024791365751885734,
+      "loss": 1.1727,
+      "theoretical_loss": 3.398056290485351,
+      "tokens_seen": 2490040320
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024787353554806614,
+      "loss": 1.2864,
+      "theoretical_loss": 3.3980421314435367,
+      "tokens_seen": 2490171392
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024783341357727495,
+      "loss": 1.2013,
+      "theoretical_loss": 3.398027973355638,
+      "tokens_seen": 2490302464
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002477932916064837,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3980138162215403,
+      "tokens_seen": 2490433536
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002477531696356925,
+      "loss": 1.2065,
+      "theoretical_loss": 3.397999660041129,
+      "tokens_seen": 2490564608
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002477130476649013,
+      "loss": 1.2649,
+      "theoretical_loss": 3.39798550481429,
+      "tokens_seen": 2490695680
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002476729256941101,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3979713505409084,
+      "tokens_seen": 2490826752
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024763280372331887,
+      "loss": 1.291,
+      "theoretical_loss": 3.3979571972208706,
+      "tokens_seen": 2490957824
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002475926817525277,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3979430448540615,
+      "tokens_seen": 2491088896
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002475525597817365,
+      "loss": 1.266,
+      "theoretical_loss": 3.397928893440367,
+      "tokens_seen": 2491219968
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002475124378109453,
+      "loss": 1.1783,
+      "theoretical_loss": 3.3979147429796726,
+      "tokens_seen": 2491351040
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024747231584015404,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3979005934718645,
+      "tokens_seen": 2491482112
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024743219386936285,
+      "loss": 1.2817,
+      "theoretical_loss": 3.3978864449168276,
+      "tokens_seen": 2491613184
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024739207189857166,
+      "loss": 1.246,
+      "theoretical_loss": 3.3978722973144486,
+      "tokens_seen": 2491744256
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024735194992778046,
+      "loss": 1.3388,
+      "theoretical_loss": 3.3978581506646126,
+      "tokens_seen": 2491875328
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024731182795698927,
+      "loss": 1.3068,
+      "theoretical_loss": 3.3978440049672054,
+      "tokens_seen": 2492006400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002472717059861981,
+      "loss": 1.2994,
+      "theoretical_loss": 3.397829860222113,
+      "tokens_seen": 2492137472
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024723158401540683,
+      "loss": 1.2263,
+      "theoretical_loss": 3.397815716429221,
+      "tokens_seen": 2492268544
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024719146204461563,
+      "loss": 1.1886,
+      "theoretical_loss": 3.3978015735884153,
+      "tokens_seen": 2492399616
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024715134007382444,
+      "loss": 1.1848,
+      "theoretical_loss": 3.397787431699582,
+      "tokens_seen": 2492530688
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024711121810303325,
+      "loss": 1.2143,
+      "theoretical_loss": 3.3977732907626064,
+      "tokens_seen": 2492661760
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000247071096132242,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3977591507773752,
+      "tokens_seen": 2492792832
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002470309741614508,
+      "loss": 1.211,
+      "theoretical_loss": 3.3977450117437735,
+      "tokens_seen": 2492923904
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002469908521906596,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3977308736616876,
+      "tokens_seen": 2493054976
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002469507302198684,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3977167365310033,
+      "tokens_seen": 2493186048
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002469106082490772,
+      "loss": 1.1855,
+      "theoretical_loss": 3.397702600351607,
+      "tokens_seen": 2493317120
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000246870486278286,
+      "loss": 1.2462,
+      "theoretical_loss": 3.397688465123384,
+      "tokens_seen": 2493448192
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002468303643074948,
+      "loss": 1.2661,
+      "theoretical_loss": 3.397674330846221,
+      "tokens_seen": 2493579264
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002467902423367036,
+      "loss": 1.2159,
+      "theoretical_loss": 3.397660197520003,
+      "tokens_seen": 2493710336
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002467501203659124,
+      "loss": 1.2831,
+      "theoretical_loss": 3.3976460651446176,
+      "tokens_seen": 2493841408
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002467099983951212,
+      "loss": 1.283,
+      "theoretical_loss": 3.3976319337199494,
+      "tokens_seen": 2493972480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024666987642432995,
+      "loss": 1.1999,
+      "theoretical_loss": 3.3976178032458852,
+      "tokens_seen": 2494103552
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024662975445353876,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3976036737223114,
+      "tokens_seen": 2494234624
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024658963248274757,
+      "loss": 1.306,
+      "theoretical_loss": 3.3975895451491134,
+      "tokens_seen": 2494365696
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002465495105119563,
+      "loss": 1.2657,
+      "theoretical_loss": 3.3975754175261774,
+      "tokens_seen": 2494496768
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002465093885411651,
+      "loss": 1.2181,
+      "theoretical_loss": 3.39756129085339,
+      "tokens_seen": 2494627840
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024646926657037393,
+      "loss": 1.2648,
+      "theoretical_loss": 3.3975471651306375,
+      "tokens_seen": 2494758912
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024642914459958274,
+      "loss": 1.1737,
+      "theoretical_loss": 3.3975330403578052,
+      "tokens_seen": 2494889984
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024638902262879154,
+      "loss": 1.2157,
+      "theoretical_loss": 3.3975189165347803,
+      "tokens_seen": 2495021056
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024634890065800035,
+      "loss": 1.2625,
+      "theoretical_loss": 3.397504793661448,
+      "tokens_seen": 2495152128
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024630877868720916,
+      "loss": 1.1661,
+      "theoretical_loss": 3.397490671737696,
+      "tokens_seen": 2495283200
+    },
+    {
+      "epoch": 0.51,
+      "objective/train/docs_used": 837794,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2193864583969116,
+      "objective/train/theoretical_loss": 3.3974765507634097,
+      "objective/train/tokens_used": 865873376,
+      "theoretical_loss": 3.3974765507634097,
+      "tokens_seen": 2495414272
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002462686567164179,
+      "loss": 1.1785,
+      "theoretical_loss": 3.3974765507634097,
+      "tokens_seen": 2495414272
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002462285347456267,
+      "loss": 1.2946,
+      "theoretical_loss": 3.397462430738475,
+      "tokens_seen": 2495545344
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002461884127748355,
+      "loss": 1.218,
+      "theoretical_loss": 3.397448311662779,
+      "tokens_seen": 2495676416
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002461482908040443,
+      "loss": 1.2415,
+      "theoretical_loss": 3.397434193536208,
+      "tokens_seen": 2495807488
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002461081688332531,
+      "loss": 1.3193,
+      "theoretical_loss": 3.3974200763586477,
+      "tokens_seen": 2495938560
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002460680468624619,
+      "loss": 1.2348,
+      "theoretical_loss": 3.397405960129985,
+      "tokens_seen": 2496069632
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002460279248916707,
+      "loss": 1.2455,
+      "theoretical_loss": 3.397391844850106,
+      "tokens_seen": 2496200704
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024598780292087944,
+      "loss": 1.2679,
+      "theoretical_loss": 3.3973777305188975,
+      "tokens_seen": 2496331776
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024594768095008825,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3973636171362456,
+      "tokens_seen": 2496462848
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024590755897929706,
+      "loss": 1.2372,
+      "theoretical_loss": 3.397349504702037,
+      "tokens_seen": 2496593920
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024586743700850586,
+      "loss": 1.2319,
+      "theoretical_loss": 3.3973353932161574,
+      "tokens_seen": 2496724992
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024582731503771467,
+      "loss": 1.2969,
+      "theoretical_loss": 3.3973212826784946,
+      "tokens_seen": 2496856064
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002457871930669235,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3973071730889344,
+      "tokens_seen": 2496987136
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002457470710961323,
+      "loss": 1.2598,
+      "theoretical_loss": 3.3972930644473633,
+      "tokens_seen": 2497118208
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024570694912534103,
+      "loss": 1.2489,
+      "theoretical_loss": 3.397278956753668,
+      "tokens_seen": 2497249280
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024566682715454984,
+      "loss": 1.1965,
+      "theoretical_loss": 3.397264850007735,
+      "tokens_seen": 2497380352
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024562670518375865,
+      "loss": 1.331,
+      "theoretical_loss": 3.3972507442094506,
+      "tokens_seen": 2497511424
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002455865832129674,
+      "loss": 1.2572,
+      "theoretical_loss": 3.397236639358702,
+      "tokens_seen": 2497642496
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002455464612421762,
+      "loss": 1.2848,
+      "theoretical_loss": 3.3972225354553753,
+      "tokens_seen": 2497773568
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000245506339271385,
+      "loss": 1.2371,
+      "theoretical_loss": 3.3972084324993577,
+      "tokens_seen": 2497904640
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002454662173005938,
+      "loss": 1.1774,
+      "theoretical_loss": 3.397194330490535,
+      "tokens_seen": 2498035712
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002454260953298026,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3971802294287956,
+      "tokens_seen": 2498166784
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002453859733590114,
+      "loss": 1.2372,
+      "theoretical_loss": 3.397166129314024,
+      "tokens_seen": 2498297856
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002453458513882202,
+      "loss": 1.3196,
+      "theoretical_loss": 3.3971520301461084,
+      "tokens_seen": 2498428928
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000245305729417429,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3971379319249353,
+      "tokens_seen": 2498560000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002452656074466378,
+      "loss": 1.2299,
+      "theoretical_loss": 3.397123834650391,
+      "tokens_seen": 2498691072
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002452254854758466,
+      "loss": 1.2517,
+      "theoretical_loss": 3.3971097383223627,
+      "tokens_seen": 2498822144
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002451853635050554,
+      "loss": 1.2095,
+      "theoretical_loss": 3.397095642940737,
+      "tokens_seen": 2498953216
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024514524153426416,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3970815485054007,
+      "tokens_seen": 2499084288
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024510511956347297,
+      "loss": 1.216,
+      "theoretical_loss": 3.397067455016241,
+      "tokens_seen": 2499215360
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024506499759268177,
+      "loss": 1.2121,
+      "theoretical_loss": 3.3970533624731445,
+      "tokens_seen": 2499346432
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002450248756218905,
+      "loss": 1.247,
+      "theoretical_loss": 3.397039270875998,
+      "tokens_seen": 2499477504
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024498475365109933,
+      "loss": 1.2223,
+      "theoretical_loss": 3.3970251802246887,
+      "tokens_seen": 2499608576
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024494463168030814,
+      "loss": 1.2154,
+      "theoretical_loss": 3.3970110905191033,
+      "tokens_seen": 2499739648
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024490450970951694,
+      "loss": 1.1808,
+      "theoretical_loss": 3.3969970017591287,
+      "tokens_seen": 2499870720
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024486438773872575,
+      "loss": 1.269,
+      "theoretical_loss": 3.396982913944652,
+      "tokens_seen": 2500001792
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024482426576793456,
+      "loss": 1.2522,
+      "theoretical_loss": 3.39696882707556,
+      "tokens_seen": 2500132864
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002447841437971433,
+      "loss": 1.2737,
+      "theoretical_loss": 3.39695474115174,
+      "tokens_seen": 2500263936
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002447440218263521,
+      "loss": 1.2154,
+      "theoretical_loss": 3.3969406561730784,
+      "tokens_seen": 2500395008
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002447038998555609,
+      "loss": 1.2906,
+      "theoretical_loss": 3.3969265721394635,
+      "tokens_seen": 2500526080
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002446637778847697,
+      "loss": 1.2787,
+      "theoretical_loss": 3.3969124890507807,
+      "tokens_seen": 2500657152
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002446236559139785,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3968984069069186,
+      "tokens_seen": 2500788224
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002445835339431873,
+      "loss": 1.2391,
+      "theoretical_loss": 3.396884325707763,
+      "tokens_seen": 2500919296
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002445434119723961,
+      "loss": 1.309,
+      "theoretical_loss": 3.3968702454532025,
+      "tokens_seen": 2501050368
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002445032900016049,
+      "loss": 1.2399,
+      "theoretical_loss": 3.396856166143123,
+      "tokens_seen": 2501181440
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024446316803081365,
+      "loss": 1.2367,
+      "theoretical_loss": 3.3968420877774115,
+      "tokens_seen": 2501312512
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024442304606002246,
+      "loss": 1.2634,
+      "theoretical_loss": 3.3968280103559563,
+      "tokens_seen": 2501443584
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024438292408923126,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3968139338786436,
+      "tokens_seen": 2501574656
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024434280211844007,
+      "loss": 1.1963,
+      "theoretical_loss": 3.396799858345361,
+      "tokens_seen": 2501705728
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002443026801476489,
+      "loss": 1.213,
+      "theoretical_loss": 3.396785783755996,
+      "tokens_seen": 2501836800
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 839945,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2034003734588623,
+      "objective/train/theoretical_loss": 3.3967717101104355,
+      "objective/train/tokens_used": 872426976,
+      "theoretical_loss": 3.3967717101104355,
+      "tokens_seen": 2501967872
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002442625581768577,
+      "loss": 1.246,
+      "theoretical_loss": 3.3967717101104355,
+      "tokens_seen": 2501967872
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024422243620606643,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3967576374085673,
+      "tokens_seen": 2502098944
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024418231423527524,
+      "loss": 1.2106,
+      "theoretical_loss": 3.3967435656502776,
+      "tokens_seen": 2502230016
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024414219226448405,
+      "loss": 1.251,
+      "theoretical_loss": 3.396729494835455,
+      "tokens_seen": 2502361088
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024410207029369282,
+      "loss": 1.2253,
+      "theoretical_loss": 3.396715424963986,
+      "tokens_seen": 2502492160
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024406194832290163,
+      "loss": 1.2704,
+      "theoretical_loss": 3.396701356035758,
+      "tokens_seen": 2502623232
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024402182635211044,
+      "loss": 1.2485,
+      "theoretical_loss": 3.3966872880506584,
+      "tokens_seen": 2502754304
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024398170438131922,
+      "loss": 1.1975,
+      "theoretical_loss": 3.396673221008575,
+      "tokens_seen": 2502885376
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024394158241052802,
+      "loss": 1.2803,
+      "theoretical_loss": 3.396659154909395,
+      "tokens_seen": 2503016448
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002439014604397368,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3966450897530054,
+      "tokens_seen": 2503147520
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024386133846894558,
+      "loss": 1.2644,
+      "theoretical_loss": 3.3966310255392944,
+      "tokens_seen": 2503278592
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002438212164981544,
+      "loss": 1.3237,
+      "theoretical_loss": 3.3966169622681486,
+      "tokens_seen": 2503409664
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002437810945273632,
+      "loss": 1.3002,
+      "theoretical_loss": 3.3966028999394564,
+      "tokens_seen": 2503540736
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.000243740972556572,
+      "loss": 1.193,
+      "theoretical_loss": 3.3965888385531047,
+      "tokens_seen": 2503671808
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024370085058578078,
+      "loss": 1.2163,
+      "theoretical_loss": 3.396574778108981,
+      "tokens_seen": 2503802880
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024366072861498959,
+      "loss": 1.2439,
+      "theoretical_loss": 3.396560718606973,
+      "tokens_seen": 2503933952
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024362060664419836,
+      "loss": 1.1582,
+      "theoretical_loss": 3.3965466600469685,
+      "tokens_seen": 2504065024
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024358048467340714,
+      "loss": 1.2718,
+      "theoretical_loss": 3.396532602428855,
+      "tokens_seen": 2504196096
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024354036270261595,
+      "loss": 1.2571,
+      "theoretical_loss": 3.3965185457525195,
+      "tokens_seen": 2504327168
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024350024073182476,
+      "loss": 1.256,
+      "theoretical_loss": 3.3965044900178505,
+      "tokens_seen": 2504458240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024346011876103356,
+      "loss": 1.1596,
+      "theoretical_loss": 3.396490435224735,
+      "tokens_seen": 2504589312
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024341999679024234,
+      "loss": 1.2237,
+      "theoretical_loss": 3.396476381373061,
+      "tokens_seen": 2504720384
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024337987481945115,
+      "loss": 1.246,
+      "theoretical_loss": 3.396462328462716,
+      "tokens_seen": 2504851456
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024333975284865993,
+      "loss": 1.2917,
+      "theoretical_loss": 3.3964482764935875,
+      "tokens_seen": 2504982528
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002432996308778687,
+      "loss": 1.2922,
+      "theoretical_loss": 3.3964342254655637,
+      "tokens_seen": 2505113600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002432595089070775,
+      "loss": 1.2557,
+      "theoretical_loss": 3.396420175378532,
+      "tokens_seen": 2505244672
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024321938693628632,
+      "loss": 1.2053,
+      "theoretical_loss": 3.396406126232381,
+      "tokens_seen": 2505375744
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002431792649654951,
+      "loss": 1.2421,
+      "theoretical_loss": 3.396392078026997,
+      "tokens_seen": 2505506816
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002431391429947039,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3963780307622686,
+      "tokens_seen": 2505637888
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002430990210239127,
+      "loss": 1.2685,
+      "theoretical_loss": 3.3963639844380835,
+      "tokens_seen": 2505768960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024305889905312152,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3963499390543297,
+      "tokens_seen": 2505900032
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024301877708233027,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3963358946108952,
+      "tokens_seen": 2506031104
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024297865511153908,
+      "loss": 1.1796,
+      "theoretical_loss": 3.3963218511076674,
+      "tokens_seen": 2506162176
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024293853314074788,
+      "loss": 1.2365,
+      "theoretical_loss": 3.396307808544534,
+      "tokens_seen": 2506293248
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024289841116995666,
+      "loss": 1.2397,
+      "theoretical_loss": 3.3962937669213837,
+      "tokens_seen": 2506424320
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024285828919916547,
+      "loss": 1.2351,
+      "theoretical_loss": 3.396279726238104,
+      "tokens_seen": 2506555392
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024281816722837427,
+      "loss": 1.2454,
+      "theoretical_loss": 3.396265686494583,
+      "tokens_seen": 2506686464
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024277804525758308,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3962516476907085,
+      "tokens_seen": 2506817536
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024273792328679183,
+      "loss": 1.1991,
+      "theoretical_loss": 3.3962376098263682,
+      "tokens_seen": 2506948608
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024269780131600064,
+      "loss": 1.2733,
+      "theoretical_loss": 3.3962235729014507,
+      "tokens_seen": 2507079680
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024265767934520945,
+      "loss": 1.3188,
+      "theoretical_loss": 3.3962095369158436,
+      "tokens_seen": 2507210752
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024261755737441822,
+      "loss": 1.2539,
+      "theoretical_loss": 3.3961955018694354,
+      "tokens_seen": 2507341824
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024257743540362703,
+      "loss": 1.2645,
+      "theoretical_loss": 3.396181467762114,
+      "tokens_seen": 2507472896
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024253731343283584,
+      "loss": 1.2338,
+      "theoretical_loss": 3.3961674345937665,
+      "tokens_seen": 2507603968
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024249719146204464,
+      "loss": 1.2165,
+      "theoretical_loss": 3.3961534023642828,
+      "tokens_seen": 2507735040
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024245706949125342,
+      "loss": 1.2882,
+      "theoretical_loss": 3.3961393710735495,
+      "tokens_seen": 2507866112
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002424169475204622,
+      "loss": 1.2642,
+      "theoretical_loss": 3.3961253407214556,
+      "tokens_seen": 2507997184
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.000242376825549671,
+      "loss": 1.2288,
+      "theoretical_loss": 3.3961113113078887,
+      "tokens_seen": 2508128256
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002423367035788798,
+      "loss": 1.2559,
+      "theoretical_loss": 3.3960972828327374,
+      "tokens_seen": 2508259328
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002422965816080886,
+      "loss": 1.3048,
+      "theoretical_loss": 3.3960832552958897,
+      "tokens_seen": 2508390400
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 842199,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2703614234924316,
+      "objective/train/theoretical_loss": 3.396069228697234,
+      "objective/train/tokens_used": 878980576,
+      "theoretical_loss": 3.396069228697234,
+      "tokens_seen": 2508521472
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002422564596372974,
+      "loss": 1.2637,
+      "theoretical_loss": 3.396069228697234,
+      "tokens_seen": 2508521472
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024221633766650618,
+      "loss": 1.2617,
+      "theoretical_loss": 3.396055203036658,
+      "tokens_seen": 2508652544
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024217621569571499,
+      "loss": 1.2434,
+      "theoretical_loss": 3.396041178314051,
+      "tokens_seen": 2508783616
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024213609372492376,
+      "loss": 1.1896,
+      "theoretical_loss": 3.3960271545293,
+      "tokens_seen": 2508914688
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024209597175413257,
+      "loss": 1.266,
+      "theoretical_loss": 3.3960131316822944,
+      "tokens_seen": 2509045760
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024205584978334135,
+      "loss": 1.2186,
+      "theoretical_loss": 3.3959991097729216,
+      "tokens_seen": 2509176832
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024201572781255016,
+      "loss": 1.2367,
+      "theoretical_loss": 3.3959850888010705,
+      "tokens_seen": 2509307904
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024197560584175896,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3959710687666296,
+      "tokens_seen": 2509438976
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024193548387096774,
+      "loss": 1.1997,
+      "theoretical_loss": 3.395957049669487,
+      "tokens_seen": 2509570048
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024189536190017655,
+      "loss": 1.2657,
+      "theoretical_loss": 3.3959430315095305,
+      "tokens_seen": 2509701120
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024185523992938533,
+      "loss": 1.2929,
+      "theoretical_loss": 3.3959290142866494,
+      "tokens_seen": 2509832192
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024181511795859413,
+      "loss": 1.1921,
+      "theoretical_loss": 3.395914998000732,
+      "tokens_seen": 2509963264
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002417749959878029,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3959009826516664,
+      "tokens_seen": 2510094336
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024173487401701172,
+      "loss": 1.2744,
+      "theoretical_loss": 3.395886968239341,
+      "tokens_seen": 2510225408
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024169475204622053,
+      "loss": 1.2972,
+      "theoretical_loss": 3.395872954763645,
+      "tokens_seen": 2510356480
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002416546300754293,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3958589422244656,
+      "tokens_seen": 2510487552
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002416145081046381,
+      "loss": 1.2791,
+      "theoretical_loss": 3.395844930621693,
+      "tokens_seen": 2510618624
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024157438613384692,
+      "loss": 1.3114,
+      "theoretical_loss": 3.3958309199552144,
+      "tokens_seen": 2510749696
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002415342641630557,
+      "loss": 1.2676,
+      "theoretical_loss": 3.3958169102249185,
+      "tokens_seen": 2510880768
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024149414219226448,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3958029014306947,
+      "tokens_seen": 2511011840
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024145402022147328,
+      "loss": 1.2202,
+      "theoretical_loss": 3.395788893572431,
+      "tokens_seen": 2511142912
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002414138982506821,
+      "loss": 1.3742,
+      "theoretical_loss": 3.3957748866500164,
+      "tokens_seen": 2511273984
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024137377627989087,
+      "loss": 1.3125,
+      "theoretical_loss": 3.395760880663339,
+      "tokens_seen": 2511405056
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024133365430909967,
+      "loss": 1.2126,
+      "theoretical_loss": 3.3957468756122875,
+      "tokens_seen": 2511536128
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024129353233830848,
+      "loss": 1.2167,
+      "theoretical_loss": 3.395732871496751,
+      "tokens_seen": 2511667200
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024125341036751723,
+      "loss": 1.244,
+      "theoretical_loss": 3.395718868316618,
+      "tokens_seen": 2511798272
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024121328839672604,
+      "loss": 1.234,
+      "theoretical_loss": 3.3957048660717772,
+      "tokens_seen": 2511929344
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024117316642593484,
+      "loss": 1.2156,
+      "theoretical_loss": 3.395690864762117,
+      "tokens_seen": 2512060416
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024113304445514365,
+      "loss": 1.2479,
+      "theoretical_loss": 3.395676864387527,
+      "tokens_seen": 2512191488
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024109292248435243,
+      "loss": 1.2549,
+      "theoretical_loss": 3.3956628649478953,
+      "tokens_seen": 2512322560
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024105280051356124,
+      "loss": 1.1871,
+      "theoretical_loss": 3.3956488664431106,
+      "tokens_seen": 2512453632
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024101267854277004,
+      "loss": 1.1731,
+      "theoretical_loss": 3.395634868873062,
+      "tokens_seen": 2512584704
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002409725565719788,
+      "loss": 1.2308,
+      "theoretical_loss": 3.3956208722376386,
+      "tokens_seen": 2512715776
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002409324346011876,
+      "loss": 1.2881,
+      "theoretical_loss": 3.3956068765367284,
+      "tokens_seen": 2512846848
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002408923126303964,
+      "loss": 1.1891,
+      "theoretical_loss": 3.395592881770221,
+      "tokens_seen": 2512977920
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024085219065960521,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3955788879380053,
+      "tokens_seen": 2513108992
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.000240812068688814,
+      "loss": 1.2067,
+      "theoretical_loss": 3.39556489503997,
+      "tokens_seen": 2513240064
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002407719467180228,
+      "loss": 1.2004,
+      "theoretical_loss": 3.395550903076004,
+      "tokens_seen": 2513371136
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002407318247472316,
+      "loss": 1.2577,
+      "theoretical_loss": 3.395536912045996,
+      "tokens_seen": 2513502208
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024069170277644038,
+      "loss": 1.2457,
+      "theoretical_loss": 3.395522921949835,
+      "tokens_seen": 2513633280
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024065158080564916,
+      "loss": 1.2763,
+      "theoretical_loss": 3.3955089327874104,
+      "tokens_seen": 2513764352
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024061145883485797,
+      "loss": 1.2223,
+      "theoretical_loss": 3.395494944558611,
+      "tokens_seen": 2513895424
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024057133686406678,
+      "loss": 1.3025,
+      "theoretical_loss": 3.3954809572633255,
+      "tokens_seen": 2514026496
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024053121489327556,
+      "loss": 1.2939,
+      "theoretical_loss": 3.3954669709014436,
+      "tokens_seen": 2514157568
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024049109292248436,
+      "loss": 1.2686,
+      "theoretical_loss": 3.395452985472854,
+      "tokens_seen": 2514288640
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024045097095169317,
+      "loss": 1.2599,
+      "theoretical_loss": 3.3954390009774458,
+      "tokens_seen": 2514419712
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024041084898090195,
+      "loss": 1.2881,
+      "theoretical_loss": 3.395425017415108,
+      "tokens_seen": 2514550784
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024037072701011073,
+      "loss": 1.236,
+      "theoretical_loss": 3.3954110347857296,
+      "tokens_seen": 2514681856
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024033060503931953,
+      "loss": 1.2238,
+      "theoretical_loss": 3.3953970530891997,
+      "tokens_seen": 2514812928
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024029048306852834,
+      "loss": 1.221,
+      "theoretical_loss": 3.395383072325408,
+      "tokens_seen": 2514944000
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 844467,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2637052536010742,
+      "objective/train/theoretical_loss": 3.395369092494243,
+      "objective/train/tokens_used": 885534176,
+      "theoretical_loss": 3.395369092494243,
+      "tokens_seen": 2515075072
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024025036109773712,
+      "loss": 1.2888,
+      "theoretical_loss": 3.395369092494243,
+      "tokens_seen": 2515075072
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024021023912694593,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3953551135955946,
+      "tokens_seen": 2515206144
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024017011715615473,
+      "loss": 1.2389,
+      "theoretical_loss": 3.3953411356293515,
+      "tokens_seen": 2515337216
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002401299951853635,
+      "loss": 1.2293,
+      "theoretical_loss": 3.3953271585954026,
+      "tokens_seen": 2515468288
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024008987321457232,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3953131824936382,
+      "tokens_seen": 2515599360
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002400497512437811,
+      "loss": 1.2404,
+      "theoretical_loss": 3.3952992073239465,
+      "tokens_seen": 2515730432
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024000962927298988,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3952852330862173,
+      "tokens_seen": 2515861504
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00023996950730219868,
+      "loss": 1.2245,
+      "theoretical_loss": 3.39527125978034,
+      "tokens_seen": 2515992576
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002399293853314075,
+      "loss": 1.2952,
+      "theoretical_loss": 3.3952572874062037,
+      "tokens_seen": 2516123648
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002398892633606163,
+      "loss": 1.2786,
+      "theoretical_loss": 3.3952433159636977,
+      "tokens_seen": 2516254720
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023984914138982507,
+      "loss": 1.2703,
+      "theoretical_loss": 3.3952293454527114,
+      "tokens_seen": 2516385792
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023980901941903388,
+      "loss": 1.3105,
+      "theoretical_loss": 3.3952153758731347,
+      "tokens_seen": 2516516864
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023976889744824266,
+      "loss": 1.2506,
+      "theoretical_loss": 3.3952014072248557,
+      "tokens_seen": 2516647936
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023972877547745144,
+      "loss": 1.2637,
+      "theoretical_loss": 3.3951874395077652,
+      "tokens_seen": 2516779008
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023968865350666024,
+      "loss": 1.2342,
+      "theoretical_loss": 3.395173472721752,
+      "tokens_seen": 2516910080
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023964853153586905,
+      "loss": 1.2079,
+      "theoretical_loss": 3.3951595068667055,
+      "tokens_seen": 2517041152
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023960840956507786,
+      "loss": 1.2293,
+      "theoretical_loss": 3.395145541942515,
+      "tokens_seen": 2517172224
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023956828759428664,
+      "loss": 1.3096,
+      "theoretical_loss": 3.395131577949071,
+      "tokens_seen": 2517303296
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023952816562349544,
+      "loss": 1.2588,
+      "theoretical_loss": 3.395117614886262,
+      "tokens_seen": 2517434368
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023948804365270422,
+      "loss": 1.2228,
+      "theoretical_loss": 3.3951036527539773,
+      "tokens_seen": 2517565440
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.000239447921681913,
+      "loss": 1.2814,
+      "theoretical_loss": 3.395089691552107,
+      "tokens_seen": 2517696512
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002394077997111218,
+      "loss": 1.19,
+      "theoretical_loss": 3.3950757312805413,
+      "tokens_seen": 2517827584
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002393676777403306,
+      "loss": 1.2102,
+      "theoretical_loss": 3.3950617719391687,
+      "tokens_seen": 2517958656
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023932755576953942,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3950478135278797,
+      "tokens_seen": 2518089728
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002392874337987482,
+      "loss": 1.3371,
+      "theoretical_loss": 3.3950338560465627,
+      "tokens_seen": 2518220800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.000239247311827957,
+      "loss": 1.1799,
+      "theoretical_loss": 3.395019899495108,
+      "tokens_seen": 2518351872
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002392071898571658,
+      "loss": 1.2073,
+      "theoretical_loss": 3.3950059438734064,
+      "tokens_seen": 2518482944
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023916706788637456,
+      "loss": 1.2136,
+      "theoretical_loss": 3.394991989181346,
+      "tokens_seen": 2518614016
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023912694591558337,
+      "loss": 1.1323,
+      "theoretical_loss": 3.3949780354188164,
+      "tokens_seen": 2518745088
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023908682394479218,
+      "loss": 1.2569,
+      "theoretical_loss": 3.394964082585709,
+      "tokens_seen": 2518876160
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023904670197400096,
+      "loss": 1.2352,
+      "theoretical_loss": 3.3949501306819116,
+      "tokens_seen": 2519007232
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023900658000320976,
+      "loss": 1.2519,
+      "theoretical_loss": 3.394936179707315,
+      "tokens_seen": 2519138304
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023896645803241857,
+      "loss": 1.2923,
+      "theoretical_loss": 3.394922229661809,
+      "tokens_seen": 2519269376
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023892633606162737,
+      "loss": 1.2585,
+      "theoretical_loss": 3.394908280545283,
+      "tokens_seen": 2519400448
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023888621409083613,
+      "loss": 1.1427,
+      "theoretical_loss": 3.394894332357627,
+      "tokens_seen": 2519531520
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023884609212004493,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3948803850987312,
+      "tokens_seen": 2519662592
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023880597014925374,
+      "loss": 1.2455,
+      "theoretical_loss": 3.3948664387684846,
+      "tokens_seen": 2519793664
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023876584817846252,
+      "loss": 1.2357,
+      "theoretical_loss": 3.394852493366778,
+      "tokens_seen": 2519924736
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023872572620767132,
+      "loss": 1.2828,
+      "theoretical_loss": 3.3948385488935005,
+      "tokens_seen": 2520055808
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023868560423688013,
+      "loss": 1.1533,
+      "theoretical_loss": 3.3948246053485427,
+      "tokens_seen": 2520186880
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023864548226608894,
+      "loss": 1.2549,
+      "theoretical_loss": 3.3948106627317935,
+      "tokens_seen": 2520317952
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002386053602952977,
+      "loss": 1.2603,
+      "theoretical_loss": 3.394796721043144,
+      "tokens_seen": 2520449024
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002385652383245065,
+      "loss": 1.2001,
+      "theoretical_loss": 3.3947827802824837,
+      "tokens_seen": 2520580096
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002385251163537153,
+      "loss": 1.2556,
+      "theoretical_loss": 3.3947688404497023,
+      "tokens_seen": 2520711168
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023848499438292408,
+      "loss": 1.2946,
+      "theoretical_loss": 3.39475490154469,
+      "tokens_seen": 2520842240
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002384448724121329,
+      "loss": 1.33,
+      "theoretical_loss": 3.3947409635673367,
+      "tokens_seen": 2520973312
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002384047504413417,
+      "loss": 1.2444,
+      "theoretical_loss": 3.394727026517533,
+      "tokens_seen": 2521104384
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002383646284705505,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3947130903951686,
+      "tokens_seen": 2521235456
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023832450649975928,
+      "loss": 1.3006,
+      "theoretical_loss": 3.3946991552001333,
+      "tokens_seen": 2521366528
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023828438452896806,
+      "loss": 1.2105,
+      "theoretical_loss": 3.3946852209323173,
+      "tokens_seen": 2521497600
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/docs_used": 846407,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2130433320999146,
+      "objective/train/theoretical_loss": 3.394671287591611,
+      "objective/train/tokens_used": 892087776,
+      "theoretical_loss": 3.394671287591611,
+      "tokens_seen": 2521628672
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023824426255817686,
+      "loss": 1.27,
+      "theoretical_loss": 3.394671287591611,
+      "tokens_seen": 2521628672
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023820414058738564,
+      "loss": 1.2471,
+      "theoretical_loss": 3.3946573551779045,
+      "tokens_seen": 2521759744
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023816401861659445,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3946434236910874,
+      "tokens_seen": 2521890816
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023812389664580326,
+      "loss": 1.2372,
+      "theoretical_loss": 3.394629493131051,
+      "tokens_seen": 2522021888
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023808377467501204,
+      "loss": 1.2333,
+      "theoretical_loss": 3.394615563497684,
+      "tokens_seen": 2522152960
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023804365270422084,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3946016347908774,
+      "tokens_seen": 2522284032
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023800353073342962,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3945877070105217,
+      "tokens_seen": 2522415104
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023796340876263843,
+      "loss": 1.2606,
+      "theoretical_loss": 3.394573780156507,
+      "tokens_seen": 2522546176
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002379232867918472,
+      "loss": 1.2525,
+      "theoretical_loss": 3.3945598542287225,
+      "tokens_seen": 2522677248
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.000237883164821056,
+      "loss": 1.2718,
+      "theoretical_loss": 3.3945459292270606,
+      "tokens_seen": 2522808320
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023784304285026482,
+      "loss": 1.2523,
+      "theoretical_loss": 3.3945320051514094,
+      "tokens_seen": 2522939392
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002378029208794736,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3945180820016603,
+      "tokens_seen": 2523070464
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002377627989086824,
+      "loss": 1.2256,
+      "theoretical_loss": 3.394504159777704,
+      "tokens_seen": 2523201536
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002377226769378912,
+      "loss": 1.1956,
+      "theoretical_loss": 3.39449023847943,
+      "tokens_seen": 2523332608
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002376825549671,
+      "loss": 1.2333,
+      "theoretical_loss": 3.394476318106729,
+      "tokens_seen": 2523463680
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023764243299630877,
+      "loss": 1.1488,
+      "theoretical_loss": 3.394462398659492,
+      "tokens_seen": 2523594752
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023760231102551758,
+      "loss": 1.2286,
+      "theoretical_loss": 3.394448480137608,
+      "tokens_seen": 2523725824
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023756218905472638,
+      "loss": 1.2048,
+      "theoretical_loss": 3.394434562540969,
+      "tokens_seen": 2523856896
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023752206708393516,
+      "loss": 1.2202,
+      "theoretical_loss": 3.3944206458694643,
+      "tokens_seen": 2523987968
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023748194511314397,
+      "loss": 1.2107,
+      "theoretical_loss": 3.3944067301229848,
+      "tokens_seen": 2524119040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023744182314235277,
+      "loss": 1.3092,
+      "theoretical_loss": 3.394392815301421,
+      "tokens_seen": 2524250112
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023740170117156155,
+      "loss": 1.2732,
+      "theoretical_loss": 3.3943789014046635,
+      "tokens_seen": 2524381184
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023736157920077033,
+      "loss": 1.194,
+      "theoretical_loss": 3.394364988432603,
+      "tokens_seen": 2524512256
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023732145722997914,
+      "loss": 1.2301,
+      "theoretical_loss": 3.394351076385129,
+      "tokens_seen": 2524643328
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023728133525918795,
+      "loss": 1.1954,
+      "theoretical_loss": 3.394337165262133,
+      "tokens_seen": 2524774400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023724121328839672,
+      "loss": 1.2634,
+      "theoretical_loss": 3.394323255063506,
+      "tokens_seen": 2524905472
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023720109131760553,
+      "loss": 1.3229,
+      "theoretical_loss": 3.3943093457891376,
+      "tokens_seen": 2525036544
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023716096934681434,
+      "loss": 1.2612,
+      "theoretical_loss": 3.3942954374389185,
+      "tokens_seen": 2525167616
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002371208473760231,
+      "loss": 1.2574,
+      "theoretical_loss": 3.39428153001274,
+      "tokens_seen": 2525298688
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002370807254052319,
+      "loss": 1.1592,
+      "theoretical_loss": 3.394267623510492,
+      "tokens_seen": 2525429760
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002370406034344407,
+      "loss": 1.1754,
+      "theoretical_loss": 3.3942537179320658,
+      "tokens_seen": 2525560832
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002370004814636495,
+      "loss": 1.2667,
+      "theoretical_loss": 3.394239813277352,
+      "tokens_seen": 2525691904
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002369603594928583,
+      "loss": 1.2981,
+      "theoretical_loss": 3.3942259095462406,
+      "tokens_seen": 2525822976
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002369202375220671,
+      "loss": 1.3336,
+      "theoretical_loss": 3.3942120067386234,
+      "tokens_seen": 2525954048
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002368801155512759,
+      "loss": 1.3097,
+      "theoretical_loss": 3.3941981048543903,
+      "tokens_seen": 2526085120
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023683999358048468,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3941842038934324,
+      "tokens_seen": 2526216192
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023679987160969346,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3941703038556406,
+      "tokens_seen": 2526347264
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023675974963890226,
+      "loss": 1.274,
+      "theoretical_loss": 3.3941564047409054,
+      "tokens_seen": 2526478336
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023671962766811107,
+      "loss": 1.2324,
+      "theoretical_loss": 3.394142506549118,
+      "tokens_seen": 2526609408
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023667950569731985,
+      "loss": 1.2507,
+      "theoretical_loss": 3.394128609280169,
+      "tokens_seen": 2526740480
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023663938372652866,
+      "loss": 1.2851,
+      "theoretical_loss": 3.3941147129339493,
+      "tokens_seen": 2526871552
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023659926175573746,
+      "loss": 1.2451,
+      "theoretical_loss": 3.3941008175103495,
+      "tokens_seen": 2527002624
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023655913978494624,
+      "loss": 1.2204,
+      "theoretical_loss": 3.394086923009261,
+      "tokens_seen": 2527133696
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023651901781415502,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3940730294305745,
+      "tokens_seen": 2527264768
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023647889584336383,
+      "loss": 1.2562,
+      "theoretical_loss": 3.394059136774181,
+      "tokens_seen": 2527395840
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023643877387257263,
+      "loss": 1.2437,
+      "theoretical_loss": 3.3940452450399707,
+      "tokens_seen": 2527526912
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002363986519017814,
+      "loss": 1.2663,
+      "theoretical_loss": 3.394031354227836,
+      "tokens_seen": 2527657984
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023635852993099022,
+      "loss": 1.2177,
+      "theoretical_loss": 3.394017464337667,
+      "tokens_seen": 2527789056
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023631840796019903,
+      "loss": 1.2617,
+      "theoretical_loss": 3.3940035753693545,
+      "tokens_seen": 2527920128
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002362782859894078,
+      "loss": 1.1083,
+      "theoretical_loss": 3.3939896873227897,
+      "tokens_seen": 2528051200
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/docs_used": 848730,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1785964965820312,
+      "objective/train/theoretical_loss": 3.3939758001978637,
+      "objective/train/tokens_used": 898641376,
+      "theoretical_loss": 3.3939758001978637,
+      "tokens_seen": 2528182272
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023623816401861658,
+      "loss": 1.2155,
+      "theoretical_loss": 3.3939758001978637,
+      "tokens_seen": 2528182272
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002361980420478254,
+      "loss": 1.1591,
+      "theoretical_loss": 3.393961913994468,
+      "tokens_seen": 2528313344
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002361579200770342,
+      "loss": 1.25,
+      "theoretical_loss": 3.3939480287124932,
+      "tokens_seen": 2528444416
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023611779810624298,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3939341443518307,
+      "tokens_seen": 2528575488
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023607767613545178,
+      "loss": 1.2716,
+      "theoretical_loss": 3.3939202609123713,
+      "tokens_seen": 2528706560
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002360375541646606,
+      "loss": 1.2724,
+      "theoretical_loss": 3.393906378394006,
+      "tokens_seen": 2528837632
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023599743219386937,
+      "loss": 1.2335,
+      "theoretical_loss": 3.393892496796626,
+      "tokens_seen": 2528968704
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023595731022307817,
+      "loss": 1.2753,
+      "theoretical_loss": 3.3938786161201238,
+      "tokens_seen": 2529099776
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023591718825228695,
+      "loss": 1.3351,
+      "theoretical_loss": 3.393864736364389,
+      "tokens_seen": 2529230848
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023587706628149573,
+      "loss": 1.2805,
+      "theoretical_loss": 3.393850857529313,
+      "tokens_seen": 2529361920
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023583694431070454,
+      "loss": 1.259,
+      "theoretical_loss": 3.393836979614787,
+      "tokens_seen": 2529492992
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023579682233991334,
+      "loss": 1.2971,
+      "theoretical_loss": 3.3938231026207033,
+      "tokens_seen": 2529624064
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023575670036912215,
+      "loss": 1.2423,
+      "theoretical_loss": 3.393809226546952,
+      "tokens_seen": 2529755136
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023571657839833093,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3937953513934245,
+      "tokens_seen": 2529886208
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023567645642753974,
+      "loss": 1.2165,
+      "theoretical_loss": 3.393781477160013,
+      "tokens_seen": 2530017280
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023563633445674852,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3937676038466082,
+      "tokens_seen": 2530148352
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002355962124859573,
+      "loss": 1.2815,
+      "theoretical_loss": 3.3937537314531014,
+      "tokens_seen": 2530279424
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002355560905151661,
+      "loss": 1.2337,
+      "theoretical_loss": 3.3937398599793838,
+      "tokens_seen": 2530410496
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002355159685443749,
+      "loss": 1.2691,
+      "theoretical_loss": 3.3937259894253473,
+      "tokens_seen": 2530541568
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023547584657358371,
+      "loss": 1.3152,
+      "theoretical_loss": 3.393712119790883,
+      "tokens_seen": 2530672640
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002354357246027925,
+      "loss": 1.2972,
+      "theoretical_loss": 3.393698251075882,
+      "tokens_seen": 2530803712
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002353956026320013,
+      "loss": 1.2269,
+      "theoretical_loss": 3.3936843832802364,
+      "tokens_seen": 2530934784
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002353554806612101,
+      "loss": 1.3145,
+      "theoretical_loss": 3.3936705164038368,
+      "tokens_seen": 2531065856
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023531535869041886,
+      "loss": 1.2247,
+      "theoretical_loss": 3.3936566504465757,
+      "tokens_seen": 2531196928
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023527523671962766,
+      "loss": 1.3379,
+      "theoretical_loss": 3.3936427854083435,
+      "tokens_seen": 2531328000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023523511474883647,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3936289212890327,
+      "tokens_seen": 2531459072
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023519499277804528,
+      "loss": 1.1617,
+      "theoretical_loss": 3.3936150580885345,
+      "tokens_seen": 2531590144
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023515487080725406,
+      "loss": 1.2096,
+      "theoretical_loss": 3.39360119580674,
+      "tokens_seen": 2531721216
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023511474883646286,
+      "loss": 1.2695,
+      "theoretical_loss": 3.393587334443541,
+      "tokens_seen": 2531852288
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023507462686567167,
+      "loss": 1.3159,
+      "theoretical_loss": 3.393573473998829,
+      "tokens_seen": 2531983360
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023503450489488042,
+      "loss": 1.2066,
+      "theoretical_loss": 3.393559614472496,
+      "tokens_seen": 2532114432
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023499438292408923,
+      "loss": 1.1644,
+      "theoretical_loss": 3.3935457558644337,
+      "tokens_seen": 2532245504
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023495426095329803,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3935318981745333,
+      "tokens_seen": 2532376576
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002349141389825068,
+      "loss": 1.2168,
+      "theoretical_loss": 3.393518041402686,
+      "tokens_seen": 2532507648
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023487401701171562,
+      "loss": 1.2111,
+      "theoretical_loss": 3.3935041855487844,
+      "tokens_seen": 2532638720
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023483389504092443,
+      "loss": 1.2735,
+      "theoretical_loss": 3.3934903306127198,
+      "tokens_seen": 2532769792
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023479377307013323,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3934764765943837,
+      "tokens_seen": 2532900864
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023475365109934198,
+      "loss": 1.2018,
+      "theoretical_loss": 3.3934626234936682,
+      "tokens_seen": 2533031936
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002347135291285508,
+      "loss": 1.2768,
+      "theoretical_loss": 3.393448771310465,
+      "tokens_seen": 2533163008
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002346734071577596,
+      "loss": 1.2663,
+      "theoretical_loss": 3.393434920044666,
+      "tokens_seen": 2533294080
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023463328518696838,
+      "loss": 1.2224,
+      "theoretical_loss": 3.3934210696961618,
+      "tokens_seen": 2533425152
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023459316321617718,
+      "loss": 1.1886,
+      "theoretical_loss": 3.3934072202648458,
+      "tokens_seen": 2533556224
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.000234553041245386,
+      "loss": 1.2014,
+      "theoretical_loss": 3.393393371750609,
+      "tokens_seen": 2533687296
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002345129192745948,
+      "loss": 1.2807,
+      "theoretical_loss": 3.3933795241533433,
+      "tokens_seen": 2533818368
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023447279730380357,
+      "loss": 1.338,
+      "theoretical_loss": 3.393365677472941,
+      "tokens_seen": 2533949440
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023443267533301235,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3933518317092934,
+      "tokens_seen": 2534080512
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023439255336222116,
+      "loss": 1.2887,
+      "theoretical_loss": 3.3933379868622926,
+      "tokens_seen": 2534211584
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023435243139142994,
+      "loss": 1.2712,
+      "theoretical_loss": 3.3933241429318306,
+      "tokens_seen": 2534342656
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023431230942063874,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3933102999177986,
+      "tokens_seen": 2534473728
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023427218744984755,
+      "loss": 1.2912,
+      "theoretical_loss": 3.3932964578200897,
+      "tokens_seen": 2534604800
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 850982,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2052044868469238,
+      "objective/train/theoretical_loss": 3.3932826166385954,
+      "objective/train/tokens_used": 905194976,
+      "theoretical_loss": 3.3932826166385954,
+      "tokens_seen": 2534735872
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023423206547905636,
+      "loss": 1.192,
+      "theoretical_loss": 3.3932826166385954,
+      "tokens_seen": 2534735872
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023419194350826514,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3932687763732075,
+      "tokens_seen": 2534866944
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023415182153747392,
+      "loss": 1.3003,
+      "theoretical_loss": 3.3932549370238183,
+      "tokens_seen": 2534998016
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023411169956668272,
+      "loss": 1.224,
+      "theoretical_loss": 3.3932410985903196,
+      "tokens_seen": 2535129088
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002340715775958915,
+      "loss": 1.2071,
+      "theoretical_loss": 3.393227261072603,
+      "tokens_seen": 2535260160
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002340314556251003,
+      "loss": 1.2354,
+      "theoretical_loss": 3.3932134244705616,
+      "tokens_seen": 2535391232
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002339913336543091,
+      "loss": 1.2726,
+      "theoretical_loss": 3.393199588784087,
+      "tokens_seen": 2535522304
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002339512116835179,
+      "loss": 1.2105,
+      "theoretical_loss": 3.3931857540130705,
+      "tokens_seen": 2535653376
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002339110897127267,
+      "loss": 1.3787,
+      "theoretical_loss": 3.3931719201574055,
+      "tokens_seen": 2535784448
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023387096774193548,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3931580872169835,
+      "tokens_seen": 2535915520
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023383084577114428,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3931442551916966,
+      "tokens_seen": 2536046592
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023379072380035306,
+      "loss": 1.25,
+      "theoretical_loss": 3.393130424081437,
+      "tokens_seen": 2536177664
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023375060182956187,
+      "loss": 1.2524,
+      "theoretical_loss": 3.3931165938860968,
+      "tokens_seen": 2536308736
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023371047985877068,
+      "loss": 1.1774,
+      "theoretical_loss": 3.3931027646055685,
+      "tokens_seen": 2536439808
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023367035788797946,
+      "loss": 1.2482,
+      "theoretical_loss": 3.393088936239744,
+      "tokens_seen": 2536570880
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023363023591718826,
+      "loss": 1.2938,
+      "theoretical_loss": 3.393075108788516,
+      "tokens_seen": 2536701952
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023359011394639707,
+      "loss": 1.257,
+      "theoretical_loss": 3.3930612822517765,
+      "tokens_seen": 2536833024
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023354999197560585,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3930474566294175,
+      "tokens_seen": 2536964096
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023350987000481463,
+      "loss": 1.244,
+      "theoretical_loss": 3.3930336319213312,
+      "tokens_seen": 2537095168
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023346974803402343,
+      "loss": 1.2651,
+      "theoretical_loss": 3.393019808127411,
+      "tokens_seen": 2537226240
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023342962606323224,
+      "loss": 1.2056,
+      "theoretical_loss": 3.393005985247548,
+      "tokens_seen": 2537357312
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023338950409244102,
+      "loss": 1.2646,
+      "theoretical_loss": 3.392992163281635,
+      "tokens_seen": 2537488384
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023334938212164982,
+      "loss": 1.2367,
+      "theoretical_loss": 3.3929783422295645,
+      "tokens_seen": 2537619456
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023330926015085863,
+      "loss": 1.3,
+      "theoretical_loss": 3.392964522091229,
+      "tokens_seen": 2537750528
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002332691381800674,
+      "loss": 1.2156,
+      "theoretical_loss": 3.39295070286652,
+      "tokens_seen": 2537881600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002332290162092762,
+      "loss": 1.2593,
+      "theoretical_loss": 3.392936884555331,
+      "tokens_seen": 2538012672
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.000233188894238485,
+      "loss": 1.1966,
+      "theoretical_loss": 3.392923067157554,
+      "tokens_seen": 2538143744
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002331487722676938,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3929092506730814,
+      "tokens_seen": 2538274816
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023310865029690258,
+      "loss": 1.2544,
+      "theoretical_loss": 3.392895435101806,
+      "tokens_seen": 2538405888
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002330685283261114,
+      "loss": 1.256,
+      "theoretical_loss": 3.39288162044362,
+      "tokens_seen": 2538536960
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002330284063553202,
+      "loss": 1.2711,
+      "theoretical_loss": 3.3928678066984155,
+      "tokens_seen": 2538668032
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.000232988284384529,
+      "loss": 1.2652,
+      "theoretical_loss": 3.392853993866086,
+      "tokens_seen": 2538799104
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023294816241373775,
+      "loss": 1.201,
+      "theoretical_loss": 3.3928401819465233,
+      "tokens_seen": 2538930176
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023290804044294656,
+      "loss": 1.2877,
+      "theoretical_loss": 3.3928263709396207,
+      "tokens_seen": 2539061248
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023286791847215536,
+      "loss": 1.192,
+      "theoretical_loss": 3.3928125608452695,
+      "tokens_seen": 2539192320
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023282779650136414,
+      "loss": 1.2821,
+      "theoretical_loss": 3.3927987516633635,
+      "tokens_seen": 2539323392
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023278767453057295,
+      "loss": 1.188,
+      "theoretical_loss": 3.392784943393795,
+      "tokens_seen": 2539454464
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023274755255978176,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3927711360364565,
+      "tokens_seen": 2539585536
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023270743058899054,
+      "loss": 1.2281,
+      "theoretical_loss": 3.3927573295912405,
+      "tokens_seen": 2539716608
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023266730861819932,
+      "loss": 1.3254,
+      "theoretical_loss": 3.39274352405804,
+      "tokens_seen": 2539847680
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023262718664740812,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3927297194367476,
+      "tokens_seen": 2539978752
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023258706467661693,
+      "loss": 1.3155,
+      "theoretical_loss": 3.392715915727256,
+      "tokens_seen": 2540109824
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002325469427058257,
+      "loss": 1.2493,
+      "theoretical_loss": 3.392702112929458,
+      "tokens_seen": 2540240896
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002325068207350345,
+      "loss": 1.2366,
+      "theoretical_loss": 3.3926883110432464,
+      "tokens_seen": 2540371968
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023246669876424332,
+      "loss": 1.2771,
+      "theoretical_loss": 3.3926745100685136,
+      "tokens_seen": 2540503040
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002324265767934521,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3926607100051527,
+      "tokens_seen": 2540634112
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023238645482266088,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3926469108530566,
+      "tokens_seen": 2540765184
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023234633285186968,
+      "loss": 1.249,
+      "theoretical_loss": 3.3926331126121174,
+      "tokens_seen": 2540896256
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002323062108810785,
+      "loss": 1.2033,
+      "theoretical_loss": 3.392619315282229,
+      "tokens_seen": 2541027328
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023226608891028727,
+      "loss": 1.2268,
+      "theoretical_loss": 3.3926055188632835,
+      "tokens_seen": 2541158400
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 852916,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2012100219726562,
+      "objective/train/theoretical_loss": 3.3925917233551743,
+      "objective/train/tokens_used": 911748576,
+      "theoretical_loss": 3.3925917233551743,
+      "tokens_seen": 2541289472
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023222596693949608,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3925917233551743,
+      "tokens_seen": 2541289472
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023218584496870488,
+      "loss": 1.2257,
+      "theoretical_loss": 3.392577928757794,
+      "tokens_seen": 2541420544
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023214572299791366,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3925641350710354,
+      "tokens_seen": 2541551616
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023210560102712247,
+      "loss": 1.2501,
+      "theoretical_loss": 3.392550342294791,
+      "tokens_seen": 2541682688
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023206547905633125,
+      "loss": 1.3119,
+      "theoretical_loss": 3.392536550428955,
+      "tokens_seen": 2541813760
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023202535708554003,
+      "loss": 1.2371,
+      "theoretical_loss": 3.3925227594734197,
+      "tokens_seen": 2541944832
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023198523511474883,
+      "loss": 1.1688,
+      "theoretical_loss": 3.3925089694280777,
+      "tokens_seen": 2542075904
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023194511314395764,
+      "loss": 1.2826,
+      "theoretical_loss": 3.3924951802928223,
+      "tokens_seen": 2542206976
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023190499117316645,
+      "loss": 1.2372,
+      "theoretical_loss": 3.392481392067547,
+      "tokens_seen": 2542338048
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023186486920237522,
+      "loss": 1.2609,
+      "theoretical_loss": 3.3924676047521443,
+      "tokens_seen": 2542469120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023182474723158403,
+      "loss": 1.1944,
+      "theoretical_loss": 3.392453818346507,
+      "tokens_seen": 2542600192
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002317846252607928,
+      "loss": 1.2574,
+      "theoretical_loss": 3.3924400328505286,
+      "tokens_seen": 2542731264
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002317445032900016,
+      "loss": 1.2445,
+      "theoretical_loss": 3.3924262482641026,
+      "tokens_seen": 2542862336
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002317043813192104,
+      "loss": 1.2452,
+      "theoretical_loss": 3.392412464587121,
+      "tokens_seen": 2542993408
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002316642593484192,
+      "loss": 1.3113,
+      "theoretical_loss": 3.3923986818194782,
+      "tokens_seen": 2543124480
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.000231624137377628,
+      "loss": 1.277,
+      "theoretical_loss": 3.3923848999610664,
+      "tokens_seen": 2543255552
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002315840154068368,
+      "loss": 1.3055,
+      "theoretical_loss": 3.3923711190117793,
+      "tokens_seen": 2543386624
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002315438934360456,
+      "loss": 1.2439,
+      "theoretical_loss": 3.3923573389715096,
+      "tokens_seen": 2543517696
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023150377146525437,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3923435598401506,
+      "tokens_seen": 2543648768
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023146364949446315,
+      "loss": 1.308,
+      "theoretical_loss": 3.392329781617596,
+      "tokens_seen": 2543779840
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023142352752367196,
+      "loss": 1.2357,
+      "theoretical_loss": 3.392316004303739,
+      "tokens_seen": 2543910912
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023138340555288076,
+      "loss": 1.2373,
+      "theoretical_loss": 3.392302227898472,
+      "tokens_seen": 2544041984
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023134328358208957,
+      "loss": 1.2661,
+      "theoretical_loss": 3.392288452401689,
+      "tokens_seen": 2544173056
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023130316161129835,
+      "loss": 1.3284,
+      "theoretical_loss": 3.3922746778132833,
+      "tokens_seen": 2544304128
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023126303964050716,
+      "loss": 1.2026,
+      "theoretical_loss": 3.392260904133148,
+      "tokens_seen": 2544435200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023122291766971596,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3922471313611764,
+      "tokens_seen": 2544566272
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023118279569892471,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3922333594972622,
+      "tokens_seen": 2544697344
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023114267372813352,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3922195885412982,
+      "tokens_seen": 2544828416
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023110255175734233,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3922058184931783,
+      "tokens_seen": 2544959488
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023106242978655113,
+      "loss": 1.1621,
+      "theoretical_loss": 3.3921920493527953,
+      "tokens_seen": 2545090560
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002310223078157599,
+      "loss": 1.1831,
+      "theoretical_loss": 3.392178281120043,
+      "tokens_seen": 2545221632
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023098218584496872,
+      "loss": 1.1617,
+      "theoretical_loss": 3.3921645137948153,
+      "tokens_seen": 2545352704
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023094206387417753,
+      "loss": 1.3727,
+      "theoretical_loss": 3.392150747377005,
+      "tokens_seen": 2545483776
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023090194190338628,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3921369818665057,
+      "tokens_seen": 2545614848
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023086181993259508,
+      "loss": 1.2582,
+      "theoretical_loss": 3.3921232172632108,
+      "tokens_seen": 2545745920
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002308216979618039,
+      "loss": 1.2774,
+      "theoretical_loss": 3.392109453567014,
+      "tokens_seen": 2545876992
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023078157599101267,
+      "loss": 1.2844,
+      "theoretical_loss": 3.3920956907778086,
+      "tokens_seen": 2546008064
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023074145402022148,
+      "loss": 1.1823,
+      "theoretical_loss": 3.3920819288954887,
+      "tokens_seen": 2546139136
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023070133204943028,
+      "loss": 1.2781,
+      "theoretical_loss": 3.3920681679199474,
+      "tokens_seen": 2546270208
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002306612100786391,
+      "loss": 1.2294,
+      "theoretical_loss": 3.392054407851078,
+      "tokens_seen": 2546401280
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023062108810784784,
+      "loss": 1.2806,
+      "theoretical_loss": 3.392040648688775,
+      "tokens_seen": 2546532352
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023058096613705665,
+      "loss": 1.2827,
+      "theoretical_loss": 3.392026890432931,
+      "tokens_seen": 2546663424
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023054084416626545,
+      "loss": 1.274,
+      "theoretical_loss": 3.39201313308344,
+      "tokens_seen": 2546794496
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023050072219547423,
+      "loss": 1.2079,
+      "theoretical_loss": 3.391999376640196,
+      "tokens_seen": 2546925568
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023046060022468304,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3919856211030925,
+      "tokens_seen": 2547056640
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023042047825389184,
+      "loss": 1.2554,
+      "theoretical_loss": 3.391971866472023,
+      "tokens_seen": 2547187712
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023038035628310065,
+      "loss": 1.2233,
+      "theoretical_loss": 3.391958112746881,
+      "tokens_seen": 2547318784
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023034023431230943,
+      "loss": 1.2347,
+      "theoretical_loss": 3.391944359927561,
+      "tokens_seen": 2547449856
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002303001123415182,
+      "loss": 1.2414,
+      "theoretical_loss": 3.391930608013956,
+      "tokens_seen": 2547580928
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023025999037072702,
+      "loss": 1.2583,
+      "theoretical_loss": 3.39191685700596,
+      "tokens_seen": 2547712000
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 855143,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2365459203720093,
+      "objective/train/theoretical_loss": 3.391903106903467,
+      "objective/train/tokens_used": 918302176,
+      "theoretical_loss": 3.391903106903467,
+      "tokens_seen": 2547843072
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002302198683999358,
+      "loss": 1.2023,
+      "theoretical_loss": 3.391903106903467,
+      "tokens_seen": 2547843072
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002301797464291446,
+      "loss": 1.2654,
+      "theoretical_loss": 3.3918893577063702,
+      "tokens_seen": 2547974144
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002301396244583534,
+      "loss": 1.236,
+      "theoretical_loss": 3.3918756094145643,
+      "tokens_seen": 2548105216
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023009950248756221,
+      "loss": 1.2525,
+      "theoretical_loss": 3.3918618620279424,
+      "tokens_seen": 2548236288
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.000230059380516771,
+      "loss": 1.324,
+      "theoretical_loss": 3.3918481155463986,
+      "tokens_seen": 2548367360
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023001925854597977,
+      "loss": 1.2023,
+      "theoretical_loss": 3.391834369969827,
+      "tokens_seen": 2548498432
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00022997913657518858,
+      "loss": 1.1845,
+      "theoretical_loss": 3.391820625298121,
+      "tokens_seen": 2548629504
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00022993901460439736,
+      "loss": 1.1919,
+      "theoretical_loss": 3.3918068815311746,
+      "tokens_seen": 2548760576
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00022989889263360616,
+      "loss": 1.1833,
+      "theoretical_loss": 3.391793138668882,
+      "tokens_seen": 2548891648
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00022985877066281497,
+      "loss": 1.2682,
+      "theoretical_loss": 3.391779396711137,
+      "tokens_seen": 2549022720
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00022981864869202375,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3917656556578337,
+      "tokens_seen": 2549153792
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022977852672123256,
+      "loss": 1.2699,
+      "theoretical_loss": 3.391751915508866,
+      "tokens_seen": 2549284864
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022973840475044136,
+      "loss": 1.2662,
+      "theoretical_loss": 3.3917381762641274,
+      "tokens_seen": 2549415936
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022969828277965014,
+      "loss": 1.2724,
+      "theoretical_loss": 3.3917244379235125,
+      "tokens_seen": 2549547008
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022965816080885892,
+      "loss": 1.2179,
+      "theoretical_loss": 3.391710700486915,
+      "tokens_seen": 2549678080
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022961803883806773,
+      "loss": 1.3503,
+      "theoretical_loss": 3.3916969639542294,
+      "tokens_seen": 2549809152
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022957791686727653,
+      "loss": 1.271,
+      "theoretical_loss": 3.3916832283253493,
+      "tokens_seen": 2549940224
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002295377948964853,
+      "loss": 1.2978,
+      "theoretical_loss": 3.391669493600169,
+      "tokens_seen": 2550071296
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022949767292569412,
+      "loss": 1.2802,
+      "theoretical_loss": 3.3916557597785824,
+      "tokens_seen": 2550202368
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022945755095490293,
+      "loss": 1.3334,
+      "theoretical_loss": 3.391642026860484,
+      "tokens_seen": 2550333440
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002294174289841117,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3916282948457677,
+      "tokens_seen": 2550464512
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022937730701332048,
+      "loss": 1.2474,
+      "theoretical_loss": 3.3916145637343273,
+      "tokens_seen": 2550595584
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002293371850425293,
+      "loss": 1.27,
+      "theoretical_loss": 3.3916008335260575,
+      "tokens_seen": 2550726656
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002292970630717381,
+      "loss": 1.1912,
+      "theoretical_loss": 3.3915871042208523,
+      "tokens_seen": 2550857728
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022925694110094688,
+      "loss": 1.3357,
+      "theoretical_loss": 3.3915733758186057,
+      "tokens_seen": 2550988800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022921681913015568,
+      "loss": 1.3201,
+      "theoretical_loss": 3.3915596483192125,
+      "tokens_seen": 2551119872
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002291766971593645,
+      "loss": 1.2484,
+      "theoretical_loss": 3.3915459217225665,
+      "tokens_seen": 2551250944
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022913657518857327,
+      "loss": 1.2553,
+      "theoretical_loss": 3.391532196028562,
+      "tokens_seen": 2551382016
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022909645321778205,
+      "loss": 1.3131,
+      "theoretical_loss": 3.3915184712370934,
+      "tokens_seen": 2551513088
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022905633124699085,
+      "loss": 1.3118,
+      "theoretical_loss": 3.3915047473480544,
+      "tokens_seen": 2551644160
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022901620927619966,
+      "loss": 1.2278,
+      "theoretical_loss": 3.3914910243613403,
+      "tokens_seen": 2551775232
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022897608730540844,
+      "loss": 1.2015,
+      "theoretical_loss": 3.3914773022768445,
+      "tokens_seen": 2551906304
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022893596533461724,
+      "loss": 1.229,
+      "theoretical_loss": 3.391463581094462,
+      "tokens_seen": 2552037376
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022889584336382605,
+      "loss": 1.1817,
+      "theoretical_loss": 3.3914498608140873,
+      "tokens_seen": 2552168448
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022885572139303486,
+      "loss": 1.2085,
+      "theoretical_loss": 3.3914361414356144,
+      "tokens_seen": 2552299520
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002288155994222436,
+      "loss": 1.2632,
+      "theoretical_loss": 3.3914224229589376,
+      "tokens_seen": 2552430592
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022877547745145242,
+      "loss": 1.1541,
+      "theoretical_loss": 3.3914087053839514,
+      "tokens_seen": 2552561664
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022873535548066122,
+      "loss": 1.2955,
+      "theoretical_loss": 3.39139498871055,
+      "tokens_seen": 2552692736
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022869523350987,
+      "loss": 1.2812,
+      "theoretical_loss": 3.3913812729386286,
+      "tokens_seen": 2552823808
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002286551115390788,
+      "loss": 1.2366,
+      "theoretical_loss": 3.3913675580680813,
+      "tokens_seen": 2552954880
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022861498956828761,
+      "loss": 1.1885,
+      "theoretical_loss": 3.391353844098802,
+      "tokens_seen": 2553085952
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002285748675974964,
+      "loss": 1.2978,
+      "theoretical_loss": 3.391340131030686,
+      "tokens_seen": 2553217024
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022853474562670517,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3913264188636276,
+      "tokens_seen": 2553348096
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022849462365591398,
+      "loss": 1.2145,
+      "theoretical_loss": 3.391312707597521,
+      "tokens_seen": 2553479168
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022845450168512278,
+      "loss": 1.2519,
+      "theoretical_loss": 3.3912989972322616,
+      "tokens_seen": 2553610240
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022841437971433156,
+      "loss": 1.2318,
+      "theoretical_loss": 3.391285287767743,
+      "tokens_seen": 2553741312
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022837425774354037,
+      "loss": 1.2471,
+      "theoretical_loss": 3.39127157920386,
+      "tokens_seen": 2553872384
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022833413577274918,
+      "loss": 1.2234,
+      "theoretical_loss": 3.391257871540508,
+      "tokens_seen": 2554003456
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022829401380195796,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3912441647775813,
+      "tokens_seen": 2554134528
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022825389183116673,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3912304589149738,
+      "tokens_seen": 2554265600
+    },
+    {
+      "epoch": 0.55,
+      "objective/train/docs_used": 857322,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3009066581726074,
+      "objective/train/theoretical_loss": 3.3912167539525804,
+      "objective/train/tokens_used": 924855776,
+      "theoretical_loss": 3.3912167539525804,
+      "tokens_seen": 2554396672
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022821376986037554,
+      "loss": 1.2252,
+      "theoretical_loss": 3.3912167539525804,
+      "tokens_seen": 2554396672
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022817364788958435,
+      "loss": 1.2874,
+      "theoretical_loss": 3.3912030498902968,
+      "tokens_seen": 2554527744
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022813352591879313,
+      "loss": 1.2216,
+      "theoretical_loss": 3.3911893467280168,
+      "tokens_seen": 2554658816
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022809340394800193,
+      "loss": 1.2217,
+      "theoretical_loss": 3.391175644465635,
+      "tokens_seen": 2554789888
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022805328197721074,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3911619431030466,
+      "tokens_seen": 2554920960
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022801316000641952,
+      "loss": 1.1751,
+      "theoretical_loss": 3.391148242640146,
+      "tokens_seen": 2555052032
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022797303803562832,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3911345430768285,
+      "tokens_seen": 2555183104
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002279329160648371,
+      "loss": 1.1831,
+      "theoretical_loss": 3.3911208444129883,
+      "tokens_seen": 2555314176
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022789279409404588,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3911071466485208,
+      "tokens_seen": 2555445248
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002278526721232547,
+      "loss": 1.2498,
+      "theoretical_loss": 3.39109344978332,
+      "tokens_seen": 2555576320
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002278125501524635,
+      "loss": 1.2077,
+      "theoretical_loss": 3.391079753817282,
+      "tokens_seen": 2555707392
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002277724281816723,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3910660587503,
+      "tokens_seen": 2555838464
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022773230621088108,
+      "loss": 1.201,
+      "theoretical_loss": 3.3910523645822703,
+      "tokens_seen": 2555969536
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002276921842400899,
+      "loss": 1.165,
+      "theoretical_loss": 3.3910386713130873,
+      "tokens_seen": 2556100608
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022765206226929867,
+      "loss": 1.2583,
+      "theoretical_loss": 3.391024978942646,
+      "tokens_seen": 2556231680
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022761194029850745,
+      "loss": 1.2548,
+      "theoretical_loss": 3.391011287470841,
+      "tokens_seen": 2556362752
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022757181832771625,
+      "loss": 1.1912,
+      "theoretical_loss": 3.390997596897568,
+      "tokens_seen": 2556493824
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022753169635692506,
+      "loss": 1.2332,
+      "theoretical_loss": 3.390983907222721,
+      "tokens_seen": 2556624896
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022749157438613386,
+      "loss": 1.2436,
+      "theoretical_loss": 3.390970218446195,
+      "tokens_seen": 2556755968
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022745145241534264,
+      "loss": 1.1848,
+      "theoretical_loss": 3.390956530567886,
+      "tokens_seen": 2556887040
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022741133044455145,
+      "loss": 1.2569,
+      "theoretical_loss": 3.3909428435876885,
+      "tokens_seen": 2557018112
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022737120847376026,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3909291575054974,
+      "tokens_seen": 2557149184
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.000227331086502969,
+      "loss": 1.1497,
+      "theoretical_loss": 3.3909154723212076,
+      "tokens_seen": 2557280256
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022729096453217782,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3909017880347148,
+      "tokens_seen": 2557411328
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022725084256138662,
+      "loss": 1.227,
+      "theoretical_loss": 3.3908881046459136,
+      "tokens_seen": 2557542400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022721072059059543,
+      "loss": 1.215,
+      "theoretical_loss": 3.3908744221546994,
+      "tokens_seen": 2557673472
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002271705986198042,
+      "loss": 1.184,
+      "theoretical_loss": 3.390860740560967,
+      "tokens_seen": 2557804544
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.000227130476649013,
+      "loss": 1.2211,
+      "theoretical_loss": 3.390847059864612,
+      "tokens_seen": 2557935616
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022709035467822182,
+      "loss": 1.2259,
+      "theoretical_loss": 3.3908333800655286,
+      "tokens_seen": 2558066688
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022705023270743057,
+      "loss": 1.187,
+      "theoretical_loss": 3.3908197011636134,
+      "tokens_seen": 2558197760
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022701011073663938,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3908060231587602,
+      "tokens_seen": 2558328832
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022696998876584818,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3907923460508655,
+      "tokens_seen": 2558459904
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.000226929866795057,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3907786698398237,
+      "tokens_seen": 2558590976
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022688974482426577,
+      "loss": 1.3082,
+      "theoretical_loss": 3.3907649945255303,
+      "tokens_seen": 2558722048
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022684962285347458,
+      "loss": 1.1661,
+      "theoretical_loss": 3.39075132010788,
+      "tokens_seen": 2558853120
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022680950088268338,
+      "loss": 1.212,
+      "theoretical_loss": 3.390737646586769,
+      "tokens_seen": 2558984192
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022676937891189213,
+      "loss": 1.2813,
+      "theoretical_loss": 3.3907239739620922,
+      "tokens_seen": 2559115264
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022672925694110094,
+      "loss": 1.2887,
+      "theoretical_loss": 3.390710302233745,
+      "tokens_seen": 2559246336
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022668913497030975,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3906966314016227,
+      "tokens_seen": 2559377408
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022664901299951853,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3906829614656204,
+      "tokens_seen": 2559508480
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022660889102872733,
+      "loss": 1.2998,
+      "theoretical_loss": 3.390669292425634,
+      "tokens_seen": 2559639552
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022656876905793614,
+      "loss": 1.1838,
+      "theoretical_loss": 3.390655624281558,
+      "tokens_seen": 2559770624
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022652864708714495,
+      "loss": 1.2041,
+      "theoretical_loss": 3.3906419570332886,
+      "tokens_seen": 2559901696
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022648852511635372,
+      "loss": 1.1561,
+      "theoretical_loss": 3.3906282906807217,
+      "tokens_seen": 2560032768
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002264484031455625,
+      "loss": 1.1934,
+      "theoretical_loss": 3.3906146252237512,
+      "tokens_seen": 2560163840
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002264082811747713,
+      "loss": 1.2726,
+      "theoretical_loss": 3.3906009606622742,
+      "tokens_seen": 2560294912
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002263681592039801,
+      "loss": 1.1744,
+      "theoretical_loss": 3.390587296996185,
+      "tokens_seen": 2560425984
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002263280372331889,
+      "loss": 1.1888,
+      "theoretical_loss": 3.3905736342253796,
+      "tokens_seen": 2560557056
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002262879152623977,
+      "loss": 1.313,
+      "theoretical_loss": 3.3905599723497533,
+      "tokens_seen": 2560688128
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002262477932916065,
+      "loss": 1.1205,
+      "theoretical_loss": 3.390546311369202,
+      "tokens_seen": 2560819200
+    },
+    {
+      "epoch": 0.55,
+      "objective/train/docs_used": 859666,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2500356435775757,
+      "objective/train/theoretical_loss": 3.3905326512836207,
+      "objective/train/tokens_used": 931409376,
+      "theoretical_loss": 3.3905326512836207,
+      "tokens_seen": 2560950272
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002262076713208153,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3905326512836207,
+      "tokens_seen": 2560950272
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022616754935002407,
+      "loss": 1.2993,
+      "theoretical_loss": 3.3905189920929053,
+      "tokens_seen": 2561081344
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022612742737923287,
+      "loss": 1.257,
+      "theoretical_loss": 3.3905053337969515,
+      "tokens_seen": 2561212416
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022608730540844165,
+      "loss": 1.2124,
+      "theoretical_loss": 3.390491676395655,
+      "tokens_seen": 2561343488
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022604718343765046,
+      "loss": 1.2638,
+      "theoretical_loss": 3.390478019888911,
+      "tokens_seen": 2561474560
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022600706146685926,
+      "loss": 1.2198,
+      "theoretical_loss": 3.390464364276615,
+      "tokens_seen": 2561605632
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022596693949606807,
+      "loss": 1.2518,
+      "theoretical_loss": 3.390450709558663,
+      "tokens_seen": 2561736704
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022592681752527685,
+      "loss": 1.251,
+      "theoretical_loss": 3.3904370557349512,
+      "tokens_seen": 2561867776
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022588669555448563,
+      "loss": 1.225,
+      "theoretical_loss": 3.390423402805374,
+      "tokens_seen": 2561998848
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022584657358369444,
+      "loss": 1.2367,
+      "theoretical_loss": 3.3904097507698285,
+      "tokens_seen": 2562129920
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022580645161290321,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3903960996282096,
+      "tokens_seen": 2562260992
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022576632964211202,
+      "loss": 1.2552,
+      "theoretical_loss": 3.3903824493804127,
+      "tokens_seen": 2562392064
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022572620767132083,
+      "loss": 1.2953,
+      "theoretical_loss": 3.3903688000263346,
+      "tokens_seen": 2562523136
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002256860857005296,
+      "loss": 1.2554,
+      "theoretical_loss": 3.3903551515658705,
+      "tokens_seen": 2562654208
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002256459637297384,
+      "loss": 1.2877,
+      "theoretical_loss": 3.3903415039989158,
+      "tokens_seen": 2562785280
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022560584175894722,
+      "loss": 1.2285,
+      "theoretical_loss": 3.3903278573253672,
+      "tokens_seen": 2562916352
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.000225565719788156,
+      "loss": 1.2347,
+      "theoretical_loss": 3.39031421154512,
+      "tokens_seen": 2563047424
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022552559781736478,
+      "loss": 1.2068,
+      "theoretical_loss": 3.39030056665807,
+      "tokens_seen": 2563178496
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022548547584657358,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3902869226641132,
+      "tokens_seen": 2563309568
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002254453538757824,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3902732795631456,
+      "tokens_seen": 2563440640
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022540523190499117,
+      "loss": 1.2349,
+      "theoretical_loss": 3.390259637355063,
+      "tokens_seen": 2563571712
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022536510993419998,
+      "loss": 1.2499,
+      "theoretical_loss": 3.390245996039761,
+      "tokens_seen": 2563702784
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022532498796340878,
+      "loss": 1.2092,
+      "theoretical_loss": 3.390232355617136,
+      "tokens_seen": 2563833856
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022528486599261756,
+      "loss": 1.1993,
+      "theoretical_loss": 3.390218716087084,
+      "tokens_seen": 2563964928
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022524474402182634,
+      "loss": 1.1806,
+      "theoretical_loss": 3.3902050774495005,
+      "tokens_seen": 2564096000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022520462205103515,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3901914397042816,
+      "tokens_seen": 2564227072
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022516450008024395,
+      "loss": 1.3055,
+      "theoretical_loss": 3.3901778028513236,
+      "tokens_seen": 2564358144
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022512437810945273,
+      "loss": 1.3272,
+      "theoretical_loss": 3.3901641668905222,
+      "tokens_seen": 2564489216
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022508425613866154,
+      "loss": 1.2071,
+      "theoretical_loss": 3.390150531821774,
+      "tokens_seen": 2564620288
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022504413416787034,
+      "loss": 1.2474,
+      "theoretical_loss": 3.3901368976449744,
+      "tokens_seen": 2564751360
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022500401219707915,
+      "loss": 1.2789,
+      "theoretical_loss": 3.3901232643600197,
+      "tokens_seen": 2564882432
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002249638902262879,
+      "loss": 1.2127,
+      "theoretical_loss": 3.390109631966806,
+      "tokens_seen": 2565013504
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002249237682554967,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3900960004652294,
+      "tokens_seen": 2565144576
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022488364628470552,
+      "loss": 1.2894,
+      "theoretical_loss": 3.390082369855186,
+      "tokens_seen": 2565275648
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002248435243139143,
+      "loss": 1.2559,
+      "theoretical_loss": 3.390068740136572,
+      "tokens_seen": 2565406720
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002248034023431231,
+      "loss": 1.2637,
+      "theoretical_loss": 3.390055111309284,
+      "tokens_seen": 2565537792
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002247632803723319,
+      "loss": 1.2988,
+      "theoretical_loss": 3.390041483373217,
+      "tokens_seen": 2565668864
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022472315840154071,
+      "loss": 1.1976,
+      "theoretical_loss": 3.390027856328268,
+      "tokens_seen": 2565799936
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022468303643074947,
+      "loss": 1.1918,
+      "theoretical_loss": 3.3900142301743337,
+      "tokens_seen": 2565931008
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022464291445995827,
+      "loss": 1.2382,
+      "theoretical_loss": 3.3900006049113096,
+      "tokens_seen": 2566062080
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022460279248916708,
+      "loss": 1.2362,
+      "theoretical_loss": 3.389986980539092,
+      "tokens_seen": 2566193152
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022456267051837586,
+      "loss": 1.2081,
+      "theoretical_loss": 3.389973357057577,
+      "tokens_seen": 2566324224
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022452254854758466,
+      "loss": 1.2207,
+      "theoretical_loss": 3.389959734466662,
+      "tokens_seen": 2566455296
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022448242657679347,
+      "loss": 1.1515,
+      "theoretical_loss": 3.389946112766242,
+      "tokens_seen": 2566586368
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022444230460600225,
+      "loss": 1.2216,
+      "theoretical_loss": 3.3899324919562135,
+      "tokens_seen": 2566717440
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022440218263521103,
+      "loss": 1.2466,
+      "theoretical_loss": 3.389918872036473,
+      "tokens_seen": 2566848512
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022436206066441984,
+      "loss": 1.2243,
+      "theoretical_loss": 3.3899052530069174,
+      "tokens_seen": 2566979584
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022432193869362864,
+      "loss": 1.2629,
+      "theoretical_loss": 3.3898916348674426,
+      "tokens_seen": 2567110656
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022428181672283742,
+      "loss": 1.2086,
+      "theoretical_loss": 3.389878017617945,
+      "tokens_seen": 2567241728
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022424169475204623,
+      "loss": 1.2483,
+      "theoretical_loss": 3.389864401258321,
+      "tokens_seen": 2567372800
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 861842,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3484442234039307,
+      "objective/train/theoretical_loss": 3.389850785788467,
+      "objective/train/tokens_used": 937962976,
+      "theoretical_loss": 3.389850785788467,
+      "tokens_seen": 2567503872
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022420157278125503,
+      "loss": 1.2993,
+      "theoretical_loss": 3.389850785788467,
+      "tokens_seen": 2567503872
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002241614508104638,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3898371712082795,
+      "tokens_seen": 2567634944
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022412132883967262,
+      "loss": 1.2579,
+      "theoretical_loss": 3.389823557517655,
+      "tokens_seen": 2567766016
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002240812068688814,
+      "loss": 1.2215,
+      "theoretical_loss": 3.38980994471649,
+      "tokens_seen": 2567897088
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002240410848980902,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3897963328046807,
+      "tokens_seen": 2568028160
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022400096292729898,
+      "loss": 1.2506,
+      "theoretical_loss": 3.389782721782124,
+      "tokens_seen": 2568159232
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002239608409565078,
+      "loss": 1.1893,
+      "theoretical_loss": 3.3897691116487163,
+      "tokens_seen": 2568290304
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002239207189857166,
+      "loss": 1.2652,
+      "theoretical_loss": 3.3897555024043537,
+      "tokens_seen": 2568421376
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022388059701492538,
+      "loss": 1.2501,
+      "theoretical_loss": 3.3897418940489334,
+      "tokens_seen": 2568552448
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022384047504413418,
+      "loss": 1.285,
+      "theoretical_loss": 3.389728286582352,
+      "tokens_seen": 2568683520
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022380035307334296,
+      "loss": 1.2473,
+      "theoretical_loss": 3.3897146800045053,
+      "tokens_seen": 2568814592
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022376023110255174,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3897010743152904,
+      "tokens_seen": 2568945664
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022372010913176055,
+      "loss": 1.2672,
+      "theoretical_loss": 3.3896874695146044,
+      "tokens_seen": 2569076736
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022367998716096935,
+      "loss": 1.2131,
+      "theoretical_loss": 3.3896738656023433,
+      "tokens_seen": 2569207808
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022363986519017816,
+      "loss": 1.2895,
+      "theoretical_loss": 3.389660262578404,
+      "tokens_seen": 2569338880
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022359974321938694,
+      "loss": 1.2211,
+      "theoretical_loss": 3.389646660442683,
+      "tokens_seen": 2569469952
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022355962124859574,
+      "loss": 1.3096,
+      "theoretical_loss": 3.3896330591950776,
+      "tokens_seen": 2569601024
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022351949927780452,
+      "loss": 1.2499,
+      "theoretical_loss": 3.389619458835484,
+      "tokens_seen": 2569732096
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002234793773070133,
+      "loss": 1.2721,
+      "theoretical_loss": 3.3896058593637983,
+      "tokens_seen": 2569863168
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002234392553362221,
+      "loss": 1.2678,
+      "theoretical_loss": 3.389592260779919,
+      "tokens_seen": 2569994240
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022339913336543092,
+      "loss": 1.2495,
+      "theoretical_loss": 3.389578663083741,
+      "tokens_seen": 2570125312
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022335901139463972,
+      "loss": 1.2132,
+      "theoretical_loss": 3.389565066275162,
+      "tokens_seen": 2570256384
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002233188894238485,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3895514703540788,
+      "tokens_seen": 2570387456
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002232787674530573,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3895378753203884,
+      "tokens_seen": 2570518528
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022323864548226611,
+      "loss": 1.3707,
+      "theoretical_loss": 3.389524281173987,
+      "tokens_seen": 2570649600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022319852351147487,
+      "loss": 1.2645,
+      "theoretical_loss": 3.389510687914772,
+      "tokens_seen": 2570780672
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022315840154068367,
+      "loss": 1.2724,
+      "theoretical_loss": 3.38949709554264,
+      "tokens_seen": 2570911744
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022311827956989248,
+      "loss": 1.151,
+      "theoretical_loss": 3.389483504057488,
+      "tokens_seen": 2571042816
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022307815759910128,
+      "loss": 1.2811,
+      "theoretical_loss": 3.389469913459213,
+      "tokens_seen": 2571173888
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022303803562831006,
+      "loss": 1.2698,
+      "theoretical_loss": 3.3894563237477113,
+      "tokens_seen": 2571304960
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022299791365751887,
+      "loss": 1.1911,
+      "theoretical_loss": 3.3894427349228806,
+      "tokens_seen": 2571436032
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022295779168672768,
+      "loss": 1.2159,
+      "theoretical_loss": 3.389429146984618,
+      "tokens_seen": 2571567104
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022291766971593643,
+      "loss": 1.2942,
+      "theoretical_loss": 3.3894155599328197,
+      "tokens_seen": 2571698176
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022287754774514523,
+      "loss": 1.2345,
+      "theoretical_loss": 3.389401973767383,
+      "tokens_seen": 2571829248
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022283742577435404,
+      "loss": 1.2784,
+      "theoretical_loss": 3.3893883884882055,
+      "tokens_seen": 2571960320
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022279730380356285,
+      "loss": 1.2384,
+      "theoretical_loss": 3.3893748040951834,
+      "tokens_seen": 2572091392
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022275718183277163,
+      "loss": 1.1887,
+      "theoretical_loss": 3.3893612205882135,
+      "tokens_seen": 2572222464
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022271705986198043,
+      "loss": 1.2042,
+      "theoretical_loss": 3.389347637967194,
+      "tokens_seen": 2572353536
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022267693789118924,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3893340562320216,
+      "tokens_seen": 2572484608
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022263681592039802,
+      "loss": 1.256,
+      "theoretical_loss": 3.3893204753825925,
+      "tokens_seen": 2572615680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002225966939496068,
+      "loss": 1.1946,
+      "theoretical_loss": 3.389306895418805,
+      "tokens_seen": 2572746752
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002225565719788156,
+      "loss": 1.2619,
+      "theoretical_loss": 3.3892933163405554,
+      "tokens_seen": 2572877824
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022251645000802438,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3892797381477413,
+      "tokens_seen": 2573008896
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002224763280372332,
+      "loss": 1.1131,
+      "theoretical_loss": 3.3892661608402594,
+      "tokens_seen": 2573139968
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.000222436206066442,
+      "loss": 1.2264,
+      "theoretical_loss": 3.3892525844180077,
+      "tokens_seen": 2573271040
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002223960840956508,
+      "loss": 1.264,
+      "theoretical_loss": 3.3892390088808826,
+      "tokens_seen": 2573402112
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022235596212485958,
+      "loss": 1.2004,
+      "theoretical_loss": 3.389225434228782,
+      "tokens_seen": 2573533184
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022231584015406836,
+      "loss": 1.202,
+      "theoretical_loss": 3.3892118604616024,
+      "tokens_seen": 2573664256
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022227571818327717,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3891982875792417,
+      "tokens_seen": 2573795328
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022223559621248595,
+      "loss": 1.2114,
+      "theoretical_loss": 3.389184715581597,
+      "tokens_seen": 2573926400
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 864102,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1516239643096924,
+      "objective/train/theoretical_loss": 3.389171144468565,
+      "objective/train/tokens_used": 944516576,
+      "theoretical_loss": 3.389171144468565,
+      "tokens_seen": 2574057472
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022219547424169475,
+      "loss": 1.2084,
+      "theoretical_loss": 3.389171144468565,
+      "tokens_seen": 2574057472
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022215535227090356,
+      "loss": 1.2358,
+      "theoretical_loss": 3.3891575742400435,
+      "tokens_seen": 2574188544
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022211523030011236,
+      "loss": 1.2361,
+      "theoretical_loss": 3.38914400489593,
+      "tokens_seen": 2574319616
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022207510832932114,
+      "loss": 1.2897,
+      "theoretical_loss": 3.389130436436121,
+      "tokens_seen": 2574450688
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022203498635852992,
+      "loss": 1.2823,
+      "theoretical_loss": 3.389116868860515,
+      "tokens_seen": 2574581760
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022199486438773873,
+      "loss": 1.2364,
+      "theoretical_loss": 3.389103302169009,
+      "tokens_seen": 2574712832
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002219547424169475,
+      "loss": 1.176,
+      "theoretical_loss": 3.3890897363615,
+      "tokens_seen": 2574843904
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022191462044615632,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3890761714378854,
+      "tokens_seen": 2574974976
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022187449847536512,
+      "loss": 1.1923,
+      "theoretical_loss": 3.3890626073980634,
+      "tokens_seen": 2575106048
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022183437650457393,
+      "loss": 1.2965,
+      "theoretical_loss": 3.3890490442419305,
+      "tokens_seen": 2575237120
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002217942545337827,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3890354819693846,
+      "tokens_seen": 2575368192
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002217541325629915,
+      "loss": 1.2492,
+      "theoretical_loss": 3.3890219205803227,
+      "tokens_seen": 2575499264
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002217140105922003,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3890083600746435,
+      "tokens_seen": 2575630336
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022167388862140907,
+      "loss": 1.2179,
+      "theoretical_loss": 3.388994800452243,
+      "tokens_seen": 2575761408
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022163376665061788,
+      "loss": 1.1941,
+      "theoretical_loss": 3.3889812417130196,
+      "tokens_seen": 2575892480
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022159364467982668,
+      "loss": 1.2235,
+      "theoretical_loss": 3.388967683856871,
+      "tokens_seen": 2576023552
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022155352270903546,
+      "loss": 1.2041,
+      "theoretical_loss": 3.3889541268836947,
+      "tokens_seen": 2576154624
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022151340073824427,
+      "loss": 1.2256,
+      "theoretical_loss": 3.388940570793387,
+      "tokens_seen": 2576285696
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022147327876745308,
+      "loss": 1.2723,
+      "theoretical_loss": 3.388927015585847,
+      "tokens_seen": 2576416768
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022143315679666186,
+      "loss": 1.1467,
+      "theoretical_loss": 3.388913461260972,
+      "tokens_seen": 2576547840
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022139303482587063,
+      "loss": 1.2952,
+      "theoretical_loss": 3.3888999078186597,
+      "tokens_seen": 2576678912
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022135291285507944,
+      "loss": 1.3349,
+      "theoretical_loss": 3.3888863552588067,
+      "tokens_seen": 2576809984
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022131279088428825,
+      "loss": 1.2568,
+      "theoretical_loss": 3.388872803581312,
+      "tokens_seen": 2576941056
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022127266891349703,
+      "loss": 1.249,
+      "theoretical_loss": 3.388859252786072,
+      "tokens_seen": 2577072128
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022123254694270583,
+      "loss": 1.2598,
+      "theoretical_loss": 3.388845702872986,
+      "tokens_seen": 2577203200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022119242497191464,
+      "loss": 1.3323,
+      "theoretical_loss": 3.3888321538419506,
+      "tokens_seen": 2577334272
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022115230300112342,
+      "loss": 1.2089,
+      "theoretical_loss": 3.3888186056928635,
+      "tokens_seen": 2577465344
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002211121810303322,
+      "loss": 1.2089,
+      "theoretical_loss": 3.3888050584256226,
+      "tokens_seen": 2577596416
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.000221072059059541,
+      "loss": 1.2564,
+      "theoretical_loss": 3.388791512040126,
+      "tokens_seen": 2577727488
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002210319370887498,
+      "loss": 1.2173,
+      "theoretical_loss": 3.388777966536271,
+      "tokens_seen": 2577858560
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002209918151179586,
+      "loss": 1.2187,
+      "theoretical_loss": 3.388764421913956,
+      "tokens_seen": 2577989632
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002209516931471674,
+      "loss": 1.2381,
+      "theoretical_loss": 3.388750878173078,
+      "tokens_seen": 2578120704
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002209115711763762,
+      "loss": 1.2002,
+      "theoretical_loss": 3.388737335313536,
+      "tokens_seen": 2578251776
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.000220871449205585,
+      "loss": 1.2277,
+      "theoretical_loss": 3.388723793335226,
+      "tokens_seen": 2578382848
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022083132723479376,
+      "loss": 1.1918,
+      "theoretical_loss": 3.388710252238048,
+      "tokens_seen": 2578513920
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022079120526400257,
+      "loss": 1.2375,
+      "theoretical_loss": 3.3886967120218987,
+      "tokens_seen": 2578644992
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022075108329321137,
+      "loss": 1.2182,
+      "theoretical_loss": 3.388683172686676,
+      "tokens_seen": 2578776064
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022071096132242015,
+      "loss": 1.2651,
+      "theoretical_loss": 3.3886696342322775,
+      "tokens_seen": 2578907136
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022067083935162896,
+      "loss": 1.2206,
+      "theoretical_loss": 3.388656096658602,
+      "tokens_seen": 2579038208
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022063071738083776,
+      "loss": 1.2084,
+      "theoretical_loss": 3.3886425599655476,
+      "tokens_seen": 2579169280
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022059059541004654,
+      "loss": 1.1834,
+      "theoretical_loss": 3.3886290241530115,
+      "tokens_seen": 2579300352
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022055047343925532,
+      "loss": 1.2192,
+      "theoretical_loss": 3.3886154892208915,
+      "tokens_seen": 2579431424
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022051035146846413,
+      "loss": 1.2481,
+      "theoretical_loss": 3.3886019551690865,
+      "tokens_seen": 2579562496
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022047022949767294,
+      "loss": 1.1925,
+      "theoretical_loss": 3.3885884219974933,
+      "tokens_seen": 2579693568
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022043010752688171,
+      "loss": 1.2653,
+      "theoretical_loss": 3.3885748897060113,
+      "tokens_seen": 2579824640
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022038998555609052,
+      "loss": 1.3165,
+      "theoretical_loss": 3.388561358294538,
+      "tokens_seen": 2579955712
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022034986358529933,
+      "loss": 1.2912,
+      "theoretical_loss": 3.388547827762971,
+      "tokens_seen": 2580086784
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002203097416145081,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3885342981112094,
+      "tokens_seen": 2580217856
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002202696196437169,
+      "loss": 1.2123,
+      "theoretical_loss": 3.3885207693391504,
+      "tokens_seen": 2580348928
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002202294976729257,
+      "loss": 1.2825,
+      "theoretical_loss": 3.388507241446692,
+      "tokens_seen": 2580480000
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 866470,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2558972835540771,
+      "objective/train/theoretical_loss": 3.388493714433733,
+      "objective/train/tokens_used": 951070176,
+      "theoretical_loss": 3.388493714433733,
+      "tokens_seen": 2580611072
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002201893757021345,
+      "loss": 1.2406,
+      "theoretical_loss": 3.388493714433733,
+      "tokens_seen": 2580611072
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022014925373134328,
+      "loss": 1.2623,
+      "theoretical_loss": 3.3884801883001714,
+      "tokens_seen": 2580742144
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022010913176055208,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3884666630459055,
+      "tokens_seen": 2580873216
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002200690097897609,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3884531386708328,
+      "tokens_seen": 2581004288
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022002888781896967,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3884396151748524,
+      "tokens_seen": 2581135360
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021998876584817848,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3884260925578618,
+      "tokens_seen": 2581266432
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021994864387738725,
+      "loss": 1.2071,
+      "theoretical_loss": 3.388412570819759,
+      "tokens_seen": 2581397504
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021990852190659606,
+      "loss": 1.26,
+      "theoretical_loss": 3.388399049960444,
+      "tokens_seen": 2581528576
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021986839993580484,
+      "loss": 1.2974,
+      "theoretical_loss": 3.3883855299798133,
+      "tokens_seen": 2581659648
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021982827796501365,
+      "loss": 1.2431,
+      "theoretical_loss": 3.3883720108777657,
+      "tokens_seen": 2581790720
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021978815599422245,
+      "loss": 1.304,
+      "theoretical_loss": 3.3883584926541994,
+      "tokens_seen": 2581921792
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021974803402343123,
+      "loss": 1.1491,
+      "theoretical_loss": 3.388344975309013,
+      "tokens_seen": 2582052864
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00021970791205264004,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3883314588421047,
+      "tokens_seen": 2582183936
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021966779008184882,
+      "loss": 1.2866,
+      "theoretical_loss": 3.388317943253373,
+      "tokens_seen": 2582315008
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002196276681110576,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3883044285427157,
+      "tokens_seen": 2582446080
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002195875461402664,
+      "loss": 1.2099,
+      "theoretical_loss": 3.388290914710032,
+      "tokens_seen": 2582577152
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002195474241694752,
+      "loss": 1.2177,
+      "theoretical_loss": 3.38827740175522,
+      "tokens_seen": 2582708224
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021950730219868402,
+      "loss": 1.2062,
+      "theoretical_loss": 3.388263889678178,
+      "tokens_seen": 2582839296
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002194671802278928,
+      "loss": 1.22,
+      "theoretical_loss": 3.3882503784788045,
+      "tokens_seen": 2582970368
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002194270582571016,
+      "loss": 1.2142,
+      "theoretical_loss": 3.388236868156998,
+      "tokens_seen": 2583101440
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002193869362863104,
+      "loss": 1.313,
+      "theoretical_loss": 3.388223358712657,
+      "tokens_seen": 2583232512
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021934681431551916,
+      "loss": 1.2797,
+      "theoretical_loss": 3.38820985014568,
+      "tokens_seen": 2583363584
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021930669234472797,
+      "loss": 1.2315,
+      "theoretical_loss": 3.388196342455965,
+      "tokens_seen": 2583494656
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021926657037393677,
+      "loss": 1.2166,
+      "theoretical_loss": 3.3881828356434114,
+      "tokens_seen": 2583625728
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021922644840314558,
+      "loss": 1.2112,
+      "theoretical_loss": 3.3881693297079174,
+      "tokens_seen": 2583756800
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021918632643235436,
+      "loss": 1.2358,
+      "theoretical_loss": 3.388155824649381,
+      "tokens_seen": 2583887872
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021914620446156316,
+      "loss": 1.2517,
+      "theoretical_loss": 3.3881423204677015,
+      "tokens_seen": 2584018944
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021910608249077197,
+      "loss": 1.162,
+      "theoretical_loss": 3.3881288171627775,
+      "tokens_seen": 2584150016
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021906596051998072,
+      "loss": 1.2598,
+      "theoretical_loss": 3.3881153147345073,
+      "tokens_seen": 2584281088
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021902583854918953,
+      "loss": 1.2501,
+      "theoretical_loss": 3.3881018131827894,
+      "tokens_seen": 2584412160
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021898571657839834,
+      "loss": 1.2485,
+      "theoretical_loss": 3.3880883125075227,
+      "tokens_seen": 2584543232
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021894559460760714,
+      "loss": 1.1941,
+      "theoretical_loss": 3.3880748127086058,
+      "tokens_seen": 2584674304
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021890547263681592,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3880613137859372,
+      "tokens_seen": 2584805376
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021886535066602473,
+      "loss": 1.2522,
+      "theoretical_loss": 3.388047815739416,
+      "tokens_seen": 2584936448
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021882522869523353,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3880343185689408,
+      "tokens_seen": 2585067520
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021878510672444229,
+      "loss": 1.2549,
+      "theoretical_loss": 3.38802082227441,
+      "tokens_seen": 2585198592
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002187449847536511,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3880073268557225,
+      "tokens_seen": 2585329664
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002187048627828599,
+      "loss": 1.2781,
+      "theoretical_loss": 3.3879938323127776,
+      "tokens_seen": 2585460736
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002186647408120687,
+      "loss": 1.2504,
+      "theoretical_loss": 3.3879803386454728,
+      "tokens_seen": 2585591808
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021862461884127748,
+      "loss": 1.256,
+      "theoretical_loss": 3.3879668458537084,
+      "tokens_seen": 2585722880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002185844968704863,
+      "loss": 1.1804,
+      "theoretical_loss": 3.387953353937382,
+      "tokens_seen": 2585853952
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002185443748996951,
+      "loss": 1.1958,
+      "theoretical_loss": 3.3879398628963933,
+      "tokens_seen": 2585985024
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021850425292890388,
+      "loss": 1.2204,
+      "theoretical_loss": 3.387926372730641,
+      "tokens_seen": 2586116096
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021846413095811265,
+      "loss": 1.2717,
+      "theoretical_loss": 3.387912883440023,
+      "tokens_seen": 2586247168
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021842400898732146,
+      "loss": 1.2666,
+      "theoretical_loss": 3.3878993950244394,
+      "tokens_seen": 2586378240
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021838388701653024,
+      "loss": 1.2431,
+      "theoretical_loss": 3.3878859074837884,
+      "tokens_seen": 2586509312
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021834376504573905,
+      "loss": 1.2177,
+      "theoretical_loss": 3.3878724208179696,
+      "tokens_seen": 2586640384
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021830364307494785,
+      "loss": 1.2209,
+      "theoretical_loss": 3.387858935026881,
+      "tokens_seen": 2586771456
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021826352110415666,
+      "loss": 1.2882,
+      "theoretical_loss": 3.3878454501104223,
+      "tokens_seen": 2586902528
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021822339913336544,
+      "loss": 1.3062,
+      "theoretical_loss": 3.387831966068492,
+      "tokens_seen": 2587033600
+    },
+    {
+      "epoch": 0.57,
+      "objective/train/docs_used": 868686,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2645580768585205,
+      "objective/train/theoretical_loss": 3.387818482900989,
+      "objective/train/tokens_used": 957623776,
+      "theoretical_loss": 3.387818482900989,
+      "tokens_seen": 2587164672
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021818327716257422,
+      "loss": 1.2876,
+      "theoretical_loss": 3.387818482900989,
+      "tokens_seen": 2587164672
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021814315519178302,
+      "loss": 1.228,
+      "theoretical_loss": 3.387805000607813,
+      "tokens_seen": 2587295744
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002181030332209918,
+      "loss": 1.2807,
+      "theoretical_loss": 3.3877915191888626,
+      "tokens_seen": 2587426816
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002180629112502006,
+      "loss": 1.2645,
+      "theoretical_loss": 3.387778038644037,
+      "tokens_seen": 2587557888
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021802278927940942,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3877645589732346,
+      "tokens_seen": 2587688960
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021798266730861822,
+      "loss": 1.2265,
+      "theoretical_loss": 3.387751080176355,
+      "tokens_seen": 2587820032
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.000217942545337827,
+      "loss": 1.2248,
+      "theoretical_loss": 3.3877376022532975,
+      "tokens_seen": 2587951104
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002179024233670358,
+      "loss": 1.1608,
+      "theoretical_loss": 3.3877241252039605,
+      "tokens_seen": 2588082176
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002178623013962446,
+      "loss": 1.2154,
+      "theoretical_loss": 3.3877106490282434,
+      "tokens_seen": 2588213248
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021782217942545337,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3876971737260457,
+      "tokens_seen": 2588344320
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021778205745466217,
+      "loss": 1.2599,
+      "theoretical_loss": 3.3876836992972663,
+      "tokens_seen": 2588475392
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021774193548387098,
+      "loss": 1.2915,
+      "theoretical_loss": 3.3876702257418048,
+      "tokens_seen": 2588606464
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021770181351307978,
+      "loss": 1.2227,
+      "theoretical_loss": 3.3876567530595594,
+      "tokens_seen": 2588737536
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021766169154228856,
+      "loss": 1.2775,
+      "theoretical_loss": 3.38764328125043,
+      "tokens_seen": 2588868608
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021762156957149737,
+      "loss": 1.1546,
+      "theoretical_loss": 3.3876298103143156,
+      "tokens_seen": 2588999680
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021758144760070615,
+      "loss": 1.2315,
+      "theoretical_loss": 3.3876163402511157,
+      "tokens_seen": 2589130752
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021754132562991493,
+      "loss": 1.2747,
+      "theoretical_loss": 3.3876028710607287,
+      "tokens_seen": 2589261824
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021750120365912373,
+      "loss": 1.2481,
+      "theoretical_loss": 3.387589402743055,
+      "tokens_seen": 2589392896
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021746108168833254,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3875759352979937,
+      "tokens_seen": 2589523968
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021742095971754132,
+      "loss": 1.181,
+      "theoretical_loss": 3.3875624687254433,
+      "tokens_seen": 2589655040
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021738083774675013,
+      "loss": 1.2058,
+      "theoretical_loss": 3.387549003025304,
+      "tokens_seen": 2589786112
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021734071577595893,
+      "loss": 1.182,
+      "theoretical_loss": 3.3875355381974743,
+      "tokens_seen": 2589917184
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002173005938051677,
+      "loss": 1.2428,
+      "theoretical_loss": 3.387522074241854,
+      "tokens_seen": 2590048256
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002172604718343765,
+      "loss": 1.2288,
+      "theoretical_loss": 3.387508611158343,
+      "tokens_seen": 2590179328
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002172203498635853,
+      "loss": 1.2932,
+      "theoretical_loss": 3.3874951489468397,
+      "tokens_seen": 2590310400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002171802278927941,
+      "loss": 1.2205,
+      "theoretical_loss": 3.387481687607244,
+      "tokens_seen": 2590441472
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021714010592200288,
+      "loss": 1.2724,
+      "theoretical_loss": 3.387468227139455,
+      "tokens_seen": 2590572544
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002170999839512117,
+      "loss": 1.2072,
+      "theoretical_loss": 3.387454767543373,
+      "tokens_seen": 2590703616
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002170598619804205,
+      "loss": 1.2349,
+      "theoretical_loss": 3.3874413088188966,
+      "tokens_seen": 2590834688
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002170197400096293,
+      "loss": 1.1946,
+      "theoretical_loss": 3.387427850965925,
+      "tokens_seen": 2590965760
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021697961803883805,
+      "loss": 1.2958,
+      "theoretical_loss": 3.3874143939843586,
+      "tokens_seen": 2591096832
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021693949606804686,
+      "loss": 1.2835,
+      "theoretical_loss": 3.3874009378740966,
+      "tokens_seen": 2591227904
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021689937409725567,
+      "loss": 1.2524,
+      "theoretical_loss": 3.387387482635038,
+      "tokens_seen": 2591358976
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021685925212646445,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3873740282670832,
+      "tokens_seen": 2591490048
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021681913015567325,
+      "loss": 1.2121,
+      "theoretical_loss": 3.387360574770131,
+      "tokens_seen": 2591621120
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021677900818488206,
+      "loss": 1.1936,
+      "theoretical_loss": 3.3873471221440816,
+      "tokens_seen": 2591752192
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021673888621409087,
+      "loss": 1.221,
+      "theoretical_loss": 3.387333670388834,
+      "tokens_seen": 2591883264
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021669876424329962,
+      "loss": 1.2255,
+      "theoretical_loss": 3.387320219504288,
+      "tokens_seen": 2592014336
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021665864227250842,
+      "loss": 1.2218,
+      "theoretical_loss": 3.387306769490343,
+      "tokens_seen": 2592145408
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021661852030171723,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3872933203468993,
+      "tokens_seen": 2592276480
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.000216578398330926,
+      "loss": 1.2541,
+      "theoretical_loss": 3.387279872073856,
+      "tokens_seen": 2592407552
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021653827636013482,
+      "loss": 1.2221,
+      "theoretical_loss": 3.387266424671113,
+      "tokens_seen": 2592538624
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021649815438934362,
+      "loss": 1.2438,
+      "theoretical_loss": 3.38725297813857,
+      "tokens_seen": 2592669696
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002164580324185524,
+      "loss": 1.209,
+      "theoretical_loss": 3.387239532476126,
+      "tokens_seen": 2592800768
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021641791044776118,
+      "loss": 1.2324,
+      "theoretical_loss": 3.3872260876836817,
+      "tokens_seen": 2592931840
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021637778847696999,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3872126437611363,
+      "tokens_seen": 2593062912
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002163376665061788,
+      "loss": 1.2313,
+      "theoretical_loss": 3.38719920070839,
+      "tokens_seen": 2593193984
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021629754453538757,
+      "loss": 1.302,
+      "theoretical_loss": 3.3871857585253418,
+      "tokens_seen": 2593325056
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021625742256459638,
+      "loss": 1.167,
+      "theoretical_loss": 3.3871723172118924,
+      "tokens_seen": 2593456128
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021621730059380518,
+      "loss": 1.2798,
+      "theoretical_loss": 3.387158876767941,
+      "tokens_seen": 2593587200
+    },
+    {
+      "epoch": 0.57,
+      "objective/train/docs_used": 870893,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2933621406555176,
+      "objective/train/theoretical_loss": 3.387145437193387,
+      "objective/train/tokens_used": 964177376,
+      "theoretical_loss": 3.387145437193387,
+      "tokens_seen": 2593718272
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021617717862301396,
+      "loss": 1.2627,
+      "theoretical_loss": 3.387145437193387,
+      "tokens_seen": 2593718272
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021613705665222277,
+      "loss": 1.2075,
+      "theoretical_loss": 3.3871319984881314,
+      "tokens_seen": 2593849344
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021609693468143155,
+      "loss": 1.205,
+      "theoretical_loss": 3.3871185606520733,
+      "tokens_seen": 2593980416
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021605681271064036,
+      "loss": 1.238,
+      "theoretical_loss": 3.3871051236851124,
+      "tokens_seen": 2594111488
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021601669073984913,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3870916875871493,
+      "tokens_seen": 2594242560
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021597656876905794,
+      "loss": 1.164,
+      "theoretical_loss": 3.387078252358083,
+      "tokens_seen": 2594373632
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021593644679826675,
+      "loss": 1.1883,
+      "theoretical_loss": 3.3870648179978144,
+      "tokens_seen": 2594504704
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021589632482747553,
+      "loss": 1.2326,
+      "theoretical_loss": 3.3870513845062424,
+      "tokens_seen": 2594635776
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021585620285668433,
+      "loss": 1.2431,
+      "theoretical_loss": 3.387037951883268,
+      "tokens_seen": 2594766848
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002158160808858931,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3870245201287905,
+      "tokens_seen": 2594897920
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021577595891510192,
+      "loss": 1.27,
+      "theoretical_loss": 3.38701108924271,
+      "tokens_seen": 2595028992
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002157358369443107,
+      "loss": 1.2557,
+      "theoretical_loss": 3.3869976592249262,
+      "tokens_seen": 2595160064
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002156957149735195,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3869842300753397,
+      "tokens_seen": 2595291136
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002156555930027283,
+      "loss": 1.2488,
+      "theoretical_loss": 3.3869708017938502,
+      "tokens_seen": 2595422208
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002156154710319371,
+      "loss": 1.2849,
+      "theoretical_loss": 3.386957374380358,
+      "tokens_seen": 2595553280
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002155753490611459,
+      "loss": 1.2389,
+      "theoretical_loss": 3.3869439478347627,
+      "tokens_seen": 2595684352
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002155352270903547,
+      "loss": 1.2836,
+      "theoretical_loss": 3.3869305221569648,
+      "tokens_seen": 2595815424
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021549510511956345,
+      "loss": 1.1999,
+      "theoretical_loss": 3.3869170973468643,
+      "tokens_seen": 2595946496
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021545498314877226,
+      "loss": 1.2706,
+      "theoretical_loss": 3.386903673404361,
+      "tokens_seen": 2596077568
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021541486117798107,
+      "loss": 1.2621,
+      "theoretical_loss": 3.3868902503293556,
+      "tokens_seen": 2596208640
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021537473920718987,
+      "loss": 1.2621,
+      "theoretical_loss": 3.3868768281217476,
+      "tokens_seen": 2596339712
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021533461723639865,
+      "loss": 1.2549,
+      "theoretical_loss": 3.386863406781438,
+      "tokens_seen": 2596470784
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021529449526560746,
+      "loss": 1.2114,
+      "theoretical_loss": 3.3868499863083263,
+      "tokens_seen": 2596601856
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021525437329481626,
+      "loss": 1.1902,
+      "theoretical_loss": 3.386836566702313,
+      "tokens_seen": 2596732928
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021521425132402502,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3868231479632978,
+      "tokens_seen": 2596864000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021517412935323382,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3868097300911817,
+      "tokens_seen": 2596995072
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021513400738244263,
+      "loss": 1.2467,
+      "theoretical_loss": 3.3867963130858643,
+      "tokens_seen": 2597126144
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021509388541165144,
+      "loss": 1.2699,
+      "theoretical_loss": 3.3867828969472464,
+      "tokens_seen": 2597257216
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021505376344086021,
+      "loss": 1.2332,
+      "theoretical_loss": 3.386769481675228,
+      "tokens_seen": 2597388288
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021501364147006902,
+      "loss": 1.254,
+      "theoretical_loss": 3.386756067269709,
+      "tokens_seen": 2597519360
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021497351949927783,
+      "loss": 1.199,
+      "theoretical_loss": 3.3867426537305905,
+      "tokens_seen": 2597650432
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021493339752848658,
+      "loss": 1.2524,
+      "theoretical_loss": 3.3867292410577723,
+      "tokens_seen": 2597781504
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021489327555769539,
+      "loss": 1.3244,
+      "theoretical_loss": 3.386715829251155,
+      "tokens_seen": 2597912576
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002148531535869042,
+      "loss": 1.2971,
+      "theoretical_loss": 3.386702418310639,
+      "tokens_seen": 2598043648
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.000214813031616113,
+      "loss": 1.2871,
+      "theoretical_loss": 3.3866890082361243,
+      "tokens_seen": 2598174720
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021477290964532178,
+      "loss": 1.2855,
+      "theoretical_loss": 3.3866755990275115,
+      "tokens_seen": 2598305792
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021473278767453058,
+      "loss": 1.2297,
+      "theoretical_loss": 3.386662190684701,
+      "tokens_seen": 2598436864
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002146926657037394,
+      "loss": 1.264,
+      "theoretical_loss": 3.386648783207594,
+      "tokens_seen": 2598567936
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021465254373294817,
+      "loss": 1.1841,
+      "theoretical_loss": 3.3866353765960895,
+      "tokens_seen": 2598699008
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021461242176215695,
+      "loss": 1.2393,
+      "theoretical_loss": 3.386621970850089,
+      "tokens_seen": 2598830080
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021457229979136576,
+      "loss": 1.1909,
+      "theoretical_loss": 3.3866085659694924,
+      "tokens_seen": 2598961152
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021453217782057456,
+      "loss": 1.2489,
+      "theoretical_loss": 3.386595161954201,
+      "tokens_seen": 2599092224
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021449205584978334,
+      "loss": 1.257,
+      "theoretical_loss": 3.3865817588041143,
+      "tokens_seen": 2599223296
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021445193387899215,
+      "loss": 1.2086,
+      "theoretical_loss": 3.3865683565191333,
+      "tokens_seen": 2599354368
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021441181190820095,
+      "loss": 1.2871,
+      "theoretical_loss": 3.3865549550991587,
+      "tokens_seen": 2599485440
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021437168993740973,
+      "loss": 1.2238,
+      "theoretical_loss": 3.386541554544091,
+      "tokens_seen": 2599616512
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002143315679666185,
+      "loss": 1.2891,
+      "theoretical_loss": 3.3865281548538304,
+      "tokens_seen": 2599747584
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021429144599582732,
+      "loss": 1.2495,
+      "theoretical_loss": 3.386514756028278,
+      "tokens_seen": 2599878656
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002142513240250361,
+      "loss": 1.3096,
+      "theoretical_loss": 3.386501358067334,
+      "tokens_seen": 2600009728
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002142112020542449,
+      "loss": 1.1857,
+      "theoretical_loss": 3.386487960970899,
+      "tokens_seen": 2600140800
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 872942,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1980268955230713,
+      "objective/train/theoretical_loss": 3.386474564738874,
+      "objective/train/tokens_used": 970730976,
+      "theoretical_loss": 3.386474564738874,
+      "tokens_seen": 2600271872
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002141710800834537,
+      "loss": 1.2592,
+      "theoretical_loss": 3.386474564738874,
+      "tokens_seen": 2600271872
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021413095811266252,
+      "loss": 1.2615,
+      "theoretical_loss": 3.3864611693711595,
+      "tokens_seen": 2600402944
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002140908361418713,
+      "loss": 1.2941,
+      "theoretical_loss": 3.3864477748676562,
+      "tokens_seen": 2600534016
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021405071417108007,
+      "loss": 1.2334,
+      "theoretical_loss": 3.386434381228265,
+      "tokens_seen": 2600665088
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021401059220028888,
+      "loss": 1.2788,
+      "theoretical_loss": 3.386420988452886,
+      "tokens_seen": 2600796160
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021397047022949766,
+      "loss": 1.2539,
+      "theoretical_loss": 3.3864075965414204,
+      "tokens_seen": 2600927232
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021393034825870647,
+      "loss": 1.2536,
+      "theoretical_loss": 3.3863942054937692,
+      "tokens_seen": 2601058304
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021389022628791527,
+      "loss": 1.2992,
+      "theoretical_loss": 3.3863808153098325,
+      "tokens_seen": 2601189376
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021385010431712408,
+      "loss": 1.1661,
+      "theoretical_loss": 3.386367425989511,
+      "tokens_seen": 2601320448
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021380998234633286,
+      "loss": 1.2268,
+      "theoretical_loss": 3.3863540375327066,
+      "tokens_seen": 2601451520
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021376986037554166,
+      "loss": 1.1904,
+      "theoretical_loss": 3.386340649939319,
+      "tokens_seen": 2601582592
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021372973840475044,
+      "loss": 1.1854,
+      "theoretical_loss": 3.386327263209249,
+      "tokens_seen": 2601713664
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021368961643395922,
+      "loss": 1.2104,
+      "theoretical_loss": 3.386313877342398,
+      "tokens_seen": 2601844736
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021364949446316803,
+      "loss": 1.1963,
+      "theoretical_loss": 3.386300492338667,
+      "tokens_seen": 2601975808
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021360937249237684,
+      "loss": 1.1746,
+      "theoretical_loss": 3.3862871081979566,
+      "tokens_seen": 2602106880
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021356925052158564,
+      "loss": 1.2802,
+      "theoretical_loss": 3.3862737249201675,
+      "tokens_seen": 2602237952
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021352912855079442,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3862603425052007,
+      "tokens_seen": 2602369024
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021348900658000323,
+      "loss": 1.2327,
+      "theoretical_loss": 3.386246960952957,
+      "tokens_seen": 2602500096
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.000213448884609212,
+      "loss": 1.254,
+      "theoretical_loss": 3.3862335802633377,
+      "tokens_seen": 2602631168
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021340876263842079,
+      "loss": 1.2442,
+      "theoretical_loss": 3.386220200436244,
+      "tokens_seen": 2602762240
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002133686406676296,
+      "loss": 1.2612,
+      "theoretical_loss": 3.3862068214715757,
+      "tokens_seen": 2602893312
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002133285186968384,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3861934433692347,
+      "tokens_seen": 2603024384
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021328839672604718,
+      "loss": 1.2454,
+      "theoretical_loss": 3.386180066129122,
+      "tokens_seen": 2603155456
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021324827475525598,
+      "loss": 1.2706,
+      "theoretical_loss": 3.386166689751138,
+      "tokens_seen": 2603286528
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002132081527844648,
+      "loss": 1.2004,
+      "theoretical_loss": 3.3861533142351847,
+      "tokens_seen": 2603417600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002131680308136736,
+      "loss": 1.3128,
+      "theoretical_loss": 3.386139939581162,
+      "tokens_seen": 2603548672
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021312790884288235,
+      "loss": 1.2517,
+      "theoretical_loss": 3.386126565788972,
+      "tokens_seen": 2603679744
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021308778687209115,
+      "loss": 1.1896,
+      "theoretical_loss": 3.3861131928585158,
+      "tokens_seen": 2603810816
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021304766490129996,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3860998207896933,
+      "tokens_seen": 2603941888
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021300754293050874,
+      "loss": 1.202,
+      "theoretical_loss": 3.3860864495824066,
+      "tokens_seen": 2604072960
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021296742095971755,
+      "loss": 1.236,
+      "theoretical_loss": 3.386073079236557,
+      "tokens_seen": 2604204032
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021292729898892635,
+      "loss": 1.2267,
+      "theoretical_loss": 3.386059709752045,
+      "tokens_seen": 2604335104
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021288717701813516,
+      "loss": 1.2978,
+      "theoretical_loss": 3.386046341128772,
+      "tokens_seen": 2604466176
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002128470550473439,
+      "loss": 1.2193,
+      "theoretical_loss": 3.386032973366639,
+      "tokens_seen": 2604597248
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021280693307655272,
+      "loss": 1.3524,
+      "theoretical_loss": 3.3860196064655477,
+      "tokens_seen": 2604728320
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021276681110576152,
+      "loss": 1.293,
+      "theoretical_loss": 3.3860062404253988,
+      "tokens_seen": 2604859392
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002127266891349703,
+      "loss": 1.2016,
+      "theoretical_loss": 3.385992875246094,
+      "tokens_seen": 2604990464
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002126865671641791,
+      "loss": 1.2176,
+      "theoretical_loss": 3.3859795109275344,
+      "tokens_seen": 2605121536
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021264644519338792,
+      "loss": 1.2808,
+      "theoretical_loss": 3.385966147469621,
+      "tokens_seen": 2605252608
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021260632322259672,
+      "loss": 1.2908,
+      "theoretical_loss": 3.3859527848722553,
+      "tokens_seen": 2605383680
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021256620125180547,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3859394231353384,
+      "tokens_seen": 2605514752
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021252607928101428,
+      "loss": 1.2715,
+      "theoretical_loss": 3.385926062258772,
+      "tokens_seen": 2605645824
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002124859573102231,
+      "loss": 1.2857,
+      "theoretical_loss": 3.385912702242457,
+      "tokens_seen": 2605776896
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021244583533943187,
+      "loss": 1.2786,
+      "theoretical_loss": 3.3858993430862947,
+      "tokens_seen": 2605907968
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021240571336864067,
+      "loss": 1.2486,
+      "theoretical_loss": 3.385885984790187,
+      "tokens_seen": 2606039040
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021236559139784948,
+      "loss": 1.3138,
+      "theoretical_loss": 3.385872627354035,
+      "tokens_seen": 2606170112
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021232546942705826,
+      "loss": 1.2217,
+      "theoretical_loss": 3.38585927077774,
+      "tokens_seen": 2606301184
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021228534745626706,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3858459150612035,
+      "tokens_seen": 2606432256
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021224522548547584,
+      "loss": 1.2603,
+      "theoretical_loss": 3.385832560204327,
+      "tokens_seen": 2606563328
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021220510351468465,
+      "loss": 1.3183,
+      "theoretical_loss": 3.3858192062070116,
+      "tokens_seen": 2606694400
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 875060,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2990853786468506,
+      "objective/train/theoretical_loss": 3.3858058530691593,
+      "objective/train/tokens_used": 977284576,
+      "theoretical_loss": 3.3858058530691593,
+      "tokens_seen": 2606825472
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021216498154389343,
+      "loss": 1.2962,
+      "theoretical_loss": 3.3858058530691593,
+      "tokens_seen": 2606825472
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021212485957310223,
+      "loss": 1.2518,
+      "theoretical_loss": 3.385792500790671,
+      "tokens_seen": 2606956544
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021208473760231104,
+      "loss": 1.2371,
+      "theoretical_loss": 3.385779149371449,
+      "tokens_seen": 2607087616
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021204461563151982,
+      "loss": 1.2236,
+      "theoretical_loss": 3.3857657988113936,
+      "tokens_seen": 2607218688
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021200449366072863,
+      "loss": 1.2736,
+      "theoretical_loss": 3.385752449110407,
+      "tokens_seen": 2607349760
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002119643716899374,
+      "loss": 1.2257,
+      "theoretical_loss": 3.385739100268391,
+      "tokens_seen": 2607480832
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002119242497191462,
+      "loss": 1.3559,
+      "theoretical_loss": 3.3857257522852473,
+      "tokens_seen": 2607611904
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.000211884127748355,
+      "loss": 1.234,
+      "theoretical_loss": 3.3857124051608767,
+      "tokens_seen": 2607742976
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002118440057775638,
+      "loss": 1.3299,
+      "theoretical_loss": 3.385699058895181,
+      "tokens_seen": 2607874048
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002118038838067726,
+      "loss": 1.24,
+      "theoretical_loss": 3.385685713488062,
+      "tokens_seen": 2608005120
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021176376183598138,
+      "loss": 1.2353,
+      "theoretical_loss": 3.3856723689394217,
+      "tokens_seen": 2608136192
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002117236398651902,
+      "loss": 1.2462,
+      "theoretical_loss": 3.385659025249161,
+      "tokens_seen": 2608267264
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021168351789439897,
+      "loss": 1.3302,
+      "theoretical_loss": 3.3856456824171817,
+      "tokens_seen": 2608398336
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021164339592360778,
+      "loss": 1.2793,
+      "theoretical_loss": 3.3856323404433857,
+      "tokens_seen": 2608529408
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021160327395281655,
+      "loss": 1.2573,
+      "theoretical_loss": 3.385618999327675,
+      "tokens_seen": 2608660480
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021156315198202536,
+      "loss": 1.2185,
+      "theoretical_loss": 3.3856056590699506,
+      "tokens_seen": 2608791552
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021152303001123417,
+      "loss": 1.2797,
+      "theoretical_loss": 3.3855923196701148,
+      "tokens_seen": 2608922624
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021148290804044295,
+      "loss": 1.2841,
+      "theoretical_loss": 3.3855789811280688,
+      "tokens_seen": 2609053696
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021144278606965175,
+      "loss": 1.2519,
+      "theoretical_loss": 3.385565643443715,
+      "tokens_seen": 2609184768
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021140266409886056,
+      "loss": 1.2591,
+      "theoretical_loss": 3.3855523066169546,
+      "tokens_seen": 2609315840
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002113625421280693,
+      "loss": 1.2939,
+      "theoretical_loss": 3.3855389706476893,
+      "tokens_seen": 2609446912
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021132242015727812,
+      "loss": 1.2102,
+      "theoretical_loss": 3.3855256355358216,
+      "tokens_seen": 2609577984
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021128229818648692,
+      "loss": 1.2332,
+      "theoretical_loss": 3.3855123012812527,
+      "tokens_seen": 2609709056
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021124217621569573,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3854989678838847,
+      "tokens_seen": 2609840128
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002112020542449045,
+      "loss": 1.2908,
+      "theoretical_loss": 3.3854856353436196,
+      "tokens_seen": 2609971200
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021116193227411332,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3854723036603587,
+      "tokens_seen": 2610102272
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021112181030332212,
+      "loss": 1.2862,
+      "theoretical_loss": 3.3854589728340043,
+      "tokens_seen": 2610233344
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021108168833253087,
+      "loss": 1.1227,
+      "theoretical_loss": 3.3854456428644584,
+      "tokens_seen": 2610364416
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021104156636173968,
+      "loss": 1.2145,
+      "theoretical_loss": 3.3854323137516227,
+      "tokens_seen": 2610495488
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021100144439094849,
+      "loss": 1.2116,
+      "theoretical_loss": 3.385418985495399,
+      "tokens_seen": 2610626560
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002109613224201573,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3854056580956895,
+      "tokens_seen": 2610757632
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021092120044936607,
+      "loss": 1.2426,
+      "theoretical_loss": 3.385392331552396,
+      "tokens_seen": 2610888704
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021088107847857488,
+      "loss": 1.2021,
+      "theoretical_loss": 3.3853790058654205,
+      "tokens_seen": 2611019776
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021084095650778368,
+      "loss": 1.2668,
+      "theoretical_loss": 3.3853656810346653,
+      "tokens_seen": 2611150848
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021080083453699246,
+      "loss": 1.1899,
+      "theoretical_loss": 3.3853523570600323,
+      "tokens_seen": 2611281920
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021076071256620124,
+      "loss": 1.2181,
+      "theoretical_loss": 3.385339033941423,
+      "tokens_seen": 2611412992
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021072059059541005,
+      "loss": 1.277,
+      "theoretical_loss": 3.38532571167874,
+      "tokens_seen": 2611544064
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021068046862461886,
+      "loss": 1.2815,
+      "theoretical_loss": 3.385312390271885,
+      "tokens_seen": 2611675136
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021064034665382763,
+      "loss": 1.2775,
+      "theoretical_loss": 3.38529906972076,
+      "tokens_seen": 2611806208
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021060022468303644,
+      "loss": 1.2639,
+      "theoretical_loss": 3.385285750025268,
+      "tokens_seen": 2611937280
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021056010271224525,
+      "loss": 1.2786,
+      "theoretical_loss": 3.38527243118531,
+      "tokens_seen": 2612068352
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021051998074145403,
+      "loss": 1.297,
+      "theoretical_loss": 3.3852591132007888,
+      "tokens_seen": 2612199424
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002104798587706628,
+      "loss": 1.3534,
+      "theoretical_loss": 3.3852457960716063,
+      "tokens_seen": 2612330496
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002104397367998716,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3852324797976645,
+      "tokens_seen": 2612461568
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002103996148290804,
+      "loss": 1.2769,
+      "theoretical_loss": 3.385219164378866,
+      "tokens_seen": 2612592640
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002103594928582892,
+      "loss": 1.2068,
+      "theoretical_loss": 3.3852058498151125,
+      "tokens_seen": 2612723712
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.000210319370887498,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3851925361063064,
+      "tokens_seen": 2612854784
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002102792489167068,
+      "loss": 1.2228,
+      "theoretical_loss": 3.38517922325235,
+      "tokens_seen": 2612985856
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002102391269459156,
+      "loss": 1.1907,
+      "theoretical_loss": 3.3851659112531456,
+      "tokens_seen": 2613116928
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021019900497512437,
+      "loss": 1.2188,
+      "theoretical_loss": 3.385152600108595,
+      "tokens_seen": 2613248000
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 877266,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2339770793914795,
+      "objective/train/theoretical_loss": 3.385139289818601,
+      "objective/train/tokens_used": 983838176,
+      "theoretical_loss": 3.385139289818601,
+      "tokens_seen": 2613379072
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021015888300433317,
+      "loss": 1.2721,
+      "theoretical_loss": 3.385139289818601,
+      "tokens_seen": 2613379072
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021011876103354195,
+      "loss": 1.4044,
+      "theoretical_loss": 3.385125980383066,
+      "tokens_seen": 2613510144
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021007863906275076,
+      "loss": 1.2599,
+      "theoretical_loss": 3.3851126718018913,
+      "tokens_seen": 2613641216
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021003851709195957,
+      "loss": 1.2695,
+      "theoretical_loss": 3.3850993640749802,
+      "tokens_seen": 2613772288
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020999839512116837,
+      "loss": 1.167,
+      "theoretical_loss": 3.3850860572022348,
+      "tokens_seen": 2613903360
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020995827315037715,
+      "loss": 1.278,
+      "theoretical_loss": 3.385072751183557,
+      "tokens_seen": 2614034432
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020991815117958596,
+      "loss": 1.2591,
+      "theoretical_loss": 3.3850594460188503,
+      "tokens_seen": 2614165504
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020987802920879474,
+      "loss": 1.27,
+      "theoretical_loss": 3.385046141708016,
+      "tokens_seen": 2614296576
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020983790723800352,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3850328382509565,
+      "tokens_seen": 2614427648
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020979778526721232,
+      "loss": 1.2303,
+      "theoretical_loss": 3.385019535647575,
+      "tokens_seen": 2614558720
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020975766329642113,
+      "loss": 1.2518,
+      "theoretical_loss": 3.385006233897773,
+      "tokens_seen": 2614689792
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020971754132562994,
+      "loss": 1.2422,
+      "theoretical_loss": 3.3849929330014534,
+      "tokens_seen": 2614820864
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020967741935483871,
+      "loss": 1.2797,
+      "theoretical_loss": 3.384979632958519,
+      "tokens_seen": 2614951936
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020963729738404752,
+      "loss": 1.3495,
+      "theoretical_loss": 3.3849663337688716,
+      "tokens_seen": 2615083008
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002095971754132563,
+      "loss": 1.2292,
+      "theoretical_loss": 3.384953035432414,
+      "tokens_seen": 2615214080
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020955705344246508,
+      "loss": 1.3251,
+      "theoretical_loss": 3.384939737949049,
+      "tokens_seen": 2615345152
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020951693147167389,
+      "loss": 1.3069,
+      "theoretical_loss": 3.3849264413186786,
+      "tokens_seen": 2615476224
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002094768095008827,
+      "loss": 1.211,
+      "theoretical_loss": 3.384913145541206,
+      "tokens_seen": 2615607296
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002094366875300915,
+      "loss": 1.2684,
+      "theoretical_loss": 3.384899850616533,
+      "tokens_seen": 2615738368
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020939656555930028,
+      "loss": 1.285,
+      "theoretical_loss": 3.3848865565445627,
+      "tokens_seen": 2615869440
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020935644358850908,
+      "loss": 1.2096,
+      "theoretical_loss": 3.3848732633251974,
+      "tokens_seen": 2616000512
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020931632161771786,
+      "loss": 1.2122,
+      "theoretical_loss": 3.38485997095834,
+      "tokens_seen": 2616131584
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020927619964692664,
+      "loss": 1.2515,
+      "theoretical_loss": 3.3848466794438927,
+      "tokens_seen": 2616262656
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020923607767613545,
+      "loss": 1.2467,
+      "theoretical_loss": 3.3848333887817583,
+      "tokens_seen": 2616393728
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020919595570534426,
+      "loss": 1.2798,
+      "theoretical_loss": 3.3848200989718396,
+      "tokens_seen": 2616524800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020915583373455303,
+      "loss": 1.1744,
+      "theoretical_loss": 3.384806810014039,
+      "tokens_seen": 2616655872
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020911571176376184,
+      "loss": 1.2835,
+      "theoretical_loss": 3.38479352190826,
+      "tokens_seen": 2616786944
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020907558979297065,
+      "loss": 1.246,
+      "theoretical_loss": 3.384780234654404,
+      "tokens_seen": 2616918016
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020903546782217945,
+      "loss": 1.2348,
+      "theoretical_loss": 3.384766948252375,
+      "tokens_seen": 2617049088
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002089953458513882,
+      "loss": 1.2977,
+      "theoretical_loss": 3.3847536627020745,
+      "tokens_seen": 2617180160
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.000208955223880597,
+      "loss": 1.1707,
+      "theoretical_loss": 3.384740378003406,
+      "tokens_seen": 2617311232
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020891510190980582,
+      "loss": 1.2631,
+      "theoretical_loss": 3.384727094156273,
+      "tokens_seen": 2617442304
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002088749799390146,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3847138111605766,
+      "tokens_seen": 2617573376
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002088348579682234,
+      "loss": 1.2398,
+      "theoretical_loss": 3.38470052901622,
+      "tokens_seen": 2617704448
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002087947359974322,
+      "loss": 1.2522,
+      "theoretical_loss": 3.3846872477231074,
+      "tokens_seen": 2617835520
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020875461402664102,
+      "loss": 1.3749,
+      "theoretical_loss": 3.38467396728114,
+      "tokens_seen": 2617966592
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020871449205584977,
+      "loss": 1.2909,
+      "theoretical_loss": 3.3846606876902214,
+      "tokens_seen": 2618097664
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020867437008505857,
+      "loss": 1.3024,
+      "theoretical_loss": 3.3846474089502543,
+      "tokens_seen": 2618228736
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020863424811426738,
+      "loss": 1.2651,
+      "theoretical_loss": 3.384634131061142,
+      "tokens_seen": 2618359808
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020859412614347616,
+      "loss": 1.2186,
+      "theoretical_loss": 3.3846208540227867,
+      "tokens_seen": 2618490880
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020855400417268497,
+      "loss": 1.2156,
+      "theoretical_loss": 3.3846075778350917,
+      "tokens_seen": 2618621952
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020851388220189377,
+      "loss": 1.2362,
+      "theoretical_loss": 3.38459430249796,
+      "tokens_seen": 2618753024
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020847376023110258,
+      "loss": 1.2501,
+      "theoretical_loss": 3.384581028011294,
+      "tokens_seen": 2618884096
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020843363826031136,
+      "loss": 1.2766,
+      "theoretical_loss": 3.3845677543749972,
+      "tokens_seen": 2619015168
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020839351628952014,
+      "loss": 1.2837,
+      "theoretical_loss": 3.3845544815889728,
+      "tokens_seen": 2619146240
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020835339431872894,
+      "loss": 1.2876,
+      "theoretical_loss": 3.384541209653123,
+      "tokens_seen": 2619277312
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020831327234793772,
+      "loss": 1.2461,
+      "theoretical_loss": 3.3845279385673512,
+      "tokens_seen": 2619408384
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020827315037714653,
+      "loss": 1.2674,
+      "theoretical_loss": 3.3845146683315606,
+      "tokens_seen": 2619539456
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020823302840635534,
+      "loss": 1.2966,
+      "theoretical_loss": 3.384501398945654,
+      "tokens_seen": 2619670528
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020819290643556411,
+      "loss": 1.1967,
+      "theoretical_loss": 3.3844881304095344,
+      "tokens_seen": 2619801600
+    },
+    {
+      "epoch": 0.59,
+      "objective/train/docs_used": 879580,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1803162097930908,
+      "objective/train/theoretical_loss": 3.3844748627231054,
+      "objective/train/tokens_used": 990391776,
+      "theoretical_loss": 3.3844748627231054,
+      "tokens_seen": 2619932672
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020815278446477292,
+      "loss": 1.2845,
+      "theoretical_loss": 3.3844748627231054,
+      "tokens_seen": 2619932672
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002081126624939817,
+      "loss": 1.2145,
+      "theoretical_loss": 3.384461595886269,
+      "tokens_seen": 2620063744
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002080725405231905,
+      "loss": 1.338,
+      "theoretical_loss": 3.3844483298989294,
+      "tokens_seen": 2620194816
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020803241855239929,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3844350647609893,
+      "tokens_seen": 2620325888
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002079922965816081,
+      "loss": 1.2661,
+      "theoretical_loss": 3.3844218004723516,
+      "tokens_seen": 2620456960
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002079521746108169,
+      "loss": 1.2024,
+      "theoretical_loss": 3.38440853703292,
+      "tokens_seen": 2620588032
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020791205264002568,
+      "loss": 1.2665,
+      "theoretical_loss": 3.384395274442597,
+      "tokens_seen": 2620719104
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020787193066923448,
+      "loss": 1.3434,
+      "theoretical_loss": 3.3843820127012862,
+      "tokens_seen": 2620850176
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020783180869844326,
+      "loss": 1.1934,
+      "theoretical_loss": 3.3843687518088905,
+      "tokens_seen": 2620981248
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020779168672765207,
+      "loss": 1.3267,
+      "theoretical_loss": 3.384355491765313,
+      "tokens_seen": 2621112320
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020775156475686085,
+      "loss": 1.242,
+      "theoretical_loss": 3.384342232570458,
+      "tokens_seen": 2621243392
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020771144278606965,
+      "loss": 1.2232,
+      "theoretical_loss": 3.3843289742242275,
+      "tokens_seen": 2621374464
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020767132081527846,
+      "loss": 1.27,
+      "theoretical_loss": 3.3843157167265256,
+      "tokens_seen": 2621505536
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020763119884448724,
+      "loss": 1.2875,
+      "theoretical_loss": 3.384302460077255,
+      "tokens_seen": 2621636608
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020759107687369605,
+      "loss": 1.3051,
+      "theoretical_loss": 3.384289204276319,
+      "tokens_seen": 2621767680
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020755095490290485,
+      "loss": 1.2955,
+      "theoretical_loss": 3.384275949323621,
+      "tokens_seen": 2621898752
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020751083293211363,
+      "loss": 1.3243,
+      "theoretical_loss": 3.3842626952190646,
+      "tokens_seen": 2622029824
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002074707109613224,
+      "loss": 1.2559,
+      "theoretical_loss": 3.3842494419625533,
+      "tokens_seen": 2622160896
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020743058899053122,
+      "loss": 1.3316,
+      "theoretical_loss": 3.3842361895539894,
+      "tokens_seen": 2622291968
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020739046701974002,
+      "loss": 1.2824,
+      "theoretical_loss": 3.3842229379932776,
+      "tokens_seen": 2622423040
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002073503450489488,
+      "loss": 1.216,
+      "theoretical_loss": 3.38420968728032,
+      "tokens_seen": 2622554112
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002073102230781576,
+      "loss": 1.1911,
+      "theoretical_loss": 3.3841964374150213,
+      "tokens_seen": 2622685184
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020727010110736642,
+      "loss": 1.1735,
+      "theoretical_loss": 3.3841831883972837,
+      "tokens_seen": 2622816256
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020722997913657517,
+      "loss": 1.2916,
+      "theoretical_loss": 3.384169940227012,
+      "tokens_seen": 2622947328
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020718985716578397,
+      "loss": 1.2234,
+      "theoretical_loss": 3.384156692904108,
+      "tokens_seen": 2623078400
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020714973519499278,
+      "loss": 1.2745,
+      "theoretical_loss": 3.3841434464284763,
+      "tokens_seen": 2623209472
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002071096132242016,
+      "loss": 1.2672,
+      "theoretical_loss": 3.38413020080002,
+      "tokens_seen": 2623340544
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020706949125341037,
+      "loss": 1.2662,
+      "theoretical_loss": 3.3841169560186426,
+      "tokens_seen": 2623471616
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020702936928261917,
+      "loss": 1.2674,
+      "theoretical_loss": 3.384103712084248,
+      "tokens_seen": 2623602688
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020698924731182798,
+      "loss": 1.1913,
+      "theoretical_loss": 3.3840904689967393,
+      "tokens_seen": 2623733760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020694912534103673,
+      "loss": 1.2965,
+      "theoretical_loss": 3.38407722675602,
+      "tokens_seen": 2623864832
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020690900337024554,
+      "loss": 1.2292,
+      "theoretical_loss": 3.384063985361994,
+      "tokens_seen": 2623995904
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020686888139945434,
+      "loss": 1.2224,
+      "theoretical_loss": 3.384050744814565,
+      "tokens_seen": 2624126976
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020682875942866315,
+      "loss": 1.3525,
+      "theoretical_loss": 3.384037505113636,
+      "tokens_seen": 2624258048
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020678863745787193,
+      "loss": 1.364,
+      "theoretical_loss": 3.384024266259111,
+      "tokens_seen": 2624389120
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020674851548708074,
+      "loss": 1.2349,
+      "theoretical_loss": 3.3840110282508933,
+      "tokens_seen": 2624520192
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020670839351628954,
+      "loss": 1.2161,
+      "theoretical_loss": 3.3839977910888868,
+      "tokens_seen": 2624651264
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020666827154549832,
+      "loss": 1.2099,
+      "theoretical_loss": 3.383984554772995,
+      "tokens_seen": 2624782336
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002066281495747071,
+      "loss": 1.2462,
+      "theoretical_loss": 3.3839713193031216,
+      "tokens_seen": 2624913408
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002065880276039159,
+      "loss": 1.2562,
+      "theoretical_loss": 3.3839580846791706,
+      "tokens_seen": 2625044480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002065479056331247,
+      "loss": 1.2183,
+      "theoretical_loss": 3.3839448509010452,
+      "tokens_seen": 2625175552
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002065077836623335,
+      "loss": 1.3061,
+      "theoretical_loss": 3.38393161796865,
+      "tokens_seen": 2625306624
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002064676616915423,
+      "loss": 1.2766,
+      "theoretical_loss": 3.3839183858818878,
+      "tokens_seen": 2625437696
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002064275397207511,
+      "loss": 1.3271,
+      "theoretical_loss": 3.3839051546406624,
+      "tokens_seen": 2625568768
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020638741774995988,
+      "loss": 1.3105,
+      "theoretical_loss": 3.383891924244878,
+      "tokens_seen": 2625699840
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020634729577916866,
+      "loss": 1.2465,
+      "theoretical_loss": 3.383878694694438,
+      "tokens_seen": 2625830912
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020630717380837747,
+      "loss": 1.3279,
+      "theoretical_loss": 3.3838654659892464,
+      "tokens_seen": 2625961984
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020626705183758625,
+      "loss": 1.2709,
+      "theoretical_loss": 3.383852238129207,
+      "tokens_seen": 2626093056
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020622692986679505,
+      "loss": 1.1896,
+      "theoretical_loss": 3.3838390111142242,
+      "tokens_seen": 2626224128
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020618680789600386,
+      "loss": 1.2186,
+      "theoretical_loss": 3.3838257849442006,
+      "tokens_seen": 2626355200
+    },
+    {
+      "epoch": 0.59,
+      "objective/train/docs_used": 881674,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1921513080596924,
+      "objective/train/theoretical_loss": 3.383812559619041,
+      "objective/train/tokens_used": 996945376,
+      "theoretical_loss": 3.383812559619041,
+      "tokens_seen": 2626486272
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020614668592521267,
+      "loss": 1.233,
+      "theoretical_loss": 3.383812559619041,
+      "tokens_seen": 2626486272
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020610656395442145,
+      "loss": 1.2386,
+      "theoretical_loss": 3.383799335138649,
+      "tokens_seen": 2626617344
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020606644198363025,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3837861115029284,
+      "tokens_seen": 2626748416
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020602632001283903,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3837728887117833,
+      "tokens_seen": 2626879488
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002059861980420478,
+      "loss": 1.3546,
+      "theoretical_loss": 3.3837596667651173,
+      "tokens_seen": 2627010560
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020594607607125662,
+      "loss": 1.2437,
+      "theoretical_loss": 3.383746445662835,
+      "tokens_seen": 2627141632
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020590595410046542,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3837332254048396,
+      "tokens_seen": 2627272704
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020586583212967423,
+      "loss": 1.2769,
+      "theoretical_loss": 3.383720005991035,
+      "tokens_seen": 2627403776
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.000205825710158883,
+      "loss": 1.2506,
+      "theoretical_loss": 3.383706787421326,
+      "tokens_seen": 2627534848
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020578558818809182,
+      "loss": 1.2657,
+      "theoretical_loss": 3.383693569695616,
+      "tokens_seen": 2627665920
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002057454662173006,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3836803528138093,
+      "tokens_seen": 2627796992
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020570534424650937,
+      "loss": 1.2994,
+      "theoretical_loss": 3.38366713677581,
+      "tokens_seen": 2627928064
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020566522227571818,
+      "loss": 1.284,
+      "theoretical_loss": 3.3836539215815216,
+      "tokens_seen": 2628059136
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020562510030492699,
+      "loss": 1.3166,
+      "theoretical_loss": 3.3836407072308488,
+      "tokens_seen": 2628190208
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002055849783341358,
+      "loss": 1.2899,
+      "theoretical_loss": 3.383627493723695,
+      "tokens_seen": 2628321280
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020554485636334457,
+      "loss": 1.2424,
+      "theoretical_loss": 3.383614281059965,
+      "tokens_seen": 2628452352
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020550473439255338,
+      "loss": 1.237,
+      "theoretical_loss": 3.3836010692395626,
+      "tokens_seen": 2628583424
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020546461242176216,
+      "loss": 1.2487,
+      "theoretical_loss": 3.3835878582623917,
+      "tokens_seen": 2628714496
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020542449045097094,
+      "loss": 1.2922,
+      "theoretical_loss": 3.3835746481283566,
+      "tokens_seen": 2628845568
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020538436848017974,
+      "loss": 1.1692,
+      "theoretical_loss": 3.383561438837362,
+      "tokens_seen": 2628976640
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020534424650938855,
+      "loss": 1.2519,
+      "theoretical_loss": 3.383548230389311,
+      "tokens_seen": 2629107712
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020530412453859736,
+      "loss": 1.2689,
+      "theoretical_loss": 3.383535022784109,
+      "tokens_seen": 2629238784
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020526400256780613,
+      "loss": 1.2675,
+      "theoretical_loss": 3.383521816021659,
+      "tokens_seen": 2629369856
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020522388059701494,
+      "loss": 1.2612,
+      "theoretical_loss": 3.383508610101866,
+      "tokens_seen": 2629500928
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020518375862622375,
+      "loss": 1.2844,
+      "theoretical_loss": 3.383495405024634,
+      "tokens_seen": 2629632000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002051436366554325,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3834822007898673,
+      "tokens_seen": 2629763072
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002051035146846413,
+      "loss": 1.3214,
+      "theoretical_loss": 3.38346899739747,
+      "tokens_seen": 2629894144
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002050633927138501,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3834557948473467,
+      "tokens_seen": 2630025216
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002050232707430589,
+      "loss": 1.2437,
+      "theoretical_loss": 3.3834425931394017,
+      "tokens_seen": 2630156288
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002049831487722677,
+      "loss": 1.2955,
+      "theoretical_loss": 3.383429392273539,
+      "tokens_seen": 2630287360
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002049430268014765,
+      "loss": 1.2398,
+      "theoretical_loss": 3.383416192249663,
+      "tokens_seen": 2630418432
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002049029048306853,
+      "loss": 1.2266,
+      "theoretical_loss": 3.383402993067678,
+      "tokens_seen": 2630549504
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020486278285989406,
+      "loss": 1.2048,
+      "theoretical_loss": 3.3833897947274885,
+      "tokens_seen": 2630680576
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020482266088910287,
+      "loss": 1.2826,
+      "theoretical_loss": 3.383376597228999,
+      "tokens_seen": 2630811648
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020478253891831167,
+      "loss": 1.2606,
+      "theoretical_loss": 3.383363400572114,
+      "tokens_seen": 2630942720
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020474241694752045,
+      "loss": 1.2917,
+      "theoretical_loss": 3.383350204756737,
+      "tokens_seen": 2631073792
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020470229497672926,
+      "loss": 1.2546,
+      "theoretical_loss": 3.3833370097827737,
+      "tokens_seen": 2631204864
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020466217300593807,
+      "loss": 1.2953,
+      "theoretical_loss": 3.3833238156501273,
+      "tokens_seen": 2631335936
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020462205103514687,
+      "loss": 1.2476,
+      "theoretical_loss": 3.383310622358703,
+      "tokens_seen": 2631467008
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020458192906435563,
+      "loss": 1.1724,
+      "theoretical_loss": 3.3832974299084055,
+      "tokens_seen": 2631598080
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020454180709356443,
+      "loss": 1.232,
+      "theoretical_loss": 3.383284238299139,
+      "tokens_seen": 2631729152
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020450168512277324,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3832710475308074,
+      "tokens_seen": 2631860224
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020446156315198202,
+      "loss": 1.2881,
+      "theoretical_loss": 3.383257857603316,
+      "tokens_seen": 2631991296
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020442144118119082,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3832446685165687,
+      "tokens_seen": 2632122368
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020438131921039963,
+      "loss": 1.3505,
+      "theoretical_loss": 3.383231480270471,
+      "tokens_seen": 2632253440
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020434119723960844,
+      "loss": 1.2516,
+      "theoretical_loss": 3.3832182928649264,
+      "tokens_seen": 2632384512
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020430107526881721,
+      "loss": 1.2154,
+      "theoretical_loss": 3.3832051062998403,
+      "tokens_seen": 2632515584
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.000204260953298026,
+      "loss": 1.257,
+      "theoretical_loss": 3.3831919205751166,
+      "tokens_seen": 2632646656
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002042208313272348,
+      "loss": 1.2858,
+      "theoretical_loss": 3.3831787356906604,
+      "tokens_seen": 2632777728
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020418070935644358,
+      "loss": 1.262,
+      "theoretical_loss": 3.383165551646376,
+      "tokens_seen": 2632908800
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 883708,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1982176303863525,
+      "objective/train/theoretical_loss": 3.3831523684421687,
+      "objective/train/tokens_used": 1003498976,
+      "theoretical_loss": 3.3831523684421687,
+      "tokens_seen": 2633039872
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020414058738565239,
+      "loss": 1.1832,
+      "theoretical_loss": 3.3831523684421687,
+      "tokens_seen": 2633039872
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002041004654148612,
+      "loss": 1.294,
+      "theoretical_loss": 3.3831391860779423,
+      "tokens_seen": 2633170944
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020406034344406997,
+      "loss": 1.227,
+      "theoretical_loss": 3.383126004553602,
+      "tokens_seen": 2633302016
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020402022147327878,
+      "loss": 1.2284,
+      "theoretical_loss": 3.383112823869052,
+      "tokens_seen": 2633433088
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020398009950248756,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3830996440241976,
+      "tokens_seen": 2633564160
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020393997753169636,
+      "loss": 1.1874,
+      "theoretical_loss": 3.3830864650189434,
+      "tokens_seen": 2633695232
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020389985556090514,
+      "loss": 1.2187,
+      "theoretical_loss": 3.383073286853194,
+      "tokens_seen": 2633826304
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020385973359011395,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3830601095268538,
+      "tokens_seen": 2633957376
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020381961161932276,
+      "loss": 1.0924,
+      "theoretical_loss": 3.383046933039828,
+      "tokens_seen": 2634088448
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020377948964853153,
+      "loss": 1.2048,
+      "theoretical_loss": 3.3830337573920213,
+      "tokens_seen": 2634219520
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020373936767774034,
+      "loss": 1.2261,
+      "theoretical_loss": 3.3830205825833386,
+      "tokens_seen": 2634350592
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020369924570694915,
+      "loss": 1.1583,
+      "theoretical_loss": 3.3830074086136843,
+      "tokens_seen": 2634481664
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020365912373615793,
+      "loss": 1.2415,
+      "theoretical_loss": 3.382994235482964,
+      "tokens_seen": 2634612736
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002036190017653667,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3829810631910817,
+      "tokens_seen": 2634743808
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002035788797945755,
+      "loss": 1.2502,
+      "theoretical_loss": 3.382967891737943,
+      "tokens_seen": 2634874880
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020353875782378432,
+      "loss": 1.2919,
+      "theoretical_loss": 3.382954721123452,
+      "tokens_seen": 2635005952
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002034986358529931,
+      "loss": 1.2194,
+      "theoretical_loss": 3.382941551347514,
+      "tokens_seen": 2635137024
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002034585138822019,
+      "loss": 1.2138,
+      "theoretical_loss": 3.382928382410034,
+      "tokens_seen": 2635268096
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002034183919114107,
+      "loss": 1.2613,
+      "theoretical_loss": 3.3829152143109167,
+      "tokens_seen": 2635399168
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002033782699406195,
+      "loss": 1.2919,
+      "theoretical_loss": 3.3829020470500675,
+      "tokens_seen": 2635530240
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020333814796982827,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3828888806273905,
+      "tokens_seen": 2635661312
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020329802599903707,
+      "loss": 1.3189,
+      "theoretical_loss": 3.3828757150427915,
+      "tokens_seen": 2635792384
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020325790402824588,
+      "loss": 1.2115,
+      "theoretical_loss": 3.382862550296175,
+      "tokens_seen": 2635923456
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020321778205745466,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3828493863874463,
+      "tokens_seen": 2636054528
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020317766008666347,
+      "loss": 1.2086,
+      "theoretical_loss": 3.38283622331651,
+      "tokens_seen": 2636185600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020313753811587227,
+      "loss": 1.24,
+      "theoretical_loss": 3.3828230610832715,
+      "tokens_seen": 2636316672
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020309741614508102,
+      "loss": 1.2427,
+      "theoretical_loss": 3.3828098996876355,
+      "tokens_seen": 2636447744
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020305729417428983,
+      "loss": 1.2261,
+      "theoretical_loss": 3.3827967391295077,
+      "tokens_seen": 2636578816
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020301717220349864,
+      "loss": 1.2384,
+      "theoretical_loss": 3.382783579408793,
+      "tokens_seen": 2636709888
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020297705023270744,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3827704205253957,
+      "tokens_seen": 2636840960
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020293692826191622,
+      "loss": 1.2912,
+      "theoretical_loss": 3.3827572624792213,
+      "tokens_seen": 2636972032
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020289680629112503,
+      "loss": 1.3279,
+      "theoretical_loss": 3.3827441052701754,
+      "tokens_seen": 2637103104
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020285668432033384,
+      "loss": 1.2402,
+      "theoretical_loss": 3.382730948898163,
+      "tokens_seen": 2637234176
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020281656234954261,
+      "loss": 1.2865,
+      "theoretical_loss": 3.382717793363089,
+      "tokens_seen": 2637365248
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002027764403787514,
+      "loss": 1.1847,
+      "theoretical_loss": 3.3827046386648583,
+      "tokens_seen": 2637496320
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002027363184079602,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3826914848033764,
+      "tokens_seen": 2637627392
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.000202696196437169,
+      "loss": 1.2716,
+      "theoretical_loss": 3.382678331778549,
+      "tokens_seen": 2637758464
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020265607446637779,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3826651795902807,
+      "tokens_seen": 2637889536
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002026159524955866,
+      "loss": 1.3006,
+      "theoretical_loss": 3.3826520282384767,
+      "tokens_seen": 2638020608
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002025758305247954,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3826388777230427,
+      "tokens_seen": 2638151680
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020253570855400418,
+      "loss": 1.2398,
+      "theoretical_loss": 3.3826257280438834,
+      "tokens_seen": 2638282752
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020249558658321296,
+      "loss": 1.275,
+      "theoretical_loss": 3.382612579200904,
+      "tokens_seen": 2638413824
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020245546461242176,
+      "loss": 1.3036,
+      "theoretical_loss": 3.382599431194011,
+      "tokens_seen": 2638544896
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020241534264163057,
+      "loss": 1.149,
+      "theoretical_loss": 3.382586284023108,
+      "tokens_seen": 2638675968
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020237522067083935,
+      "loss": 1.1585,
+      "theoretical_loss": 3.3825731376881016,
+      "tokens_seen": 2638807040
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020233509870004815,
+      "loss": 1.2187,
+      "theoretical_loss": 3.382559992188897,
+      "tokens_seen": 2638938112
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020229497672925696,
+      "loss": 1.1434,
+      "theoretical_loss": 3.382546847525399,
+      "tokens_seen": 2639069184
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020225485475846574,
+      "loss": 1.2615,
+      "theoretical_loss": 3.382533703697513,
+      "tokens_seen": 2639200256
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020221473278767452,
+      "loss": 1.1889,
+      "theoretical_loss": 3.382520560705145,
+      "tokens_seen": 2639331328
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020217461081688333,
+      "loss": 1.2473,
+      "theoretical_loss": 3.3825074185482,
+      "tokens_seen": 2639462400
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 885915,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2280280590057373,
+      "objective/train/theoretical_loss": 3.382494277226583,
+      "objective/train/tokens_used": 1010052576,
+      "theoretical_loss": 3.382494277226583,
+      "tokens_seen": 2639593472
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002021344888460921,
+      "loss": 1.1849,
+      "theoretical_loss": 3.382494277226583,
+      "tokens_seen": 2639593472
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002020943668753009,
+      "loss": 1.253,
+      "theoretical_loss": 3.3824811367402,
+      "tokens_seen": 2639724544
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020205424490450972,
+      "loss": 1.227,
+      "theoretical_loss": 3.3824679970889564,
+      "tokens_seen": 2639855616
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020201412293371852,
+      "loss": 1.2254,
+      "theoretical_loss": 3.382454858272758,
+      "tokens_seen": 2639986688
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002019740009629273,
+      "loss": 1.3304,
+      "theoretical_loss": 3.382441720291509,
+      "tokens_seen": 2640117760
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002019338789921361,
+      "loss": 1.269,
+      "theoretical_loss": 3.382428583145116,
+      "tokens_seen": 2640248832
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002018937570213449,
+      "loss": 1.2455,
+      "theoretical_loss": 3.382415446833485,
+      "tokens_seen": 2640379904
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020185363505055367,
+      "loss": 1.2078,
+      "theoretical_loss": 3.38240231135652,
+      "tokens_seen": 2640510976
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020181351307976247,
+      "loss": 1.1819,
+      "theoretical_loss": 3.3823891767141276,
+      "tokens_seen": 2640642048
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020177339110897128,
+      "loss": 1.1448,
+      "theoretical_loss": 3.3823760429062126,
+      "tokens_seen": 2640773120
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002017332691381801,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3823629099326817,
+      "tokens_seen": 2640904192
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020169314716738887,
+      "loss": 1.2326,
+      "theoretical_loss": 3.3823497777934395,
+      "tokens_seen": 2641035264
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020165302519659767,
+      "loss": 1.1645,
+      "theoretical_loss": 3.3823366464883917,
+      "tokens_seen": 2641166336
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020161290322580645,
+      "loss": 1.3129,
+      "theoretical_loss": 3.3823235160174443,
+      "tokens_seen": 2641297408
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020157278125501523,
+      "loss": 1.2856,
+      "theoretical_loss": 3.382310386380503,
+      "tokens_seen": 2641428480
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020153265928422404,
+      "loss": 1.2089,
+      "theoretical_loss": 3.382297257577473,
+      "tokens_seen": 2641559552
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020149253731343284,
+      "loss": 1.3261,
+      "theoretical_loss": 3.38228412960826,
+      "tokens_seen": 2641690624
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020145241534264165,
+      "loss": 1.2434,
+      "theoretical_loss": 3.3822710024727702,
+      "tokens_seen": 2641821696
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020141229337185043,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3822578761709083,
+      "tokens_seen": 2641952768
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020137217140105924,
+      "loss": 1.247,
+      "theoretical_loss": 3.3822447507025815,
+      "tokens_seen": 2642083840
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020133204943026804,
+      "loss": 1.2176,
+      "theoretical_loss": 3.382231626067694,
+      "tokens_seen": 2642214912
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002012919274594768,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3822185022661526,
+      "tokens_seen": 2642345984
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002012518054886856,
+      "loss": 1.2631,
+      "theoretical_loss": 3.3822053792978624,
+      "tokens_seen": 2642477056
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002012116835178944,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3821922571627296,
+      "tokens_seen": 2642608128
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002011715615471032,
+      "loss": 1.2375,
+      "theoretical_loss": 3.3821791358606594,
+      "tokens_seen": 2642739200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.000201131439576312,
+      "loss": 1.3068,
+      "theoretical_loss": 3.382166015391559,
+      "tokens_seen": 2642870272
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002010913176055208,
+      "loss": 1.2711,
+      "theoretical_loss": 3.382152895755332,
+      "tokens_seen": 2643001344
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002010511956347296,
+      "loss": 1.1446,
+      "theoretical_loss": 3.3821397769518864,
+      "tokens_seen": 2643132416
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020101107366393836,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3821266589811265,
+      "tokens_seen": 2643263488
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020097095169314716,
+      "loss": 1.2712,
+      "theoretical_loss": 3.382113541842959,
+      "tokens_seen": 2643394560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020093082972235597,
+      "loss": 1.228,
+      "theoretical_loss": 3.3821004255372893,
+      "tokens_seen": 2643525632
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020089070775156475,
+      "loss": 1.1591,
+      "theoretical_loss": 3.382087310064024,
+      "tokens_seen": 2643656704
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020085058578077355,
+      "loss": 1.2218,
+      "theoretical_loss": 3.382074195423068,
+      "tokens_seen": 2643787776
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020081046380998236,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3820610816143284,
+      "tokens_seen": 2643918848
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020077034183919117,
+      "loss": 1.312,
+      "theoretical_loss": 3.38204796863771,
+      "tokens_seen": 2644049920
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020073021986839992,
+      "loss": 1.2032,
+      "theoretical_loss": 3.382034856493119,
+      "tokens_seen": 2644180992
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020069009789760873,
+      "loss": 1.2096,
+      "theoretical_loss": 3.382021745180462,
+      "tokens_seen": 2644312064
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020064997592681753,
+      "loss": 1.3223,
+      "theoretical_loss": 3.3820086346996447,
+      "tokens_seen": 2644443136
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002006098539560263,
+      "loss": 1.2065,
+      "theoretical_loss": 3.3819955250505727,
+      "tokens_seen": 2644574208
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020056973198523512,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3819824162331527,
+      "tokens_seen": 2644705280
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020052961001444392,
+      "loss": 1.2648,
+      "theoretical_loss": 3.3819693082472897,
+      "tokens_seen": 2644836352
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020048948804365273,
+      "loss": 1.2402,
+      "theoretical_loss": 3.381956201092891,
+      "tokens_seen": 2644967424
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002004493660728615,
+      "loss": 1.2319,
+      "theoretical_loss": 3.381943094769862,
+      "tokens_seen": 2645098496
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002004092441020703,
+      "loss": 1.2679,
+      "theoretical_loss": 3.3819299892781083,
+      "tokens_seen": 2645229568
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002003691221312791,
+      "loss": 1.2979,
+      "theoretical_loss": 3.381916884617537,
+      "tokens_seen": 2645360640
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020032900016048787,
+      "loss": 1.2472,
+      "theoretical_loss": 3.381903780788053,
+      "tokens_seen": 2645491712
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020028887818969668,
+      "loss": 1.2335,
+      "theoretical_loss": 3.3818906777895634,
+      "tokens_seen": 2645622784
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020024875621890549,
+      "loss": 1.1961,
+      "theoretical_loss": 3.3818775756219748,
+      "tokens_seen": 2645753856
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002002086342481143,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3818644742851918,
+      "tokens_seen": 2645884928
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020016851227732307,
+      "loss": 1.1997,
+      "theoretical_loss": 3.3818513737791216,
+      "tokens_seen": 2646016000
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 888056,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.206704020500183,
+      "objective/train/theoretical_loss": 3.3818382741036705,
+      "objective/train/tokens_used": 1016606176,
+      "theoretical_loss": 3.3818382741036705,
+      "tokens_seen": 2646147072
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020012839030653185,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3818382741036705,
+      "tokens_seen": 2646147072
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020008826833574066,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3818251752587436,
+      "tokens_seen": 2646278144
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020004814636494944,
+      "loss": 1.2358,
+      "theoretical_loss": 3.3818120772442484,
+      "tokens_seen": 2646409216
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020000802439415824,
+      "loss": 1.2097,
+      "theoretical_loss": 3.3817989800600907,
+      "tokens_seen": 2646540288
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00019996790242336705,
+      "loss": 1.2942,
+      "theoretical_loss": 3.3817858837061765,
+      "tokens_seen": 2646671360
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00019992778045257583,
+      "loss": 1.1748,
+      "theoretical_loss": 3.381772788182412,
+      "tokens_seen": 2646802432
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00019988765848178463,
+      "loss": 1.2415,
+      "theoretical_loss": 3.381759693488704,
+      "tokens_seen": 2646933504
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00019984753651099341,
+      "loss": 1.2563,
+      "theoretical_loss": 3.3817465996249583,
+      "tokens_seen": 2647064576
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00019980741454020222,
+      "loss": 1.3309,
+      "theoretical_loss": 3.381733506591081,
+      "tokens_seen": 2647195648
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.000199767292569411,
+      "loss": 1.2571,
+      "theoretical_loss": 3.3817204143869795,
+      "tokens_seen": 2647326720
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001997271705986198,
+      "loss": 1.251,
+      "theoretical_loss": 3.3817073230125585,
+      "tokens_seen": 2647457792
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001996870486278286,
+      "loss": 1.2624,
+      "theoretical_loss": 3.381694232467726,
+      "tokens_seen": 2647588864
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001996469266570374,
+      "loss": 1.2542,
+      "theoretical_loss": 3.381681142752387,
+      "tokens_seen": 2647719936
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001996068046862462,
+      "loss": 1.2747,
+      "theoretical_loss": 3.3816680538664494,
+      "tokens_seen": 2647851008
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.000199566682715455,
+      "loss": 1.3079,
+      "theoretical_loss": 3.381654965809818,
+      "tokens_seen": 2647982080
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00019952656074466378,
+      "loss": 1.269,
+      "theoretical_loss": 3.3816418785824003,
+      "tokens_seen": 2648113152
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019948643877387256,
+      "loss": 1.2513,
+      "theoretical_loss": 3.381628792184102,
+      "tokens_seen": 2648244224
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019944631680308137,
+      "loss": 1.2928,
+      "theoretical_loss": 3.38161570661483,
+      "tokens_seen": 2648375296
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019940619483229017,
+      "loss": 1.3194,
+      "theoretical_loss": 3.3816026218744906,
+      "tokens_seen": 2648506368
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019936607286149895,
+      "loss": 1.266,
+      "theoretical_loss": 3.38158953796299,
+      "tokens_seen": 2648637440
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019932595089070776,
+      "loss": 1.2454,
+      "theoretical_loss": 3.3815764548802356,
+      "tokens_seen": 2648768512
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019928582891991657,
+      "loss": 1.2826,
+      "theoretical_loss": 3.381563372626133,
+      "tokens_seen": 2648899584
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019924570694912535,
+      "loss": 1.2507,
+      "theoretical_loss": 3.381550291200589,
+      "tokens_seen": 2649030656
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019920558497833413,
+      "loss": 1.2499,
+      "theoretical_loss": 3.38153721060351,
+      "tokens_seen": 2649161728
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019916546300754293,
+      "loss": 1.3056,
+      "theoretical_loss": 3.3815241308348027,
+      "tokens_seen": 2649292800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019912534103675174,
+      "loss": 1.229,
+      "theoretical_loss": 3.381511051894374,
+      "tokens_seen": 2649423872
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019908521906596052,
+      "loss": 1.2555,
+      "theoretical_loss": 3.3814979737821296,
+      "tokens_seen": 2649554944
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019904509709516932,
+      "loss": 1.1843,
+      "theoretical_loss": 3.3814848964979767,
+      "tokens_seen": 2649686016
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019900497512437813,
+      "loss": 1.237,
+      "theoretical_loss": 3.3814718200418223,
+      "tokens_seen": 2649817088
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001989648531535869,
+      "loss": 1.2861,
+      "theoretical_loss": 3.381458744413572,
+      "tokens_seen": 2649948160
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001989247311827957,
+      "loss": 1.2041,
+      "theoretical_loss": 3.381445669613133,
+      "tokens_seen": 2650079232
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001988846092120045,
+      "loss": 1.2086,
+      "theoretical_loss": 3.381432595640412,
+      "tokens_seen": 2650210304
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001988444872412133,
+      "loss": 1.2678,
+      "theoretical_loss": 3.3814195224953156,
+      "tokens_seen": 2650341376
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019880436527042208,
+      "loss": 1.3099,
+      "theoretical_loss": 3.3814064501777508,
+      "tokens_seen": 2650472448
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019876424329963089,
+      "loss": 1.2582,
+      "theoretical_loss": 3.3813933786876236,
+      "tokens_seen": 2650603520
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001987241213288397,
+      "loss": 1.2279,
+      "theoretical_loss": 3.3813803080248412,
+      "tokens_seen": 2650734592
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019868399935804847,
+      "loss": 1.2398,
+      "theoretical_loss": 3.38136723818931,
+      "tokens_seen": 2650865664
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019864387738725725,
+      "loss": 1.1593,
+      "theoretical_loss": 3.3813541691809372,
+      "tokens_seen": 2650996736
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019860375541646606,
+      "loss": 1.2349,
+      "theoretical_loss": 3.3813411009996295,
+      "tokens_seen": 2651127808
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019856363344567486,
+      "loss": 1.1958,
+      "theoretical_loss": 3.381328033645293,
+      "tokens_seen": 2651258880
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019852351147488364,
+      "loss": 1.2404,
+      "theoretical_loss": 3.3813149671178353,
+      "tokens_seen": 2651389952
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019848338950409245,
+      "loss": 1.2526,
+      "theoretical_loss": 3.381301901417163,
+      "tokens_seen": 2651521024
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019844326753330126,
+      "loss": 1.2044,
+      "theoretical_loss": 3.381288836543183,
+      "tokens_seen": 2651652096
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019840314556251003,
+      "loss": 1.2359,
+      "theoretical_loss": 3.381275772495801,
+      "tokens_seen": 2651783168
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019836302359171881,
+      "loss": 1.2491,
+      "theoretical_loss": 3.3812627092749254,
+      "tokens_seen": 2651914240
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019832290162092762,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3812496468804625,
+      "tokens_seen": 2652045312
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019828277965013643,
+      "loss": 1.2099,
+      "theoretical_loss": 3.381236585312319,
+      "tokens_seen": 2652176384
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001982426576793452,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3812235245704017,
+      "tokens_seen": 2652307456
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.000198202535708554,
+      "loss": 1.3,
+      "theoretical_loss": 3.3812104646546177,
+      "tokens_seen": 2652438528
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019816241373776282,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3811974055648744,
+      "tokens_seen": 2652569600
+    },
+    {
+      "epoch": 0.61,
+      "objective/train/docs_used": 890384,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2129204273223877,
+      "objective/train/theoretical_loss": 3.381184347301078,
+      "objective/train/tokens_used": 1023159776,
+      "theoretical_loss": 3.381184347301078,
+      "tokens_seen": 2652700672
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001981222917669716,
+      "loss": 1.227,
+      "theoretical_loss": 3.381184347301078,
+      "tokens_seen": 2652700672
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001980821697961804,
+      "loss": 1.251,
+      "theoretical_loss": 3.381171289863136,
+      "tokens_seen": 2652831744
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019804204782538918,
+      "loss": 1.2486,
+      "theoretical_loss": 3.3811582332509547,
+      "tokens_seen": 2652962816
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019800192585459796,
+      "loss": 1.2302,
+      "theoretical_loss": 3.381145177464442,
+      "tokens_seen": 2653093888
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019796180388380677,
+      "loss": 1.2836,
+      "theoretical_loss": 3.381132122503504,
+      "tokens_seen": 2653224960
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019792168191301557,
+      "loss": 1.2553,
+      "theoretical_loss": 3.381119068368048,
+      "tokens_seen": 2653356032
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019788155994222438,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3811060150579815,
+      "tokens_seen": 2653487104
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019784143797143316,
+      "loss": 1.2699,
+      "theoretical_loss": 3.381092962573211,
+      "tokens_seen": 2653618176
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019780131600064197,
+      "loss": 1.2422,
+      "theoretical_loss": 3.381079910913644,
+      "tokens_seen": 2653749248
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019776119402985075,
+      "loss": 1.231,
+      "theoretical_loss": 3.381066860079187,
+      "tokens_seen": 2653880320
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019772107205905952,
+      "loss": 1.2235,
+      "theoretical_loss": 3.381053810069748,
+      "tokens_seen": 2654011392
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019768095008826833,
+      "loss": 1.203,
+      "theoretical_loss": 3.3810407608852326,
+      "tokens_seen": 2654142464
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019764082811747714,
+      "loss": 1.2362,
+      "theoretical_loss": 3.3810277125255492,
+      "tokens_seen": 2654273536
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019760070614668594,
+      "loss": 1.2873,
+      "theoretical_loss": 3.381014664990605,
+      "tokens_seen": 2654404608
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019756058417589472,
+      "loss": 1.2182,
+      "theoretical_loss": 3.3810016182803064,
+      "tokens_seen": 2654535680
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019752046220510353,
+      "loss": 1.203,
+      "theoretical_loss": 3.380988572394561,
+      "tokens_seen": 2654666752
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001974803402343123,
+      "loss": 1.204,
+      "theoretical_loss": 3.3809755273332756,
+      "tokens_seen": 2654797824
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001974402182635211,
+      "loss": 1.2448,
+      "theoretical_loss": 3.380962483096358,
+      "tokens_seen": 2654928896
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001974000962927299,
+      "loss": 1.2785,
+      "theoretical_loss": 3.380949439683715,
+      "tokens_seen": 2655059968
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001973599743219387,
+      "loss": 1.2486,
+      "theoretical_loss": 3.3809363970952537,
+      "tokens_seen": 2655191040
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001973198523511475,
+      "loss": 1.2248,
+      "theoretical_loss": 3.3809233553308817,
+      "tokens_seen": 2655322112
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019727973038035629,
+      "loss": 1.2528,
+      "theoretical_loss": 3.3809103143905057,
+      "tokens_seen": 2655453184
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001972396084095651,
+      "loss": 1.2222,
+      "theoretical_loss": 3.380897274274034,
+      "tokens_seen": 2655584256
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001971994864387739,
+      "loss": 1.2313,
+      "theoretical_loss": 3.380884234981373,
+      "tokens_seen": 2655715328
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019715936446798265,
+      "loss": 1.1772,
+      "theoretical_loss": 3.3808711965124303,
+      "tokens_seen": 2655846400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019711924249719146,
+      "loss": 1.2039,
+      "theoretical_loss": 3.380858158867113,
+      "tokens_seen": 2655977472
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019707912052640026,
+      "loss": 1.1644,
+      "theoretical_loss": 3.380845122045329,
+      "tokens_seen": 2656108544
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019703899855560907,
+      "loss": 1.2746,
+      "theoretical_loss": 3.380832086046985,
+      "tokens_seen": 2656239616
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019699887658481785,
+      "loss": 1.3019,
+      "theoretical_loss": 3.380819050871988,
+      "tokens_seen": 2656370688
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019695875461402665,
+      "loss": 1.2855,
+      "theoretical_loss": 3.3808060165202467,
+      "tokens_seen": 2656501760
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019691863264323546,
+      "loss": 1.1993,
+      "theoretical_loss": 3.380792982991668,
+      "tokens_seen": 2656632832
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001968785106724442,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3807799502861586,
+      "tokens_seen": 2656763904
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019683838870165302,
+      "loss": 1.1508,
+      "theoretical_loss": 3.3807669184036264,
+      "tokens_seen": 2656894976
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019679826673086183,
+      "loss": 1.1665,
+      "theoretical_loss": 3.380753887343979,
+      "tokens_seen": 2657026048
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001967581447600706,
+      "loss": 1.2095,
+      "theoretical_loss": 3.380740857107124,
+      "tokens_seen": 2657157120
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001967180227892794,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3807278276929678,
+      "tokens_seen": 2657288192
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019667790081848822,
+      "loss": 1.1989,
+      "theoretical_loss": 3.380714799101419,
+      "tokens_seen": 2657419264
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019663777884769702,
+      "loss": 1.1881,
+      "theoretical_loss": 3.380701771332385,
+      "tokens_seen": 2657550336
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001965976568769058,
+      "loss": 1.2026,
+      "theoretical_loss": 3.3806887443857727,
+      "tokens_seen": 2657681408
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019655753490611458,
+      "loss": 1.1466,
+      "theoretical_loss": 3.38067571826149,
+      "tokens_seen": 2657812480
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001965174129353234,
+      "loss": 1.2368,
+      "theoretical_loss": 3.3806626929594445,
+      "tokens_seen": 2657943552
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019647729096453217,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3806496684795437,
+      "tokens_seen": 2658074624
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019643716899374097,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3806366448216947,
+      "tokens_seen": 2658205696
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019639704702294978,
+      "loss": 1.1201,
+      "theoretical_loss": 3.380623621985806,
+      "tokens_seen": 2658336768
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001963569250521586,
+      "loss": 1.2062,
+      "theoretical_loss": 3.3806105999717846,
+      "tokens_seen": 2658467840
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019631680308136737,
+      "loss": 1.1935,
+      "theoretical_loss": 3.380597578779538,
+      "tokens_seen": 2658598912
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019627668111057615,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3805845584089744,
+      "tokens_seen": 2658729984
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019623655913978495,
+      "loss": 1.2236,
+      "theoretical_loss": 3.380571538860001,
+      "tokens_seen": 2658861056
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019619643716899373,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3805585201325252,
+      "tokens_seen": 2658992128
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019615631519820254,
+      "loss": 1.2412,
+      "theoretical_loss": 3.3805455022264557,
+      "tokens_seen": 2659123200
+    },
+    {
+      "epoch": 0.61,
+      "objective/train/docs_used": 892248,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1585750579833984,
+      "objective/train/theoretical_loss": 3.380532485141699,
+      "objective/train/tokens_used": 1029713376,
+      "theoretical_loss": 3.380532485141699,
+      "tokens_seen": 2659254272
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019611619322741134,
+      "loss": 1.1321,
+      "theoretical_loss": 3.380532485141699,
+      "tokens_seen": 2659254272
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019607607125662015,
+      "loss": 1.2546,
+      "theoretical_loss": 3.3805194688781635,
+      "tokens_seen": 2659385344
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019603594928582893,
+      "loss": 1.2429,
+      "theoretical_loss": 3.380506453435757,
+      "tokens_seen": 2659516416
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001959958273150377,
+      "loss": 1.2543,
+      "theoretical_loss": 3.3804934388143866,
+      "tokens_seen": 2659647488
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019595570534424651,
+      "loss": 1.1348,
+      "theoretical_loss": 3.3804804250139604,
+      "tokens_seen": 2659778560
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001959155833734553,
+      "loss": 1.2454,
+      "theoretical_loss": 3.3804674120343865,
+      "tokens_seen": 2659909632
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001958754614026641,
+      "loss": 1.2932,
+      "theoretical_loss": 3.3804543998755725,
+      "tokens_seen": 2660040704
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001958353394318729,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3804413885374256,
+      "tokens_seen": 2660171776
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019579521746108169,
+      "loss": 1.2024,
+      "theoretical_loss": 3.3804283780198543,
+      "tokens_seen": 2660302848
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001957550954902905,
+      "loss": 1.1991,
+      "theoretical_loss": 3.3804153683227662,
+      "tokens_seen": 2660433920
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001957149735194993,
+      "loss": 1.2233,
+      "theoretical_loss": 3.3804023594460695,
+      "tokens_seen": 2660564992
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019567485154870808,
+      "loss": 1.0834,
+      "theoretical_loss": 3.3803893513896712,
+      "tokens_seen": 2660696064
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019563472957791686,
+      "loss": 1.2017,
+      "theoretical_loss": 3.38037634415348,
+      "tokens_seen": 2660827136
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019559460760712566,
+      "loss": 1.2304,
+      "theoretical_loss": 3.380363337737403,
+      "tokens_seen": 2660958208
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019555448563633447,
+      "loss": 1.214,
+      "theoretical_loss": 3.380350332141349,
+      "tokens_seen": 2661089280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019551436366554325,
+      "loss": 1.1495,
+      "theoretical_loss": 3.380337327365225,
+      "tokens_seen": 2661220352
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019547424169475205,
+      "loss": 1.1746,
+      "theoretical_loss": 3.38032432340894,
+      "tokens_seen": 2661351424
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019543411972396086,
+      "loss": 1.2106,
+      "theoretical_loss": 3.380311320272401,
+      "tokens_seen": 2661482496
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019539399775316964,
+      "loss": 1.1585,
+      "theoretical_loss": 3.3802983179555164,
+      "tokens_seen": 2661613568
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019535387578237842,
+      "loss": 1.1689,
+      "theoretical_loss": 3.3802853164581936,
+      "tokens_seen": 2661744640
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019531375381158723,
+      "loss": 1.1838,
+      "theoretical_loss": 3.3802723157803416,
+      "tokens_seen": 2661875712
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019527363184079603,
+      "loss": 1.1959,
+      "theoretical_loss": 3.380259315921868,
+      "tokens_seen": 2662006784
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001952335098700048,
+      "loss": 1.109,
+      "theoretical_loss": 3.3802463168826797,
+      "tokens_seen": 2662137856
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019519338789921362,
+      "loss": 1.1942,
+      "theoretical_loss": 3.380233318662687,
+      "tokens_seen": 2662268928
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019515326592842242,
+      "loss": 1.1816,
+      "theoretical_loss": 3.3802203212617954,
+      "tokens_seen": 2662400000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001951131439576312,
+      "loss": 1.1307,
+      "theoretical_loss": 3.3802073246799154,
+      "tokens_seen": 2662531072
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019507302198683998,
+      "loss": 1.1667,
+      "theoretical_loss": 3.380194328916953,
+      "tokens_seen": 2662662144
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001950329000160488,
+      "loss": 1.2428,
+      "theoretical_loss": 3.3801813339728173,
+      "tokens_seen": 2662793216
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001949927780452576,
+      "loss": 1.1881,
+      "theoretical_loss": 3.3801683398474167,
+      "tokens_seen": 2662924288
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019495265607446637,
+      "loss": 1.2331,
+      "theoretical_loss": 3.3801553465406586,
+      "tokens_seen": 2663055360
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019491253410367518,
+      "loss": 1.1337,
+      "theoretical_loss": 3.3801423540524516,
+      "tokens_seen": 2663186432
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019487241213288399,
+      "loss": 1.278,
+      "theoretical_loss": 3.3801293623827036,
+      "tokens_seen": 2663317504
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019483229016209277,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3801163715313227,
+      "tokens_seen": 2663448576
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019479216819130154,
+      "loss": 1.1687,
+      "theoretical_loss": 3.3801033814982175,
+      "tokens_seen": 2663579648
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019475204622051035,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3800903922832957,
+      "tokens_seen": 2663710720
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019471192424971916,
+      "loss": 1.2307,
+      "theoretical_loss": 3.380077403886466,
+      "tokens_seen": 2663841792
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019467180227892794,
+      "loss": 1.1608,
+      "theoretical_loss": 3.3800644163076363,
+      "tokens_seen": 2663972864
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019463168030813674,
+      "loss": 1.2254,
+      "theoretical_loss": 3.380051429546715,
+      "tokens_seen": 2664103936
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019459155833734555,
+      "loss": 1.1453,
+      "theoretical_loss": 3.38003844360361,
+      "tokens_seen": 2664235008
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019455143636655433,
+      "loss": 1.2373,
+      "theoretical_loss": 3.38002545847823,
+      "tokens_seen": 2664366080
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001945113143957631,
+      "loss": 1.1885,
+      "theoretical_loss": 3.380012474170483,
+      "tokens_seen": 2664497152
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019447119242497191,
+      "loss": 1.1462,
+      "theoretical_loss": 3.3799994906802775,
+      "tokens_seen": 2664628224
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019443107045418072,
+      "loss": 1.1606,
+      "theoretical_loss": 3.379986508007522,
+      "tokens_seen": 2664759296
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001943909484833895,
+      "loss": 1.2602,
+      "theoretical_loss": 3.379973526152124,
+      "tokens_seen": 2664890368
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001943508265125983,
+      "loss": 1.1335,
+      "theoretical_loss": 3.3799605451139927,
+      "tokens_seen": 2665021440
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001943107045418071,
+      "loss": 1.0967,
+      "theoretical_loss": 3.379947564893036,
+      "tokens_seen": 2665152512
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001942705825710159,
+      "loss": 1.0996,
+      "theoretical_loss": 3.3799345854891625,
+      "tokens_seen": 2665283584
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019423046060022467,
+      "loss": 1.2227,
+      "theoretical_loss": 3.379921606902281,
+      "tokens_seen": 2665414656
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019419033862943348,
+      "loss": 1.2729,
+      "theoretical_loss": 3.379908629132299,
+      "tokens_seen": 2665545728
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019415021665864228,
+      "loss": 1.1705,
+      "theoretical_loss": 3.3798956521791252,
+      "tokens_seen": 2665676800
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 894393,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.292906403541565,
+      "objective/train/theoretical_loss": 3.3798826760426683,
+      "objective/train/tokens_used": 1036266976,
+      "theoretical_loss": 3.3798826760426683,
+      "tokens_seen": 2665807872
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019411009468785106,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3798826760426683,
+      "tokens_seen": 2665807872
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019406997271705987,
+      "loss": 1.183,
+      "theoretical_loss": 3.379869700722837,
+      "tokens_seen": 2665938944
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019402985074626867,
+      "loss": 1.2013,
+      "theoretical_loss": 3.379856726219539,
+      "tokens_seen": 2666070016
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019398972877547745,
+      "loss": 1.2504,
+      "theoretical_loss": 3.379843752532683,
+      "tokens_seen": 2666201088
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019394960680468626,
+      "loss": 1.1384,
+      "theoretical_loss": 3.379830779662178,
+      "tokens_seen": 2666332160
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019390948483389504,
+      "loss": 1.2355,
+      "theoretical_loss": 3.3798178076079317,
+      "tokens_seen": 2666463232
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019386936286310382,
+      "loss": 1.1638,
+      "theoretical_loss": 3.3798048363698534,
+      "tokens_seen": 2666594304
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019382924089231263,
+      "loss": 1.2436,
+      "theoretical_loss": 3.3797918659478516,
+      "tokens_seen": 2666725376
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019378911892152143,
+      "loss": 1.1453,
+      "theoretical_loss": 3.379778896341834,
+      "tokens_seen": 2666856448
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019374899695073024,
+      "loss": 1.1299,
+      "theoretical_loss": 3.37976592755171,
+      "tokens_seen": 2666987520
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019370887497993902,
+      "loss": 1.1859,
+      "theoretical_loss": 3.379752959577388,
+      "tokens_seen": 2667118592
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019366875300914782,
+      "loss": 1.1235,
+      "theoretical_loss": 3.3797399924187763,
+      "tokens_seen": 2667249664
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001936286310383566,
+      "loss": 1.1506,
+      "theoretical_loss": 3.379727026075784,
+      "tokens_seen": 2667380736
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019358850906756538,
+      "loss": 1.1616,
+      "theoretical_loss": 3.379714060548319,
+      "tokens_seen": 2667511808
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001935483870967742,
+      "loss": 1.2039,
+      "theoretical_loss": 3.3797010958362907,
+      "tokens_seen": 2667642880
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.000193508265125983,
+      "loss": 1.2288,
+      "theoretical_loss": 3.3796881319396075,
+      "tokens_seen": 2667773952
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001934681431551918,
+      "loss": 1.2692,
+      "theoretical_loss": 3.379675168858178,
+      "tokens_seen": 2667905024
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019342802118440058,
+      "loss": 1.1465,
+      "theoretical_loss": 3.3796622065919104,
+      "tokens_seen": 2668036096
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019338789921360939,
+      "loss": 1.2592,
+      "theoretical_loss": 3.379649245140714,
+      "tokens_seen": 2668167168
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001933477772428182,
+      "loss": 1.2042,
+      "theoretical_loss": 3.379636284504498,
+      "tokens_seen": 2668298240
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019330765527202694,
+      "loss": 1.2265,
+      "theoretical_loss": 3.3796233246831697,
+      "tokens_seen": 2668429312
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019326753330123575,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3796103656766388,
+      "tokens_seen": 2668560384
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019322741133044456,
+      "loss": 1.2151,
+      "theoretical_loss": 3.3795974074848143,
+      "tokens_seen": 2668691456
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019318728935965336,
+      "loss": 1.1191,
+      "theoretical_loss": 3.3795844501076044,
+      "tokens_seen": 2668822528
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019314716738886214,
+      "loss": 1.2645,
+      "theoretical_loss": 3.379571493544918,
+      "tokens_seen": 2668953600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019310704541807095,
+      "loss": 1.1399,
+      "theoretical_loss": 3.379558537796664,
+      "tokens_seen": 2669084672
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019306692344727976,
+      "loss": 1.2181,
+      "theoretical_loss": 3.379545582862751,
+      "tokens_seen": 2669215744
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001930268014764885,
+      "loss": 1.1859,
+      "theoretical_loss": 3.3795326287430885,
+      "tokens_seen": 2669346816
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019298667950569731,
+      "loss": 1.1711,
+      "theoretical_loss": 3.3795196754375842,
+      "tokens_seen": 2669477888
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019294655753490612,
+      "loss": 1.1461,
+      "theoretical_loss": 3.3795067229461484,
+      "tokens_seen": 2669608960
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019290643556411493,
+      "loss": 1.1927,
+      "theoretical_loss": 3.3794937712686886,
+      "tokens_seen": 2669740032
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001928663135933237,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3794808204051145,
+      "tokens_seen": 2669871104
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001928261916225325,
+      "loss": 1.1483,
+      "theoretical_loss": 3.3794678703553345,
+      "tokens_seen": 2670002176
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019278606965174132,
+      "loss": 1.2093,
+      "theoretical_loss": 3.379454921119258,
+      "tokens_seen": 2670133248
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019274594768095007,
+      "loss": 1.2068,
+      "theoretical_loss": 3.3794419726967937,
+      "tokens_seen": 2670264320
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019270582571015888,
+      "loss": 1.2229,
+      "theoretical_loss": 3.3794290250878505,
+      "tokens_seen": 2670395392
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019266570373936768,
+      "loss": 1.2265,
+      "theoretical_loss": 3.3794160782923375,
+      "tokens_seen": 2670526464
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019262558176857646,
+      "loss": 1.2202,
+      "theoretical_loss": 3.3794031323101636,
+      "tokens_seen": 2670657536
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019258545979778527,
+      "loss": 1.2055,
+      "theoretical_loss": 3.379390187141238,
+      "tokens_seen": 2670788608
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019254533782699407,
+      "loss": 1.18,
+      "theoretical_loss": 3.3793772427854694,
+      "tokens_seen": 2670919680
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019250521585620288,
+      "loss": 1.1979,
+      "theoretical_loss": 3.379364299242767,
+      "tokens_seen": 2671050752
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019246509388541166,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3793513565130393,
+      "tokens_seen": 2671181824
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019242497191462044,
+      "loss": 1.1494,
+      "theoretical_loss": 3.379338414596196,
+      "tokens_seen": 2671312896
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019238484994382925,
+      "loss": 1.2044,
+      "theoretical_loss": 3.3793254734921465,
+      "tokens_seen": 2671443968
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019234472797303802,
+      "loss": 1.167,
+      "theoretical_loss": 3.3793125332007987,
+      "tokens_seen": 2671575040
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019230460600224683,
+      "loss": 1.1799,
+      "theoretical_loss": 3.3792995937220627,
+      "tokens_seen": 2671706112
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019226448403145564,
+      "loss": 1.221,
+      "theoretical_loss": 3.3792866550558474,
+      "tokens_seen": 2671837184
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019222436206066444,
+      "loss": 1.1495,
+      "theoretical_loss": 3.3792737172020613,
+      "tokens_seen": 2671968256
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019218424008987322,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3792607801606143,
+      "tokens_seen": 2672099328
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.000192144118119082,
+      "loss": 1.173,
+      "theoretical_loss": 3.3792478439314153,
+      "tokens_seen": 2672230400
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 896553,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2242774963378906,
+      "objective/train/theoretical_loss": 3.3792349085143734,
+      "objective/train/tokens_used": 1042820576,
+      "theoretical_loss": 3.3792349085143734,
+      "tokens_seen": 2672361472
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001921039961482908,
+      "loss": 1.2022,
+      "theoretical_loss": 3.3792349085143734,
+      "tokens_seen": 2672361472
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001920638741774996,
+      "loss": 1.2262,
+      "theoretical_loss": 3.379221973909398,
+      "tokens_seen": 2672492544
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001920237522067084,
+      "loss": 1.2097,
+      "theoretical_loss": 3.3792090401163977,
+      "tokens_seen": 2672623616
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001919836302359172,
+      "loss": 1.2349,
+      "theoretical_loss": 3.3791961071352823,
+      "tokens_seen": 2672754688
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.000191943508265126,
+      "loss": 1.267,
+      "theoretical_loss": 3.379183174965961,
+      "tokens_seen": 2672885760
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019190338629433479,
+      "loss": 1.1514,
+      "theoretical_loss": 3.3791702436083426,
+      "tokens_seen": 2673016832
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019186326432354356,
+      "loss": 1.2198,
+      "theoretical_loss": 3.379157313062337,
+      "tokens_seen": 2673147904
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019182314235275237,
+      "loss": 1.2107,
+      "theoretical_loss": 3.3791443833278527,
+      "tokens_seen": 2673278976
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019178302038196115,
+      "loss": 1.1818,
+      "theoretical_loss": 3.3791314544047997,
+      "tokens_seen": 2673410048
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019174289841116996,
+      "loss": 1.2014,
+      "theoretical_loss": 3.379118526293087,
+      "tokens_seen": 2673541120
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019170277644037876,
+      "loss": 1.1821,
+      "theoretical_loss": 3.3791055989926235,
+      "tokens_seen": 2673672192
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019166265446958754,
+      "loss": 1.2024,
+      "theoretical_loss": 3.3790926725033197,
+      "tokens_seen": 2673803264
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019162253249879635,
+      "loss": 1.1391,
+      "theoretical_loss": 3.3790797468250835,
+      "tokens_seen": 2673934336
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019158241052800515,
+      "loss": 1.2097,
+      "theoretical_loss": 3.3790668219578253,
+      "tokens_seen": 2674065408
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019154228855721393,
+      "loss": 1.1676,
+      "theoretical_loss": 3.379053897901454,
+      "tokens_seen": 2674196480
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001915021665864227,
+      "loss": 1.2013,
+      "theoretical_loss": 3.379040974655879,
+      "tokens_seen": 2674327552
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019146204461563152,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3790280522210097,
+      "tokens_seen": 2674458624
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019142192264484033,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3790151305967555,
+      "tokens_seen": 2674589696
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001913818006740491,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3790022097830263,
+      "tokens_seen": 2674720768
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001913416787032579,
+      "loss": 1.1349,
+      "theoretical_loss": 3.378989289779731,
+      "tokens_seen": 2674851840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019130155673246672,
+      "loss": 1.2545,
+      "theoretical_loss": 3.3789763705867792,
+      "tokens_seen": 2674982912
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001912614347616755,
+      "loss": 1.1787,
+      "theoretical_loss": 3.3789634522040806,
+      "tokens_seen": 2675113984
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019122131279088428,
+      "loss": 1.2208,
+      "theoretical_loss": 3.378950534631544,
+      "tokens_seen": 2675245056
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019118119082009308,
+      "loss": 1.2302,
+      "theoretical_loss": 3.37893761786908,
+      "tokens_seen": 2675376128
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001911410688493019,
+      "loss": 1.1434,
+      "theoretical_loss": 3.3789247019165973,
+      "tokens_seen": 2675507200
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019110094687851067,
+      "loss": 1.2484,
+      "theoretical_loss": 3.3789117867740055,
+      "tokens_seen": 2675638272
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019106082490771947,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3788988724412143,
+      "tokens_seen": 2675769344
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019102070293692828,
+      "loss": 1.2203,
+      "theoretical_loss": 3.3788859589181333,
+      "tokens_seen": 2675900416
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001909805809661371,
+      "loss": 1.1978,
+      "theoretical_loss": 3.378873046204672,
+      "tokens_seen": 2676031488
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019094045899534584,
+      "loss": 1.284,
+      "theoretical_loss": 3.3788601343007403,
+      "tokens_seen": 2676162560
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019090033702455465,
+      "loss": 1.2053,
+      "theoretical_loss": 3.378847223206247,
+      "tokens_seen": 2676293632
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019086021505376345,
+      "loss": 1.2341,
+      "theoretical_loss": 3.3788343129211027,
+      "tokens_seen": 2676424704
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019082009308297223,
+      "loss": 1.2874,
+      "theoretical_loss": 3.378821403445216,
+      "tokens_seen": 2676555776
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019077997111218104,
+      "loss": 1.2125,
+      "theoretical_loss": 3.3788084947784975,
+      "tokens_seen": 2676686848
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019073984914138984,
+      "loss": 1.1397,
+      "theoretical_loss": 3.3787955869208566,
+      "tokens_seen": 2676817920
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019069972717059862,
+      "loss": 1.1856,
+      "theoretical_loss": 3.3787826798722027,
+      "tokens_seen": 2676948992
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001906596051998074,
+      "loss": 1.2301,
+      "theoretical_loss": 3.378769773632446,
+      "tokens_seen": 2677080064
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001906194832290162,
+      "loss": 1.2533,
+      "theoretical_loss": 3.3787568682014952,
+      "tokens_seen": 2677211136
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019057936125822501,
+      "loss": 1.2683,
+      "theoretical_loss": 3.378743963579261,
+      "tokens_seen": 2677342208
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001905392392874338,
+      "loss": 1.2564,
+      "theoretical_loss": 3.3787310597656526,
+      "tokens_seen": 2677473280
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001904991173166426,
+      "loss": 1.1685,
+      "theoretical_loss": 3.3787181567605797,
+      "tokens_seen": 2677604352
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001904589953458514,
+      "loss": 1.2309,
+      "theoretical_loss": 3.378705254563953,
+      "tokens_seen": 2677735424
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019041887337506019,
+      "loss": 1.2034,
+      "theoretical_loss": 3.3786923531756816,
+      "tokens_seen": 2677866496
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019037875140426896,
+      "loss": 1.264,
+      "theoretical_loss": 3.3786794525956747,
+      "tokens_seen": 2677997568
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019033862943347777,
+      "loss": 1.2284,
+      "theoretical_loss": 3.3786665528238427,
+      "tokens_seen": 2678128640
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019029850746268658,
+      "loss": 1.2624,
+      "theoretical_loss": 3.3786536538600958,
+      "tokens_seen": 2678259712
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019025838549189536,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3786407557043434,
+      "tokens_seen": 2678390784
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019021826352110416,
+      "loss": 1.2142,
+      "theoretical_loss": 3.378627858356495,
+      "tokens_seen": 2678521856
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019017814155031297,
+      "loss": 1.26,
+      "theoretical_loss": 3.3786149618164614,
+      "tokens_seen": 2678652928
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019013801957952175,
+      "loss": 1.2001,
+      "theoretical_loss": 3.3786020660841514,
+      "tokens_seen": 2678784000
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 898681,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3476333618164062,
+      "objective/train/theoretical_loss": 3.3785891711594758,
+      "objective/train/tokens_used": 1049374176,
+      "theoretical_loss": 3.3785891711594758,
+      "tokens_seen": 2678915072
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019009789760873055,
+      "loss": 1.2703,
+      "theoretical_loss": 3.3785891711594758,
+      "tokens_seen": 2678915072
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019005777563793933,
+      "loss": 1.1836,
+      "theoretical_loss": 3.378576277042344,
+      "tokens_seen": 2679046144
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019001765366714814,
+      "loss": 1.3098,
+      "theoretical_loss": 3.3785633837326663,
+      "tokens_seen": 2679177216
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018997753169635692,
+      "loss": 1.2548,
+      "theoretical_loss": 3.378550491230352,
+      "tokens_seen": 2679308288
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018993740972556573,
+      "loss": 1.2421,
+      "theoretical_loss": 3.3785375995353117,
+      "tokens_seen": 2679439360
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018989728775477453,
+      "loss": 1.2288,
+      "theoretical_loss": 3.378524708647455,
+      "tokens_seen": 2679570432
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001898571657839833,
+      "loss": 1.1946,
+      "theoretical_loss": 3.3785118185666922,
+      "tokens_seen": 2679701504
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018981704381319212,
+      "loss": 1.1539,
+      "theoretical_loss": 3.378498929292933,
+      "tokens_seen": 2679832576
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001897769218424009,
+      "loss": 1.2349,
+      "theoretical_loss": 3.378486040826087,
+      "tokens_seen": 2679963648
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018973679987160968,
+      "loss": 1.1983,
+      "theoretical_loss": 3.3784731531660652,
+      "tokens_seen": 2680094720
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018969667790081848,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3784602663127776,
+      "tokens_seen": 2680225792
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001896565559300273,
+      "loss": 1.1939,
+      "theoretical_loss": 3.3784473802661332,
+      "tokens_seen": 2680356864
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001896164339592361,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3784344950260428,
+      "tokens_seen": 2680487936
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018957631198844487,
+      "loss": 1.2422,
+      "theoretical_loss": 3.378421610592416,
+      "tokens_seen": 2680619008
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018953619001765368,
+      "loss": 1.2628,
+      "theoretical_loss": 3.378408726965164,
+      "tokens_seen": 2680750080
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018949606804686246,
+      "loss": 1.2418,
+      "theoretical_loss": 3.3783958441441957,
+      "tokens_seen": 2680881152
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018945594607607124,
+      "loss": 1.181,
+      "theoretical_loss": 3.3783829621294217,
+      "tokens_seen": 2681012224
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018941582410528004,
+      "loss": 1.2383,
+      "theoretical_loss": 3.378370080920752,
+      "tokens_seen": 2681143296
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018937570213448885,
+      "loss": 1.2473,
+      "theoretical_loss": 3.3783572005180975,
+      "tokens_seen": 2681274368
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018933558016369766,
+      "loss": 1.1522,
+      "theoretical_loss": 3.378344320921367,
+      "tokens_seen": 2681405440
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018929545819290644,
+      "loss": 1.3158,
+      "theoretical_loss": 3.378331442130472,
+      "tokens_seen": 2681536512
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018925533622211524,
+      "loss": 1.2701,
+      "theoretical_loss": 3.3783185641453217,
+      "tokens_seen": 2681667584
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018921521425132405,
+      "loss": 1.2133,
+      "theoretical_loss": 3.378305686965827,
+      "tokens_seen": 2681798656
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001891750922805328,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3782928105918977,
+      "tokens_seen": 2681929728
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001891349703097416,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3782799350234445,
+      "tokens_seen": 2682060800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018909484833895041,
+      "loss": 1.1618,
+      "theoretical_loss": 3.378267060260377,
+      "tokens_seen": 2682191872
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018905472636815922,
+      "loss": 1.2282,
+      "theoretical_loss": 3.378254186302606,
+      "tokens_seen": 2682322944
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.000189014604397368,
+      "loss": 1.1915,
+      "theoretical_loss": 3.3782413131500415,
+      "tokens_seen": 2682454016
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001889744824265768,
+      "loss": 1.208,
+      "theoretical_loss": 3.378228440802594,
+      "tokens_seen": 2682585088
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001889343604557856,
+      "loss": 1.1983,
+      "theoretical_loss": 3.3782155692601736,
+      "tokens_seen": 2682716160
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018889423848499436,
+      "loss": 1.2781,
+      "theoretical_loss": 3.3782026985226907,
+      "tokens_seen": 2682847232
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018885411651420317,
+      "loss": 1.2902,
+      "theoretical_loss": 3.378189828590056,
+      "tokens_seen": 2682978304
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018881399454341198,
+      "loss": 1.315,
+      "theoretical_loss": 3.3781769594621793,
+      "tokens_seen": 2683109376
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018877387257262076,
+      "loss": 1.2031,
+      "theoretical_loss": 3.378164091138971,
+      "tokens_seen": 2683240448
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018873375060182956,
+      "loss": 1.2224,
+      "theoretical_loss": 3.378151223620342,
+      "tokens_seen": 2683371520
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018869362863103837,
+      "loss": 1.2718,
+      "theoretical_loss": 3.3781383569062022,
+      "tokens_seen": 2683502592
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018865350666024717,
+      "loss": 1.2372,
+      "theoretical_loss": 3.378125490996463,
+      "tokens_seen": 2683633664
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018861338468945595,
+      "loss": 1.1612,
+      "theoretical_loss": 3.378112625891033,
+      "tokens_seen": 2683764736
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018857326271866473,
+      "loss": 1.2047,
+      "theoretical_loss": 3.378099761589824,
+      "tokens_seen": 2683895808
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018853314074787354,
+      "loss": 1.2069,
+      "theoretical_loss": 3.378086898092746,
+      "tokens_seen": 2684026880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018849301877708232,
+      "loss": 1.2366,
+      "theoretical_loss": 3.37807403539971,
+      "tokens_seen": 2684157952
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018845289680629113,
+      "loss": 1.2401,
+      "theoretical_loss": 3.3780611735106256,
+      "tokens_seen": 2684289024
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018841277483549993,
+      "loss": 1.2634,
+      "theoretical_loss": 3.378048312425404,
+      "tokens_seen": 2684420096
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018837265286470874,
+      "loss": 1.1895,
+      "theoretical_loss": 3.3780354521439557,
+      "tokens_seen": 2684551168
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018833253089391752,
+      "loss": 1.1643,
+      "theoretical_loss": 3.3780225926661904,
+      "tokens_seen": 2684682240
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001882924089231263,
+      "loss": 1.2665,
+      "theoretical_loss": 3.3780097339920196,
+      "tokens_seen": 2684813312
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001882522869523351,
+      "loss": 1.1351,
+      "theoretical_loss": 3.3779968761213537,
+      "tokens_seen": 2684944384
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018821216498154388,
+      "loss": 1.2035,
+      "theoretical_loss": 3.377984019054103,
+      "tokens_seen": 2685075456
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001881720430107527,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3779711627901783,
+      "tokens_seen": 2685206528
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001881319210399615,
+      "loss": 1.3052,
+      "theoretical_loss": 3.3779583073294894,
+      "tokens_seen": 2685337600
+    },
+    {
+      "epoch": 0.63,
+      "objective/train/docs_used": 900968,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2355791330337524,
+      "objective/train/theoretical_loss": 3.3779454526719483,
+      "objective/train/tokens_used": 1055927776,
+      "theoretical_loss": 3.3779454526719483,
+      "tokens_seen": 2685468672
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001880917990691703,
+      "loss": 1.2042,
+      "theoretical_loss": 3.3779454526719483,
+      "tokens_seen": 2685468672
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018805167709837908,
+      "loss": 1.2062,
+      "theoretical_loss": 3.3779325988174644,
+      "tokens_seen": 2685599744
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018801155512758786,
+      "loss": 1.2114,
+      "theoretical_loss": 3.3779197457659493,
+      "tokens_seen": 2685730816
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018797143315679667,
+      "loss": 1.1608,
+      "theoretical_loss": 3.377906893517313,
+      "tokens_seen": 2685861888
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018793131118600544,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3778940420714663,
+      "tokens_seen": 2685992960
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018789118921521425,
+      "loss": 1.156,
+      "theoretical_loss": 3.3778811914283198,
+      "tokens_seen": 2686124032
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018785106724442306,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3778683415877846,
+      "tokens_seen": 2686255104
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018781094527363186,
+      "loss": 1.2268,
+      "theoretical_loss": 3.377855492549771,
+      "tokens_seen": 2686386176
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018777082330284064,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3778426443141902,
+      "tokens_seen": 2686517248
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018773070133204945,
+      "loss": 1.1641,
+      "theoretical_loss": 3.3778297968809525,
+      "tokens_seen": 2686648320
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018769057936125823,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3778169502499686,
+      "tokens_seen": 2686779392
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.000187650457390467,
+      "loss": 1.2248,
+      "theoretical_loss": 3.3778041044211493,
+      "tokens_seen": 2686910464
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018761033541967581,
+      "loss": 1.239,
+      "theoretical_loss": 3.377791259394406,
+      "tokens_seen": 2687041536
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018757021344888462,
+      "loss": 1.2587,
+      "theoretical_loss": 3.377778415169649,
+      "tokens_seen": 2687172608
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001875300914780934,
+      "loss": 1.2243,
+      "theoretical_loss": 3.3777655717467887,
+      "tokens_seen": 2687303680
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001874899695073022,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3777527291257368,
+      "tokens_seen": 2687434752
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.000187449847536511,
+      "loss": 1.2895,
+      "theoretical_loss": 3.3777398873064035,
+      "tokens_seen": 2687565824
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001874097255657198,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3777270462887,
+      "tokens_seen": 2687696896
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018736960359492857,
+      "loss": 1.2076,
+      "theoretical_loss": 3.377714206072537,
+      "tokens_seen": 2687827968
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018732948162413738,
+      "loss": 1.2228,
+      "theoretical_loss": 3.3777013666578255,
+      "tokens_seen": 2687959040
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018728935965334618,
+      "loss": 1.2157,
+      "theoretical_loss": 3.3776885280444766,
+      "tokens_seen": 2688090112
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018724923768255496,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3776756902324,
+      "tokens_seen": 2688221184
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018720911571176377,
+      "loss": 1.2695,
+      "theoretical_loss": 3.3776628532215085,
+      "tokens_seen": 2688352256
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018716899374097257,
+      "loss": 1.2148,
+      "theoretical_loss": 3.377650017011712,
+      "tokens_seen": 2688483328
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018712887177018135,
+      "loss": 1.1738,
+      "theoretical_loss": 3.377637181602921,
+      "tokens_seen": 2688614400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018708874979939013,
+      "loss": 1.2653,
+      "theoretical_loss": 3.3776243469950478,
+      "tokens_seen": 2688745472
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018704862782859894,
+      "loss": 1.2438,
+      "theoretical_loss": 3.3776115131880022,
+      "tokens_seen": 2688876544
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018700850585780775,
+      "loss": 1.2005,
+      "theoretical_loss": 3.3775986801816957,
+      "tokens_seen": 2689007616
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018696838388701652,
+      "loss": 1.2218,
+      "theoretical_loss": 3.377585847976039,
+      "tokens_seen": 2689138688
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018692826191622533,
+      "loss": 1.2165,
+      "theoretical_loss": 3.377573016570943,
+      "tokens_seen": 2689269760
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018688813994543414,
+      "loss": 1.217,
+      "theoretical_loss": 3.37756018596632,
+      "tokens_seen": 2689400832
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018684801797464294,
+      "loss": 1.2096,
+      "theoretical_loss": 3.3775473561620797,
+      "tokens_seen": 2689531904
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001868078960038517,
+      "loss": 1.2424,
+      "theoretical_loss": 3.377534527158134,
+      "tokens_seen": 2689662976
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001867677740330605,
+      "loss": 1.2582,
+      "theoretical_loss": 3.377521698954393,
+      "tokens_seen": 2689794048
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001867276520622693,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3775088715507686,
+      "tokens_seen": 2689925120
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001866875300914781,
+      "loss": 1.2261,
+      "theoretical_loss": 3.377496044947172,
+      "tokens_seen": 2690056192
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001866474081206869,
+      "loss": 1.2365,
+      "theoretical_loss": 3.377483219143514,
+      "tokens_seen": 2690187264
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001866072861498957,
+      "loss": 1.2697,
+      "theoretical_loss": 3.3774703941397055,
+      "tokens_seen": 2690318336
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018656716417910448,
+      "loss": 1.1367,
+      "theoretical_loss": 3.377457569935658,
+      "tokens_seen": 2690449408
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018652704220831326,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3774447465312827,
+      "tokens_seen": 2690580480
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018648692023752206,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3774319239264905,
+      "tokens_seen": 2690711552
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018644679826673087,
+      "loss": 1.2584,
+      "theoretical_loss": 3.377419102121193,
+      "tokens_seen": 2690842624
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018640667629593965,
+      "loss": 1.1425,
+      "theoretical_loss": 3.377406281115301,
+      "tokens_seen": 2690973696
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018636655432514846,
+      "loss": 1.2684,
+      "theoretical_loss": 3.377393460908726,
+      "tokens_seen": 2691104768
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018632643235435726,
+      "loss": 1.1545,
+      "theoretical_loss": 3.377380641501379,
+      "tokens_seen": 2691235840
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018628631038356604,
+      "loss": 1.2257,
+      "theoretical_loss": 3.377367822893172,
+      "tokens_seen": 2691366912
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018624618841277485,
+      "loss": 1.247,
+      "theoretical_loss": 3.377355005084015,
+      "tokens_seen": 2691497984
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018620606644198363,
+      "loss": 1.2295,
+      "theoretical_loss": 3.37734218807382,
+      "tokens_seen": 2691629056
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018616594447119243,
+      "loss": 1.239,
+      "theoretical_loss": 3.3773293718624986,
+      "tokens_seen": 2691760128
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001861258225004012,
+      "loss": 1.1679,
+      "theoretical_loss": 3.377316556449962,
+      "tokens_seen": 2691891200
+    },
+    {
+      "epoch": 0.63,
+      "objective/train/docs_used": 903428,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2128208875656128,
+      "objective/train/theoretical_loss": 3.377303741836121,
+      "objective/train/tokens_used": 1062481376,
+      "theoretical_loss": 3.377303741836121,
+      "tokens_seen": 2692022272
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018608570052961002,
+      "loss": 1.2446,
+      "theoretical_loss": 3.377303741836121,
+      "tokens_seen": 2692022272
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018604557855881883,
+      "loss": 1.1708,
+      "theoretical_loss": 3.377290928020887,
+      "tokens_seen": 2692153344
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001860054565880276,
+      "loss": 1.2291,
+      "theoretical_loss": 3.377278115004172,
+      "tokens_seen": 2692284416
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001859653346172364,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3772653027858865,
+      "tokens_seen": 2692415488
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001859252126464452,
+      "loss": 1.1765,
+      "theoretical_loss": 3.3772524913659425,
+      "tokens_seen": 2692546560
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.000185885090675654,
+      "loss": 1.2873,
+      "theoretical_loss": 3.3772396807442515,
+      "tokens_seen": 2692677632
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018584496870486278,
+      "loss": 1.3013,
+      "theoretical_loss": 3.3772268709207243,
+      "tokens_seen": 2692808704
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018580484673407158,
+      "loss": 1.2087,
+      "theoretical_loss": 3.377214061895273,
+      "tokens_seen": 2692939776
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001857647247632804,
+      "loss": 1.1421,
+      "theoretical_loss": 3.377201253667808,
+      "tokens_seen": 2693070848
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018572460279248917,
+      "loss": 1.2437,
+      "theoretical_loss": 3.377188446238242,
+      "tokens_seen": 2693201920
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018568448082169797,
+      "loss": 1.2523,
+      "theoretical_loss": 3.3771756396064863,
+      "tokens_seen": 2693332992
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018564435885090675,
+      "loss": 1.2516,
+      "theoretical_loss": 3.3771628337724513,
+      "tokens_seen": 2693464064
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018560423688011553,
+      "loss": 1.2855,
+      "theoretical_loss": 3.3771500287360494,
+      "tokens_seen": 2693595136
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018556411490932434,
+      "loss": 1.2164,
+      "theoretical_loss": 3.377137224497192,
+      "tokens_seen": 2693726208
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018552399293853315,
+      "loss": 1.1592,
+      "theoretical_loss": 3.3771244210557905,
+      "tokens_seen": 2693857280
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018548387096774195,
+      "loss": 1.3063,
+      "theoretical_loss": 3.377111618411756,
+      "tokens_seen": 2693988352
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018544374899695073,
+      "loss": 1.2797,
+      "theoretical_loss": 3.377098816565001,
+      "tokens_seen": 2694119424
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018540362702615954,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3770860155154367,
+      "tokens_seen": 2694250496
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018536350505536834,
+      "loss": 1.2184,
+      "theoretical_loss": 3.3770732152629743,
+      "tokens_seen": 2694381568
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001853233830845771,
+      "loss": 1.2783,
+      "theoretical_loss": 3.3770604158075255,
+      "tokens_seen": 2694512640
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001852832611137859,
+      "loss": 1.2747,
+      "theoretical_loss": 3.3770476171490023,
+      "tokens_seen": 2694643712
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001852431391429947,
+      "loss": 1.182,
+      "theoretical_loss": 3.3770348192873163,
+      "tokens_seen": 2694774784
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018520301717220351,
+      "loss": 1.1806,
+      "theoretical_loss": 3.3770220222223784,
+      "tokens_seen": 2694905856
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001851628952014123,
+      "loss": 1.2387,
+      "theoretical_loss": 3.377009225954101,
+      "tokens_seen": 2695036928
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001851227732306211,
+      "loss": 1.275,
+      "theoretical_loss": 3.3769964304823956,
+      "tokens_seen": 2695168000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001850826512598299,
+      "loss": 1.1826,
+      "theoretical_loss": 3.3769836358071736,
+      "tokens_seen": 2695299072
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018504252928903866,
+      "loss": 1.2468,
+      "theoretical_loss": 3.376970841928347,
+      "tokens_seen": 2695430144
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018500240731824746,
+      "loss": 1.3002,
+      "theoretical_loss": 3.3769580488458275,
+      "tokens_seen": 2695561216
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018496228534745627,
+      "loss": 1.2598,
+      "theoretical_loss": 3.3769452565595266,
+      "tokens_seen": 2695692288
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018492216337666508,
+      "loss": 1.2606,
+      "theoretical_loss": 3.3769324650693564,
+      "tokens_seen": 2695823360
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018488204140587386,
+      "loss": 1.2117,
+      "theoretical_loss": 3.3769196743752277,
+      "tokens_seen": 2695954432
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018484191943508266,
+      "loss": 1.22,
+      "theoretical_loss": 3.376906884477054,
+      "tokens_seen": 2696085504
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018480179746429147,
+      "loss": 1.2494,
+      "theoretical_loss": 3.376894095374745,
+      "tokens_seen": 2696216576
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018476167549350022,
+      "loss": 1.2466,
+      "theoretical_loss": 3.376881307068214,
+      "tokens_seen": 2696347648
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018472155352270903,
+      "loss": 1.2536,
+      "theoretical_loss": 3.376868519557372,
+      "tokens_seen": 2696478720
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018468143155191783,
+      "loss": 1.252,
+      "theoretical_loss": 3.376855732842132,
+      "tokens_seen": 2696609792
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001846413095811266,
+      "loss": 1.1846,
+      "theoretical_loss": 3.3768429469224044,
+      "tokens_seen": 2696740864
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018460118761033542,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3768301617981016,
+      "tokens_seen": 2696871936
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018456106563954423,
+      "loss": 1.2037,
+      "theoretical_loss": 3.376817377469136,
+      "tokens_seen": 2697003008
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018452094366875303,
+      "loss": 1.2309,
+      "theoretical_loss": 3.376804593935418,
+      "tokens_seen": 2697134080
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001844808216979618,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3767918111968616,
+      "tokens_seen": 2697265152
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001844406997271706,
+      "loss": 1.2413,
+      "theoretical_loss": 3.3767790292533766,
+      "tokens_seen": 2697396224
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001844005777563794,
+      "loss": 1.2796,
+      "theoretical_loss": 3.376766248104876,
+      "tokens_seen": 2697527296
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018436045578558818,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3767534677512723,
+      "tokens_seen": 2697658368
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018432033381479698,
+      "loss": 1.2314,
+      "theoretical_loss": 3.376740688192476,
+      "tokens_seen": 2697789440
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001842802118440058,
+      "loss": 1.2882,
+      "theoretical_loss": 3.3767279094284004,
+      "tokens_seen": 2697920512
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001842400898732146,
+      "loss": 1.2599,
+      "theoretical_loss": 3.3767151314589565,
+      "tokens_seen": 2698051584
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018419996790242337,
+      "loss": 1.2171,
+      "theoretical_loss": 3.3767023542840566,
+      "tokens_seen": 2698182656
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018415984593163215,
+      "loss": 1.1692,
+      "theoretical_loss": 3.376689577903613,
+      "tokens_seen": 2698313728
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018411972396084096,
+      "loss": 1.1717,
+      "theoretical_loss": 3.3766768023175375,
+      "tokens_seen": 2698444800
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 905717,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.269565224647522,
+      "objective/train/theoretical_loss": 3.376664027525742,
+      "objective/train/tokens_used": 1069034976,
+      "theoretical_loss": 3.376664027525742,
+      "tokens_seen": 2698575872
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018407960199004974,
+      "loss": 1.293,
+      "theoretical_loss": 3.376664027525742,
+      "tokens_seen": 2698575872
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018403948001925854,
+      "loss": 1.1846,
+      "theoretical_loss": 3.3766512535281388,
+      "tokens_seen": 2698706944
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018399935804846735,
+      "loss": 1.2821,
+      "theoretical_loss": 3.3766384803246394,
+      "tokens_seen": 2698838016
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018395923607767616,
+      "loss": 1.2373,
+      "theoretical_loss": 3.376625707915157,
+      "tokens_seen": 2698969088
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018391911410688494,
+      "loss": 1.2235,
+      "theoretical_loss": 3.376612936299602,
+      "tokens_seen": 2699100160
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018387899213609374,
+      "loss": 1.2484,
+      "theoretical_loss": 3.376600165477888,
+      "tokens_seen": 2699231232
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018383887016530252,
+      "loss": 1.2828,
+      "theoretical_loss": 3.376587395449927,
+      "tokens_seen": 2699362304
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001837987481945113,
+      "loss": 1.2044,
+      "theoretical_loss": 3.37657462621563,
+      "tokens_seen": 2699493376
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001837586262237201,
+      "loss": 1.1795,
+      "theoretical_loss": 3.3765618577749104,
+      "tokens_seen": 2699624448
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018371850425292891,
+      "loss": 1.264,
+      "theoretical_loss": 3.3765490901276793,
+      "tokens_seen": 2699755520
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018367838228213772,
+      "loss": 1.2385,
+      "theoretical_loss": 3.37653632327385,
+      "tokens_seen": 2699886592
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001836382603113465,
+      "loss": 1.2436,
+      "theoretical_loss": 3.376523557213334,
+      "tokens_seen": 2700017664
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001835981383405553,
+      "loss": 1.1983,
+      "theoretical_loss": 3.376510791946043,
+      "tokens_seen": 2700148736
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018355801636976409,
+      "loss": 1.2011,
+      "theoretical_loss": 3.37649802747189,
+      "tokens_seen": 2700279808
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018351789439897286,
+      "loss": 1.2806,
+      "theoretical_loss": 3.3764852637907867,
+      "tokens_seen": 2700410880
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018347777242818167,
+      "loss": 1.3067,
+      "theoretical_loss": 3.376472500902646,
+      "tokens_seen": 2700541952
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018343765045739048,
+      "loss": 1.2321,
+      "theoretical_loss": 3.37645973880738,
+      "tokens_seen": 2700673024
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018339752848659926,
+      "loss": 1.2173,
+      "theoretical_loss": 3.3764469775049,
+      "tokens_seen": 2700804096
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018335740651580806,
+      "loss": 1.2886,
+      "theoretical_loss": 3.37643421699512,
+      "tokens_seen": 2700935168
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018331728454501687,
+      "loss": 1.2134,
+      "theoretical_loss": 3.3764214572779507,
+      "tokens_seen": 2701066240
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018327716257422565,
+      "loss": 1.2045,
+      "theoretical_loss": 3.376408698353305,
+      "tokens_seen": 2701197312
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018323704060343443,
+      "loss": 1.2293,
+      "theoretical_loss": 3.3763959402210957,
+      "tokens_seen": 2701328384
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018319691863264323,
+      "loss": 1.2387,
+      "theoretical_loss": 3.3763831828812343,
+      "tokens_seen": 2701459456
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018315679666185204,
+      "loss": 1.2485,
+      "theoretical_loss": 3.376370426333634,
+      "tokens_seen": 2701590528
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018311667469106082,
+      "loss": 1.2639,
+      "theoretical_loss": 3.376357670578206,
+      "tokens_seen": 2701721600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018307655272026963,
+      "loss": 1.2264,
+      "theoretical_loss": 3.3763449156148635,
+      "tokens_seen": 2701852672
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018303643074947843,
+      "loss": 1.199,
+      "theoretical_loss": 3.376332161443519,
+      "tokens_seen": 2701983744
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018299630877868724,
+      "loss": 1.2379,
+      "theoretical_loss": 3.376319408064085,
+      "tokens_seen": 2702114816
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.000182956186807896,
+      "loss": 1.3043,
+      "theoretical_loss": 3.376306655476473,
+      "tokens_seen": 2702245888
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001829160648371048,
+      "loss": 1.235,
+      "theoretical_loss": 3.3762939036805966,
+      "tokens_seen": 2702376960
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001828759428663136,
+      "loss": 1.203,
+      "theoretical_loss": 3.376281152676367,
+      "tokens_seen": 2702508032
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018283582089552238,
+      "loss": 1.2794,
+      "theoretical_loss": 3.3762684024636984,
+      "tokens_seen": 2702639104
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001827956989247312,
+      "loss": 1.2922,
+      "theoretical_loss": 3.3762556530425014,
+      "tokens_seen": 2702770176
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018275557695394,
+      "loss": 1.2436,
+      "theoretical_loss": 3.3762429044126896,
+      "tokens_seen": 2702901248
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001827154549831488,
+      "loss": 1.2114,
+      "theoretical_loss": 3.376230156574175,
+      "tokens_seen": 2703032320
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018267533301235755,
+      "loss": 1.1554,
+      "theoretical_loss": 3.3762174095268707,
+      "tokens_seen": 2703163392
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018263521104156636,
+      "loss": 1.1879,
+      "theoretical_loss": 3.3762046632706886,
+      "tokens_seen": 2703294464
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018259508907077517,
+      "loss": 1.3062,
+      "theoretical_loss": 3.3761919178055413,
+      "tokens_seen": 2703425536
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018255496709998394,
+      "loss": 1.2115,
+      "theoretical_loss": 3.376179173131342,
+      "tokens_seen": 2703556608
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018251484512919275,
+      "loss": 1.2462,
+      "theoretical_loss": 3.376166429248003,
+      "tokens_seen": 2703687680
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018247472315840156,
+      "loss": 1.2385,
+      "theoretical_loss": 3.376153686155436,
+      "tokens_seen": 2703818752
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018243460118761034,
+      "loss": 1.2354,
+      "theoretical_loss": 3.376140943853555,
+      "tokens_seen": 2703949824
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018239447921681912,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3761282023422714,
+      "tokens_seen": 2704080896
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018235435724602792,
+      "loss": 1.2229,
+      "theoretical_loss": 3.376115461621499,
+      "tokens_seen": 2704211968
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018231423527523673,
+      "loss": 1.1909,
+      "theoretical_loss": 3.3761027216911494,
+      "tokens_seen": 2704343040
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001822741133044455,
+      "loss": 1.2263,
+      "theoretical_loss": 3.376089982551136,
+      "tokens_seen": 2704474112
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018223399133365431,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3760772442013707,
+      "tokens_seen": 2704605184
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018219386936286312,
+      "loss": 1.2425,
+      "theoretical_loss": 3.376064506641767,
+      "tokens_seen": 2704736256
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001821537473920719,
+      "loss": 1.2194,
+      "theoretical_loss": 3.376051769872237,
+      "tokens_seen": 2704867328
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001821136254212807,
+      "loss": 1.2052,
+      "theoretical_loss": 3.3760390338926936,
+      "tokens_seen": 2704998400
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 908174,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3148037195205688,
+      "objective/train/theoretical_loss": 3.37602629870305,
+      "objective/train/tokens_used": 1075588576,
+      "theoretical_loss": 3.37602629870305,
+      "tokens_seen": 2705129472
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018207350345048948,
+      "loss": 1.2875,
+      "theoretical_loss": 3.37602629870305,
+      "tokens_seen": 2705129472
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001820333814796983,
+      "loss": 1.2649,
+      "theoretical_loss": 3.3760135643032183,
+      "tokens_seen": 2705260544
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018199325950890707,
+      "loss": 1.2,
+      "theoretical_loss": 3.376000830693111,
+      "tokens_seen": 2705391616
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018195313753811588,
+      "loss": 1.2108,
+      "theoretical_loss": 3.375988097872642,
+      "tokens_seen": 2705522688
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018191301556732468,
+      "loss": 1.2782,
+      "theoretical_loss": 3.375975365841723,
+      "tokens_seen": 2705653760
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018187289359653346,
+      "loss": 1.1876,
+      "theoretical_loss": 3.375962634600268,
+      "tokens_seen": 2705784832
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018183277162574227,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3759499041481877,
+      "tokens_seen": 2705915904
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018179264965495105,
+      "loss": 1.1439,
+      "theoretical_loss": 3.375937174485397,
+      "tokens_seen": 2706046976
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018175252768415985,
+      "loss": 1.2009,
+      "theoretical_loss": 3.3759244456118083,
+      "tokens_seen": 2706178048
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018171240571336863,
+      "loss": 1.1428,
+      "theoretical_loss": 3.3759117175273334,
+      "tokens_seen": 2706309120
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018167228374257744,
+      "loss": 1.239,
+      "theoretical_loss": 3.375898990231886,
+      "tokens_seen": 2706440192
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018163216177178625,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3758862637253793,
+      "tokens_seen": 2706571264
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018159203980099502,
+      "loss": 1.2475,
+      "theoretical_loss": 3.375873538007726,
+      "tokens_seen": 2706702336
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018155191783020383,
+      "loss": 1.3047,
+      "theoretical_loss": 3.3758608130788383,
+      "tokens_seen": 2706833408
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018151179585941264,
+      "loss": 1.2401,
+      "theoretical_loss": 3.3758480889386293,
+      "tokens_seen": 2706964480
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001814716738886214,
+      "loss": 1.1511,
+      "theoretical_loss": 3.3758353655870126,
+      "tokens_seen": 2707095552
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001814315519178302,
+      "loss": 1.1786,
+      "theoretical_loss": 3.3758226430239007,
+      "tokens_seen": 2707226624
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.000181391429947039,
+      "loss": 1.1631,
+      "theoretical_loss": 3.375809921249207,
+      "tokens_seen": 2707357696
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001813513079762478,
+      "loss": 1.2859,
+      "theoretical_loss": 3.3757972002628436,
+      "tokens_seen": 2707488768
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001813111860054566,
+      "loss": 1.1938,
+      "theoretical_loss": 3.3757844800647243,
+      "tokens_seen": 2707619840
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001812710640346654,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3757717606547617,
+      "tokens_seen": 2707750912
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001812309420638742,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3757590420328687,
+      "tokens_seen": 2707881984
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018119082009308295,
+      "loss": 1.2331,
+      "theoretical_loss": 3.3757463241989587,
+      "tokens_seen": 2708013056
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018115069812229176,
+      "loss": 1.2787,
+      "theoretical_loss": 3.375733607152945,
+      "tokens_seen": 2708144128
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018111057615150057,
+      "loss": 1.2104,
+      "theoretical_loss": 3.3757208908947396,
+      "tokens_seen": 2708275200
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018107045418070937,
+      "loss": 1.1933,
+      "theoretical_loss": 3.375708175424257,
+      "tokens_seen": 2708406272
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018103033220991815,
+      "loss": 1.2441,
+      "theoretical_loss": 3.375695460741409,
+      "tokens_seen": 2708537344
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018099021023912696,
+      "loss": 1.3017,
+      "theoretical_loss": 3.375682746846109,
+      "tokens_seen": 2708668416
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018095008826833576,
+      "loss": 1.2291,
+      "theoretical_loss": 3.3756700337382703,
+      "tokens_seen": 2708799488
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018090996629754452,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3756573214178065,
+      "tokens_seen": 2708930560
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018086984432675332,
+      "loss": 1.2008,
+      "theoretical_loss": 3.37564460988463,
+      "tokens_seen": 2709061632
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018082972235596213,
+      "loss": 1.3021,
+      "theoretical_loss": 3.3756318991386545,
+      "tokens_seen": 2709192704
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018078960038517093,
+      "loss": 1.2287,
+      "theoretical_loss": 3.375619189179793,
+      "tokens_seen": 2709323776
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001807494784143797,
+      "loss": 1.2394,
+      "theoretical_loss": 3.375606480007958,
+      "tokens_seen": 2709454848
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018070935644358852,
+      "loss": 1.1843,
+      "theoretical_loss": 3.375593771623064,
+      "tokens_seen": 2709585920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018066923447279733,
+      "loss": 1.2386,
+      "theoretical_loss": 3.3755810640250234,
+      "tokens_seen": 2709716992
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001806291125020061,
+      "loss": 1.2658,
+      "theoretical_loss": 3.375568357213749,
+      "tokens_seen": 2709848064
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018058899053121488,
+      "loss": 1.2131,
+      "theoretical_loss": 3.375555651189155,
+      "tokens_seen": 2709979136
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001805488685604237,
+      "loss": 1.2719,
+      "theoretical_loss": 3.375542945951154,
+      "tokens_seen": 2710110208
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018050874658963247,
+      "loss": 1.1935,
+      "theoretical_loss": 3.3755302414996597,
+      "tokens_seen": 2710241280
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018046862461884128,
+      "loss": 1.1502,
+      "theoretical_loss": 3.375517537834585,
+      "tokens_seen": 2710372352
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018042850264805008,
+      "loss": 1.1691,
+      "theoretical_loss": 3.3755048349558434,
+      "tokens_seen": 2710503424
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001803883806772589,
+      "loss": 1.2531,
+      "theoretical_loss": 3.3754921328633483,
+      "tokens_seen": 2710634496
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018034825870646767,
+      "loss": 1.2368,
+      "theoretical_loss": 3.3754794315570127,
+      "tokens_seen": 2710765568
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018030813673567645,
+      "loss": 1.2135,
+      "theoretical_loss": 3.3754667310367505,
+      "tokens_seen": 2710896640
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018026801476488525,
+      "loss": 1.1922,
+      "theoretical_loss": 3.375454031302474,
+      "tokens_seen": 2711027712
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018022789279409403,
+      "loss": 1.1911,
+      "theoretical_loss": 3.3754413323540975,
+      "tokens_seen": 2711158784
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018018777082330284,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3754286341915343,
+      "tokens_seen": 2711289856
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018014764885251165,
+      "loss": 1.2474,
+      "theoretical_loss": 3.375415936814697,
+      "tokens_seen": 2711420928
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018010752688172045,
+      "loss": 1.2656,
+      "theoretical_loss": 3.3754032402235006,
+      "tokens_seen": 2711552000
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 910231,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2440369129180908,
+      "objective/train/theoretical_loss": 3.375390544417857,
+      "objective/train/tokens_used": 1082142176,
+      "theoretical_loss": 3.375390544417857,
+      "tokens_seen": 2711683072
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018006740491092923,
+      "loss": 1.2778,
+      "theoretical_loss": 3.375390544417857,
+      "tokens_seen": 2711683072
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.000180027282940138,
+      "loss": 1.224,
+      "theoretical_loss": 3.3753778493976796,
+      "tokens_seen": 2711814144
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017998716096934682,
+      "loss": 1.1751,
+      "theoretical_loss": 3.375365155162883,
+      "tokens_seen": 2711945216
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001799470389985556,
+      "loss": 1.2328,
+      "theoretical_loss": 3.37535246171338,
+      "tokens_seen": 2712076288
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001799069170277644,
+      "loss": 1.282,
+      "theoretical_loss": 3.375339769049084,
+      "tokens_seen": 2712207360
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001798667950569732,
+      "loss": 1.1925,
+      "theoretical_loss": 3.375327077169908,
+      "tokens_seen": 2712338432
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017982667308618201,
+      "loss": 1.2574,
+      "theoretical_loss": 3.3753143860757673,
+      "tokens_seen": 2712469504
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001797865511153908,
+      "loss": 1.292,
+      "theoretical_loss": 3.3753016957665736,
+      "tokens_seen": 2712600576
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001797464291445996,
+      "loss": 1.2267,
+      "theoretical_loss": 3.375289006242241,
+      "tokens_seen": 2712731648
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017970630717380838,
+      "loss": 1.2189,
+      "theoretical_loss": 3.3752763175026828,
+      "tokens_seen": 2712862720
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017966618520301716,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3752636295478133,
+      "tokens_seen": 2712993792
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017962606323222596,
+      "loss": 1.2063,
+      "theoretical_loss": 3.375250942377545,
+      "tokens_seen": 2713124864
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017958594126143477,
+      "loss": 1.1731,
+      "theoretical_loss": 3.3752382559917926,
+      "tokens_seen": 2713255936
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017954581929064358,
+      "loss": 1.3069,
+      "theoretical_loss": 3.375225570390469,
+      "tokens_seen": 2713387008
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017950569731985236,
+      "loss": 1.1835,
+      "theoretical_loss": 3.3752128855734878,
+      "tokens_seen": 2713518080
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017946557534906116,
+      "loss": 1.1658,
+      "theoretical_loss": 3.375200201540763,
+      "tokens_seen": 2713649152
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017942545337826994,
+      "loss": 1.253,
+      "theoretical_loss": 3.375187518292208,
+      "tokens_seen": 2713780224
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017938533140747872,
+      "loss": 1.1912,
+      "theoretical_loss": 3.3751748358277363,
+      "tokens_seen": 2713911296
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017934520943668753,
+      "loss": 1.1687,
+      "theoretical_loss": 3.375162154147262,
+      "tokens_seen": 2714042368
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017930508746589633,
+      "loss": 1.1822,
+      "theoretical_loss": 3.3751494732506986,
+      "tokens_seen": 2714173440
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001792649654951051,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3751367931379597,
+      "tokens_seen": 2714304512
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017922484352431392,
+      "loss": 1.2166,
+      "theoretical_loss": 3.3751241138089587,
+      "tokens_seen": 2714435584
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017918472155352273,
+      "loss": 1.214,
+      "theoretical_loss": 3.3751114352636096,
+      "tokens_seen": 2714566656
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017914459958273153,
+      "loss": 1.1167,
+      "theoretical_loss": 3.3750987575018265,
+      "tokens_seen": 2714697728
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017910447761194028,
+      "loss": 1.1794,
+      "theoretical_loss": 3.3750860805235225,
+      "tokens_seen": 2714828800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001790643556411491,
+      "loss": 1.2781,
+      "theoretical_loss": 3.375073404328612,
+      "tokens_seen": 2714959872
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001790242336703579,
+      "loss": 1.1353,
+      "theoretical_loss": 3.375060728917008,
+      "tokens_seen": 2715090944
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017898411169956668,
+      "loss": 1.1797,
+      "theoretical_loss": 3.3750480542886248,
+      "tokens_seen": 2715222016
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017894398972877548,
+      "loss": 1.2026,
+      "theoretical_loss": 3.3750353804433764,
+      "tokens_seen": 2715353088
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001789038677579843,
+      "loss": 1.1681,
+      "theoretical_loss": 3.3750227073811763,
+      "tokens_seen": 2715484160
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001788637457871931,
+      "loss": 1.1344,
+      "theoretical_loss": 3.3750100351019383,
+      "tokens_seen": 2715615232
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017882362381640185,
+      "loss": 1.1716,
+      "theoretical_loss": 3.374997363605576,
+      "tokens_seen": 2715746304
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017878350184561065,
+      "loss": 1.1643,
+      "theoretical_loss": 3.3749846928920038,
+      "tokens_seen": 2715877376
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017874337987481946,
+      "loss": 1.269,
+      "theoretical_loss": 3.3749720229611353,
+      "tokens_seen": 2716008448
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017870325790402824,
+      "loss": 1.1946,
+      "theoretical_loss": 3.3749593538128844,
+      "tokens_seen": 2716139520
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017866313593323704,
+      "loss": 1.1617,
+      "theoretical_loss": 3.3749466854471653,
+      "tokens_seen": 2716270592
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017862301396244585,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3749340178638914,
+      "tokens_seen": 2716401664
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017858289199165466,
+      "loss": 1.191,
+      "theoretical_loss": 3.3749213510629765,
+      "tokens_seen": 2716532736
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001785427700208634,
+      "loss": 1.2448,
+      "theoretical_loss": 3.374908685044335,
+      "tokens_seen": 2716663808
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017850264805007222,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3748960198078812,
+      "tokens_seen": 2716794880
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017846252607928102,
+      "loss": 1.2435,
+      "theoretical_loss": 3.374883355353528,
+      "tokens_seen": 2716925952
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001784224041084898,
+      "loss": 1.2053,
+      "theoretical_loss": 3.37487069168119,
+      "tokens_seen": 2717057024
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001783822821376986,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3748580287907815,
+      "tokens_seen": 2717188096
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017834216016690741,
+      "loss": 1.2242,
+      "theoretical_loss": 3.374845366682216,
+      "tokens_seen": 2717319168
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001783020381961162,
+      "loss": 1.2291,
+      "theoretical_loss": 3.3748327053554075,
+      "tokens_seen": 2717450240
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.000178261916225325,
+      "loss": 1.1933,
+      "theoretical_loss": 3.3748200448102703,
+      "tokens_seen": 2717581312
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017822179425453378,
+      "loss": 1.1678,
+      "theoretical_loss": 3.374807385046718,
+      "tokens_seen": 2717712384
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017818167228374259,
+      "loss": 1.1437,
+      "theoretical_loss": 3.374794726064666,
+      "tokens_seen": 2717843456
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017814155031295136,
+      "loss": 1.158,
+      "theoretical_loss": 3.374782067864026,
+      "tokens_seen": 2717974528
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017810142834216017,
+      "loss": 1.2126,
+      "theoretical_loss": 3.3747694104447143,
+      "tokens_seen": 2718105600
+    },
+    {
+      "epoch": 0.65,
+      "objective/train/docs_used": 912432,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1662516593933105,
+      "objective/train/theoretical_loss": 3.3747567538066443,
+      "objective/train/tokens_used": 1088695776,
+      "theoretical_loss": 3.3747567538066443,
+      "tokens_seen": 2718236672
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017806130637136898,
+      "loss": 1.2015,
+      "theoretical_loss": 3.3747567538066443,
+      "tokens_seen": 2718236672
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017802118440057776,
+      "loss": 1.1683,
+      "theoretical_loss": 3.3747440979497294,
+      "tokens_seen": 2718367744
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017798106242978656,
+      "loss": 1.2295,
+      "theoretical_loss": 3.374731442873885,
+      "tokens_seen": 2718498816
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017794094045899534,
+      "loss": 1.245,
+      "theoretical_loss": 3.374718788579024,
+      "tokens_seen": 2718629888
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017790081848820415,
+      "loss": 1.2129,
+      "theoretical_loss": 3.374706135065061,
+      "tokens_seen": 2718760960
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017786069651741293,
+      "loss": 1.2067,
+      "theoretical_loss": 3.3746934823319106,
+      "tokens_seen": 2718892032
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017782057454662173,
+      "loss": 1.1378,
+      "theoretical_loss": 3.3746808303794866,
+      "tokens_seen": 2719023104
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017778045257583054,
+      "loss": 1.2667,
+      "theoretical_loss": 3.3746681792077027,
+      "tokens_seen": 2719154176
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017774033060503932,
+      "loss": 1.1877,
+      "theoretical_loss": 3.3746555288164743,
+      "tokens_seen": 2719285248
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017770020863424813,
+      "loss": 1.262,
+      "theoretical_loss": 3.3746428792057146,
+      "tokens_seen": 2719416320
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001776600866634569,
+      "loss": 1.2201,
+      "theoretical_loss": 3.374630230375338,
+      "tokens_seen": 2719547392
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001776199646926657,
+      "loss": 1.2419,
+      "theoretical_loss": 3.3746175823252598,
+      "tokens_seen": 2719678464
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001775798427218745,
+      "loss": 1.1456,
+      "theoretical_loss": 3.374604935055393,
+      "tokens_seen": 2719809536
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001775397207510833,
+      "loss": 1.1967,
+      "theoretical_loss": 3.3745922885656516,
+      "tokens_seen": 2719940608
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001774995987802921,
+      "loss": 1.2013,
+      "theoretical_loss": 3.3745796428559514,
+      "tokens_seen": 2720071680
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017745947680950088,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3745669979262054,
+      "tokens_seen": 2720202752
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001774193548387097,
+      "loss": 1.2481,
+      "theoretical_loss": 3.3745543537763285,
+      "tokens_seen": 2720333824
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001773792328679185,
+      "loss": 1.1466,
+      "theoretical_loss": 3.374541710406235,
+      "tokens_seen": 2720464896
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017733911089712725,
+      "loss": 1.233,
+      "theoretical_loss": 3.374529067815839,
+      "tokens_seen": 2720595968
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017729898892633605,
+      "loss": 1.1046,
+      "theoretical_loss": 3.374516426005055,
+      "tokens_seen": 2720727040
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017725886695554486,
+      "loss": 1.1733,
+      "theoretical_loss": 3.3745037849737978,
+      "tokens_seen": 2720858112
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017721874498475367,
+      "loss": 1.1564,
+      "theoretical_loss": 3.374491144721981,
+      "tokens_seen": 2720989184
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017717862301396244,
+      "loss": 1.1623,
+      "theoretical_loss": 3.3744785052495194,
+      "tokens_seen": 2721120256
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017713850104317125,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3744658665563274,
+      "tokens_seen": 2721251328
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017709837907238006,
+      "loss": 1.2722,
+      "theoretical_loss": 3.3744532286423192,
+      "tokens_seen": 2721382400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001770582571015888,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3744405915074096,
+      "tokens_seen": 2721513472
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017701813513079762,
+      "loss": 1.3359,
+      "theoretical_loss": 3.3744279551515133,
+      "tokens_seen": 2721644544
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017697801316000642,
+      "loss": 1.148,
+      "theoretical_loss": 3.3744153195745437,
+      "tokens_seen": 2721775616
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017693789118921523,
+      "loss": 1.1985,
+      "theoretical_loss": 3.3744026847764164,
+      "tokens_seen": 2721906688
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.000176897769218424,
+      "loss": 1.2398,
+      "theoretical_loss": 3.3743900507570452,
+      "tokens_seen": 2722037760
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017685764724763281,
+      "loss": 1.218,
+      "theoretical_loss": 3.374377417516345,
+      "tokens_seen": 2722168832
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017681752527684162,
+      "loss": 1.2013,
+      "theoretical_loss": 3.37436478505423,
+      "tokens_seen": 2722299904
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001767774033060504,
+      "loss": 1.1948,
+      "theoretical_loss": 3.374352153370615,
+      "tokens_seen": 2722430976
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017673728133525918,
+      "loss": 1.2286,
+      "theoretical_loss": 3.374339522465414,
+      "tokens_seen": 2722562048
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017669715936446798,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3743268923385425,
+      "tokens_seen": 2722693120
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001766570373936768,
+      "loss": 1.2873,
+      "theoretical_loss": 3.3743142629899143,
+      "tokens_seen": 2722824192
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017661691542288557,
+      "loss": 1.2037,
+      "theoretical_loss": 3.374301634419444,
+      "tokens_seen": 2722955264
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017657679345209438,
+      "loss": 1.2526,
+      "theoretical_loss": 3.374289006627047,
+      "tokens_seen": 2723086336
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017653667148130318,
+      "loss": 1.2113,
+      "theoretical_loss": 3.3742763796126374,
+      "tokens_seen": 2723217408
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017649654951051196,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3742637533761295,
+      "tokens_seen": 2723348480
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017645642753972074,
+      "loss": 1.2327,
+      "theoretical_loss": 3.3742511279174376,
+      "tokens_seen": 2723479552
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017641630556892955,
+      "loss": 1.205,
+      "theoretical_loss": 3.374238503236478,
+      "tokens_seen": 2723610624
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017637618359813833,
+      "loss": 1.2807,
+      "theoretical_loss": 3.3742258793331636,
+      "tokens_seen": 2723741696
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017633606162734713,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3742132562074105,
+      "tokens_seen": 2723872768
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017629593965655594,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3742006338591324,
+      "tokens_seen": 2724003840
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017625581768576475,
+      "loss": 1.1418,
+      "theoretical_loss": 3.3741880122882444,
+      "tokens_seen": 2724134912
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017621569571497352,
+      "loss": 1.1927,
+      "theoretical_loss": 3.374175391494661,
+      "tokens_seen": 2724265984
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001761755737441823,
+      "loss": 1.1903,
+      "theoretical_loss": 3.3741627714782974,
+      "tokens_seen": 2724397056
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001761354517733911,
+      "loss": 1.2112,
+      "theoretical_loss": 3.374150152239068,
+      "tokens_seen": 2724528128
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001760953298025999,
+      "loss": 1.2334,
+      "theoretical_loss": 3.374137533776887,
+      "tokens_seen": 2724659200
+    },
+    {
+      "epoch": 0.65,
+      "objective/train/docs_used": 914654,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.289285659790039,
+      "objective/train/theoretical_loss": 3.37412491609167,
+      "objective/train/tokens_used": 1095249376,
+      "theoretical_loss": 3.37412491609167,
+      "tokens_seen": 2724790272
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001760552078318087,
+      "loss": 1.2619,
+      "theoretical_loss": 3.37412491609167,
+      "tokens_seen": 2724790272
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001760150858610175,
+      "loss": 1.2029,
+      "theoretical_loss": 3.3741122991833317,
+      "tokens_seen": 2724921344
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001759749638902263,
+      "loss": 1.243,
+      "theoretical_loss": 3.374099683051787,
+      "tokens_seen": 2725052416
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001759348419194351,
+      "loss": 1.2204,
+      "theoretical_loss": 3.37408706769695,
+      "tokens_seen": 2725183488
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001758947199486439,
+      "loss": 1.3018,
+      "theoretical_loss": 3.3740744531187365,
+      "tokens_seen": 2725314560
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017585459797785267,
+      "loss": 1.1906,
+      "theoretical_loss": 3.3740618393170605,
+      "tokens_seen": 2725445632
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017581447600706145,
+      "loss": 1.2253,
+      "theoretical_loss": 3.374049226291837,
+      "tokens_seen": 2725576704
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017577435403627026,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3740366140429816,
+      "tokens_seen": 2725707776
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017573423206547907,
+      "loss": 1.2611,
+      "theoretical_loss": 3.3740240025704087,
+      "tokens_seen": 2725838848
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017569411009468787,
+      "loss": 1.2338,
+      "theoretical_loss": 3.3740113918740327,
+      "tokens_seen": 2725969920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017565398812389665,
+      "loss": 1.2095,
+      "theoretical_loss": 3.373998781953769,
+      "tokens_seen": 2726100992
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017561386615310546,
+      "loss": 1.1597,
+      "theoretical_loss": 3.373986172809533,
+      "tokens_seen": 2726232064
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017557374418231424,
+      "loss": 1.1838,
+      "theoretical_loss": 3.3739735644412385,
+      "tokens_seen": 2726363136
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017553362221152302,
+      "loss": 1.2223,
+      "theoretical_loss": 3.3739609568488014,
+      "tokens_seen": 2726494208
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017549350024073182,
+      "loss": 1.2103,
+      "theoretical_loss": 3.373948350032136,
+      "tokens_seen": 2726625280
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017545337826994063,
+      "loss": 1.2056,
+      "theoretical_loss": 3.373935743991158,
+      "tokens_seen": 2726756352
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017541325629914943,
+      "loss": 1.2261,
+      "theoretical_loss": 3.3739231387257815,
+      "tokens_seen": 2726887424
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001753731343283582,
+      "loss": 1.1737,
+      "theoretical_loss": 3.3739105342359226,
+      "tokens_seen": 2727018496
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017533301235756702,
+      "loss": 1.1838,
+      "theoretical_loss": 3.3738979305214953,
+      "tokens_seen": 2727149568
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001752928903867758,
+      "loss": 1.1919,
+      "theoretical_loss": 3.373885327582415,
+      "tokens_seen": 2727280640
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017525276841598458,
+      "loss": 1.2395,
+      "theoretical_loss": 3.373872725418597,
+      "tokens_seen": 2727411712
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017521264644519338,
+      "loss": 1.1946,
+      "theoretical_loss": 3.373860124029956,
+      "tokens_seen": 2727542784
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001751725244744022,
+      "loss": 1.229,
+      "theoretical_loss": 3.3738475234164076,
+      "tokens_seen": 2727673856
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017513240250361097,
+      "loss": 1.2376,
+      "theoretical_loss": 3.373834923577866,
+      "tokens_seen": 2727804928
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017509228053281978,
+      "loss": 1.2542,
+      "theoretical_loss": 3.373822324514247,
+      "tokens_seen": 2727936000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017505215856202858,
+      "loss": 1.2477,
+      "theoretical_loss": 3.3738097262254656,
+      "tokens_seen": 2728067072
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001750120365912374,
+      "loss": 1.2102,
+      "theoretical_loss": 3.3737971287114368,
+      "tokens_seen": 2728198144
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017497191462044614,
+      "loss": 1.2422,
+      "theoretical_loss": 3.3737845319720754,
+      "tokens_seen": 2728329216
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017493179264965495,
+      "loss": 1.3175,
+      "theoretical_loss": 3.373771936007297,
+      "tokens_seen": 2728460288
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017489167067886375,
+      "loss": 1.2566,
+      "theoretical_loss": 3.373759340817017,
+      "tokens_seen": 2728591360
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017485154870807253,
+      "loss": 1.2166,
+      "theoretical_loss": 3.3737467464011504,
+      "tokens_seen": 2728722432
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017481142673728134,
+      "loss": 1.2318,
+      "theoretical_loss": 3.373734152759612,
+      "tokens_seen": 2728853504
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017477130476649015,
+      "loss": 1.2692,
+      "theoretical_loss": 3.373721559892317,
+      "tokens_seen": 2728984576
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017473118279569895,
+      "loss": 1.2177,
+      "theoretical_loss": 3.373708967799181,
+      "tokens_seen": 2729115648
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001746910608249077,
+      "loss": 1.2165,
+      "theoretical_loss": 3.3736963764801193,
+      "tokens_seen": 2729246720
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001746509388541165,
+      "loss": 1.238,
+      "theoretical_loss": 3.373683785935047,
+      "tokens_seen": 2729377792
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017461081688332532,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3736711961638792,
+      "tokens_seen": 2729508864
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001745706949125341,
+      "loss": 1.2074,
+      "theoretical_loss": 3.373658607166531,
+      "tokens_seen": 2729639936
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001745305729417429,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3736460189429183,
+      "tokens_seen": 2729771008
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001744904509709517,
+      "loss": 1.2362,
+      "theoretical_loss": 3.373633431492956,
+      "tokens_seen": 2729902080
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017445032900016051,
+      "loss": 1.2712,
+      "theoretical_loss": 3.373620844816559,
+      "tokens_seen": 2730033152
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001744102070293693,
+      "loss": 1.1844,
+      "theoretical_loss": 3.3736082589136434,
+      "tokens_seen": 2730164224
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017437008505857807,
+      "loss": 1.2615,
+      "theoretical_loss": 3.3735956737841244,
+      "tokens_seen": 2730295296
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017432996308778688,
+      "loss": 1.2879,
+      "theoretical_loss": 3.373583089427917,
+      "tokens_seen": 2730426368
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017428984111699566,
+      "loss": 1.2793,
+      "theoretical_loss": 3.373570505844937,
+      "tokens_seen": 2730557440
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017424971914620446,
+      "loss": 1.2348,
+      "theoretical_loss": 3.373557923035099,
+      "tokens_seen": 2730688512
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017420959717541327,
+      "loss": 1.1806,
+      "theoretical_loss": 3.3735453409983194,
+      "tokens_seen": 2730819584
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017416947520462205,
+      "loss": 1.2485,
+      "theoretical_loss": 3.373532759734513,
+      "tokens_seen": 2730950656
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017412935323383086,
+      "loss": 1.2506,
+      "theoretical_loss": 3.373520179243595,
+      "tokens_seen": 2731081728
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017408923126303964,
+      "loss": 1.2445,
+      "theoretical_loss": 3.3735075995254817,
+      "tokens_seen": 2731212800
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 916965,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1812257766723633,
+      "objective/train/theoretical_loss": 3.3734950205800875,
+      "objective/train/tokens_used": 1101802976,
+      "theoretical_loss": 3.3734950205800875,
+      "tokens_seen": 2731343872
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017404910929224844,
+      "loss": 1.2061,
+      "theoretical_loss": 3.3734950205800875,
+      "tokens_seen": 2731343872
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017400898732145722,
+      "loss": 1.1287,
+      "theoretical_loss": 3.3734824424073286,
+      "tokens_seen": 2731474944
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017396886535066603,
+      "loss": 1.147,
+      "theoretical_loss": 3.37346986500712,
+      "tokens_seen": 2731606016
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017392874337987483,
+      "loss": 1.2743,
+      "theoretical_loss": 3.3734572883793774,
+      "tokens_seen": 2731737088
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001738886214090836,
+      "loss": 1.2211,
+      "theoretical_loss": 3.373444712524017,
+      "tokens_seen": 2731868160
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017384849943829242,
+      "loss": 1.2269,
+      "theoretical_loss": 3.373432137440953,
+      "tokens_seen": 2731999232
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001738083774675012,
+      "loss": 1.218,
+      "theoretical_loss": 3.3734195631301014,
+      "tokens_seen": 2732130304
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017376825549671,
+      "loss": 1.2428,
+      "theoretical_loss": 3.3734069895913783,
+      "tokens_seen": 2732261376
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017372813352591878,
+      "loss": 1.1855,
+      "theoretical_loss": 3.3733944168246985,
+      "tokens_seen": 2732392448
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001736880115551276,
+      "loss": 1.2289,
+      "theoretical_loss": 3.373381844829978,
+      "tokens_seen": 2732523520
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001736478895843364,
+      "loss": 1.2623,
+      "theoretical_loss": 3.3733692736071323,
+      "tokens_seen": 2732654592
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017360776761354518,
+      "loss": 1.1783,
+      "theoretical_loss": 3.373356703156077,
+      "tokens_seen": 2732785664
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017356764564275398,
+      "loss": 1.1823,
+      "theoretical_loss": 3.373344133476728,
+      "tokens_seen": 2732916736
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001735275236719628,
+      "loss": 1.2017,
+      "theoretical_loss": 3.373331564569,
+      "tokens_seen": 2733047808
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017348740170117157,
+      "loss": 1.1832,
+      "theoretical_loss": 3.3733189964328094,
+      "tokens_seen": 2733178880
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017344727973038035,
+      "loss": 1.2268,
+      "theoretical_loss": 3.3733064290680717,
+      "tokens_seen": 2733309952
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017340715775958915,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3732938624747026,
+      "tokens_seen": 2733441024
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017336703578879796,
+      "loss": 1.1613,
+      "theoretical_loss": 3.3732812966526176,
+      "tokens_seen": 2733572096
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017332691381800674,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3732687316017325,
+      "tokens_seen": 2733703168
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017328679184721555,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3732561673219625,
+      "tokens_seen": 2733834240
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017324666987642435,
+      "loss": 1.2678,
+      "theoretical_loss": 3.3732436038132243,
+      "tokens_seen": 2733965312
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001732065479056331,
+      "loss": 1.2726,
+      "theoretical_loss": 3.373231041075433,
+      "tokens_seen": 2734096384
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001731664259348419,
+      "loss": 1.2625,
+      "theoretical_loss": 3.3732184791085045,
+      "tokens_seen": 2734227456
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017312630396405072,
+      "loss": 1.2357,
+      "theoretical_loss": 3.373205917912354,
+      "tokens_seen": 2734358528
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017308618199325952,
+      "loss": 1.21,
+      "theoretical_loss": 3.373193357486898,
+      "tokens_seen": 2734489600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001730460600224683,
+      "loss": 1.3024,
+      "theoretical_loss": 3.373180797832052,
+      "tokens_seen": 2734620672
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001730059380516771,
+      "loss": 1.2441,
+      "theoretical_loss": 3.373168238947732,
+      "tokens_seen": 2734751744
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017296581608088591,
+      "loss": 1.25,
+      "theoretical_loss": 3.3731556808338534,
+      "tokens_seen": 2734882816
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017292569411009467,
+      "loss": 1.2823,
+      "theoretical_loss": 3.3731431234903324,
+      "tokens_seen": 2735013888
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017288557213930347,
+      "loss": 1.1631,
+      "theoretical_loss": 3.3731305669170846,
+      "tokens_seen": 2735144960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017284545016851228,
+      "loss": 1.2395,
+      "theoretical_loss": 3.373118011114026,
+      "tokens_seen": 2735276032
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017280532819772109,
+      "loss": 1.1706,
+      "theoretical_loss": 3.3731054560810723,
+      "tokens_seen": 2735407104
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017276520622692986,
+      "loss": 1.2512,
+      "theoretical_loss": 3.373092901818139,
+      "tokens_seen": 2735538176
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017272508425613867,
+      "loss": 1.3165,
+      "theoretical_loss": 3.373080348325143,
+      "tokens_seen": 2735669248
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017268496228534748,
+      "loss": 1.193,
+      "theoretical_loss": 3.373067795601999,
+      "tokens_seen": 2735800320
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017264484031455626,
+      "loss": 1.1641,
+      "theoretical_loss": 3.373055243648624,
+      "tokens_seen": 2735931392
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017260471834376504,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3730426924649333,
+      "tokens_seen": 2736062464
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017256459637297384,
+      "loss": 1.269,
+      "theoretical_loss": 3.3730301420508426,
+      "tokens_seen": 2736193536
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017252447440218265,
+      "loss": 1.2039,
+      "theoretical_loss": 3.3730175924062684,
+      "tokens_seen": 2736324608
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017248435243139143,
+      "loss": 1.2391,
+      "theoretical_loss": 3.3730050435311267,
+      "tokens_seen": 2736455680
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017244423046060023,
+      "loss": 1.2042,
+      "theoretical_loss": 3.3729924954253327,
+      "tokens_seen": 2736586752
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017240410848980904,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3729799480888034,
+      "tokens_seen": 2736717824
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017236398651901782,
+      "loss": 1.257,
+      "theoretical_loss": 3.3729674015214544,
+      "tokens_seen": 2736848896
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001723238645482266,
+      "loss": 1.1836,
+      "theoretical_loss": 3.3729548557232016,
+      "tokens_seen": 2736979968
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001722837425774354,
+      "loss": 1.2019,
+      "theoretical_loss": 3.372942310693961,
+      "tokens_seen": 2737111040
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017224362060664418,
+      "loss": 1.24,
+      "theoretical_loss": 3.3729297664336486,
+      "tokens_seen": 2737242112
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.000172203498635853,
+      "loss": 1.2421,
+      "theoretical_loss": 3.3729172229421804,
+      "tokens_seen": 2737373184
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001721633766650618,
+      "loss": 1.1867,
+      "theoretical_loss": 3.372904680219473,
+      "tokens_seen": 2737504256
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001721232546942706,
+      "loss": 1.1954,
+      "theoretical_loss": 3.3728921382654415,
+      "tokens_seen": 2737635328
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017208313272347938,
+      "loss": 1.2868,
+      "theoretical_loss": 3.3728795970800034,
+      "tokens_seen": 2737766400
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 919044,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1659284830093384,
+      "objective/train/theoretical_loss": 3.3728670566630736,
+      "objective/train/tokens_used": 1108356576,
+      "theoretical_loss": 3.3728670566630736,
+      "tokens_seen": 2737897472
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001720430107526882,
+      "loss": 1.1665,
+      "theoretical_loss": 3.3728670566630736,
+      "tokens_seen": 2737897472
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017200288878189697,
+      "loss": 1.2694,
+      "theoretical_loss": 3.3728545170145683,
+      "tokens_seen": 2738028544
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017196276681110575,
+      "loss": 1.2136,
+      "theoretical_loss": 3.3728419781344043,
+      "tokens_seen": 2738159616
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017192264484031455,
+      "loss": 1.3597,
+      "theoretical_loss": 3.3728294400224974,
+      "tokens_seen": 2738290688
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017188252286952336,
+      "loss": 1.2943,
+      "theoretical_loss": 3.372816902678764,
+      "tokens_seen": 2738421760
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017184240089873217,
+      "loss": 1.2023,
+      "theoretical_loss": 3.3728043661031197,
+      "tokens_seen": 2738552832
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017180227892794094,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3727918302954807,
+      "tokens_seen": 2738683904
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017176215695714975,
+      "loss": 1.1976,
+      "theoretical_loss": 3.3727792952557643,
+      "tokens_seen": 2738814976
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017172203498635853,
+      "loss": 1.1845,
+      "theoretical_loss": 3.372766760983885,
+      "tokens_seen": 2738946048
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001716819130155673,
+      "loss": 1.2193,
+      "theoretical_loss": 3.372754227479761,
+      "tokens_seen": 2739077120
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017164179104477612,
+      "loss": 1.2069,
+      "theoretical_loss": 3.372741694743307,
+      "tokens_seen": 2739208192
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017160166907398492,
+      "loss": 1.2485,
+      "theoretical_loss": 3.3727291627744393,
+      "tokens_seen": 2739339264
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017156154710319373,
+      "loss": 1.2623,
+      "theoretical_loss": 3.372716631573075,
+      "tokens_seen": 2739470336
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001715214251324025,
+      "loss": 1.2733,
+      "theoretical_loss": 3.37270410113913,
+      "tokens_seen": 2739601408
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017148130316161131,
+      "loss": 1.2762,
+      "theoretical_loss": 3.3726915714725205,
+      "tokens_seen": 2739732480
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001714411811908201,
+      "loss": 1.24,
+      "theoretical_loss": 3.372679042573163,
+      "tokens_seen": 2739863552
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017140105922002887,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3726665144409735,
+      "tokens_seen": 2739994624
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017136093724923768,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3726539870758687,
+      "tokens_seen": 2740125696
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017132081527844648,
+      "loss": 1.1963,
+      "theoretical_loss": 3.3726414604777646,
+      "tokens_seen": 2740256768
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017128069330765526,
+      "loss": 1.1931,
+      "theoretical_loss": 3.3726289346465776,
+      "tokens_seen": 2740387840
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017124057133686407,
+      "loss": 1.2175,
+      "theoretical_loss": 3.372616409582225,
+      "tokens_seen": 2740518912
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017120044936607288,
+      "loss": 1.1604,
+      "theoretical_loss": 3.3726038852846214,
+      "tokens_seen": 2740649984
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017116032739528168,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3725913617536847,
+      "tokens_seen": 2740781056
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017112020542449044,
+      "loss": 1.2197,
+      "theoretical_loss": 3.3725788389893303,
+      "tokens_seen": 2740912128
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017108008345369924,
+      "loss": 1.237,
+      "theoretical_loss": 3.3725663169914757,
+      "tokens_seen": 2741043200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017103996148290805,
+      "loss": 1.1963,
+      "theoretical_loss": 3.372553795760036,
+      "tokens_seen": 2741174272
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017099983951211683,
+      "loss": 1.1985,
+      "theoretical_loss": 3.372541275294929,
+      "tokens_seen": 2741305344
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017095971754132563,
+      "loss": 1.1778,
+      "theoretical_loss": 3.37252875559607,
+      "tokens_seen": 2741436416
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017091959557053444,
+      "loss": 1.2115,
+      "theoretical_loss": 3.372516236663376,
+      "tokens_seen": 2741567488
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017087947359974325,
+      "loss": 1.1762,
+      "theoretical_loss": 3.3725037184967643,
+      "tokens_seen": 2741698560
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.000170839351628952,
+      "loss": 1.2679,
+      "theoretical_loss": 3.37249120109615,
+      "tokens_seen": 2741829632
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001707992296581608,
+      "loss": 1.2376,
+      "theoretical_loss": 3.3724786844614503,
+      "tokens_seen": 2741960704
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001707591076873696,
+      "loss": 1.218,
+      "theoretical_loss": 3.3724661685925814,
+      "tokens_seen": 2742091776
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001707189857165784,
+      "loss": 1.297,
+      "theoretical_loss": 3.3724536534894605,
+      "tokens_seen": 2742222848
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001706788637457872,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3724411391520035,
+      "tokens_seen": 2742353920
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.000170638741774996,
+      "loss": 1.3064,
+      "theoretical_loss": 3.372428625580127,
+      "tokens_seen": 2742484992
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001705986198042048,
+      "loss": 1.3053,
+      "theoretical_loss": 3.372416112773748,
+      "tokens_seen": 2742616064
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017055849783341356,
+      "loss": 1.1792,
+      "theoretical_loss": 3.372403600732783,
+      "tokens_seen": 2742747136
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017051837586262237,
+      "loss": 1.2976,
+      "theoretical_loss": 3.3723910894571483,
+      "tokens_seen": 2742878208
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017047825389183117,
+      "loss": 1.2456,
+      "theoretical_loss": 3.3723785789467606,
+      "tokens_seen": 2743009280
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017043813192103995,
+      "loss": 1.1836,
+      "theoretical_loss": 3.3723660692015365,
+      "tokens_seen": 2743140352
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017039800995024876,
+      "loss": 1.2297,
+      "theoretical_loss": 3.372353560221393,
+      "tokens_seen": 2743271424
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017035788797945757,
+      "loss": 1.2619,
+      "theoretical_loss": 3.3723410520062465,
+      "tokens_seen": 2743402496
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017031776600866637,
+      "loss": 1.2111,
+      "theoretical_loss": 3.3723285445560136,
+      "tokens_seen": 2743533568
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017027764403787515,
+      "loss": 1.1755,
+      "theoretical_loss": 3.372316037870611,
+      "tokens_seen": 2743664640
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017023752206708393,
+      "loss": 1.2197,
+      "theoretical_loss": 3.3723035319499557,
+      "tokens_seen": 2743795712
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017019740009629274,
+      "loss": 1.2575,
+      "theoretical_loss": 3.372291026793964,
+      "tokens_seen": 2743926784
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017015727812550152,
+      "loss": 1.225,
+      "theoretical_loss": 3.372278522402553,
+      "tokens_seen": 2744057856
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017011715615471032,
+      "loss": 1.2633,
+      "theoretical_loss": 3.372266018775639,
+      "tokens_seen": 2744188928
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017007703418391913,
+      "loss": 1.2245,
+      "theoretical_loss": 3.372253515913139,
+      "tokens_seen": 2744320000
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 921178,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2518497705459595,
+      "objective/train/theoretical_loss": 3.37224101381497,
+      "objective/train/tokens_used": 1114910176,
+      "theoretical_loss": 3.37224101381497,
+      "tokens_seen": 2744451072
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001700369122131279,
+      "loss": 1.2926,
+      "theoretical_loss": 3.37224101381497,
+      "tokens_seen": 2744451072
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001699967902423367,
+      "loss": 1.2209,
+      "theoretical_loss": 3.372228512481048,
+      "tokens_seen": 2744582144
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001699566682715455,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3722160119112905,
+      "tokens_seen": 2744713216
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001699165463007543,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3722035121056146,
+      "tokens_seen": 2744844288
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016987642432996308,
+      "loss": 1.2736,
+      "theoretical_loss": 3.3721910130639356,
+      "tokens_seen": 2744975360
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016983630235917188,
+      "loss": 1.2131,
+      "theoretical_loss": 3.3721785147861723,
+      "tokens_seen": 2745106432
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001697961803883807,
+      "loss": 1.1681,
+      "theoretical_loss": 3.37216601727224,
+      "tokens_seen": 2745237504
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016975605841758947,
+      "loss": 1.2449,
+      "theoretical_loss": 3.3721535205220565,
+      "tokens_seen": 2745368576
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016971593644679828,
+      "loss": 1.2744,
+      "theoretical_loss": 3.372141024535538,
+      "tokens_seen": 2745499648
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016967581447600708,
+      "loss": 1.1607,
+      "theoretical_loss": 3.3721285293126018,
+      "tokens_seen": 2745630720
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016963569250521586,
+      "loss": 1.2365,
+      "theoretical_loss": 3.372116034853165,
+      "tokens_seen": 2745761792
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016959557053442464,
+      "loss": 1.2311,
+      "theoretical_loss": 3.372103541157144,
+      "tokens_seen": 2745892864
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016955544856363345,
+      "loss": 1.2806,
+      "theoretical_loss": 3.3720910482244557,
+      "tokens_seen": 2746023936
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016951532659284225,
+      "loss": 1.2418,
+      "theoretical_loss": 3.3720785560550173,
+      "tokens_seen": 2746155008
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016947520462205103,
+      "loss": 1.1858,
+      "theoretical_loss": 3.3720660646487453,
+      "tokens_seen": 2746286080
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016943508265125984,
+      "loss": 1.1887,
+      "theoretical_loss": 3.3720535740055575,
+      "tokens_seen": 2746417152
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016939496068046865,
+      "loss": 1.1812,
+      "theoretical_loss": 3.37204108412537,
+      "tokens_seen": 2746548224
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016935483870967742,
+      "loss": 1.2699,
+      "theoretical_loss": 3.372028595008101,
+      "tokens_seen": 2746679296
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001693147167388862,
+      "loss": 1.1668,
+      "theoretical_loss": 3.372016106653666,
+      "tokens_seen": 2746810368
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.000169274594768095,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3720036190619824,
+      "tokens_seen": 2746941440
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016923447279730382,
+      "loss": 1.2209,
+      "theoretical_loss": 3.371991132232968,
+      "tokens_seen": 2747072512
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001691943508265126,
+      "loss": 1.2041,
+      "theoretical_loss": 3.3719786461665393,
+      "tokens_seen": 2747203584
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001691542288557214,
+      "loss": 1.1719,
+      "theoretical_loss": 3.3719661608626135,
+      "tokens_seen": 2747334656
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001691141068849302,
+      "loss": 1.24,
+      "theoretical_loss": 3.371953676321107,
+      "tokens_seen": 2747465728
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016907398491413896,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3719411925419376,
+      "tokens_seen": 2747596800
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016903386294334777,
+      "loss": 1.2564,
+      "theoretical_loss": 3.3719287095250223,
+      "tokens_seen": 2747727872
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016899374097255657,
+      "loss": 1.2094,
+      "theoretical_loss": 3.3719162272702783,
+      "tokens_seen": 2747858944
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016895361900176538,
+      "loss": 1.1912,
+      "theoretical_loss": 3.3719037457776224,
+      "tokens_seen": 2747990016
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016891349703097416,
+      "loss": 1.2095,
+      "theoretical_loss": 3.3718912650469717,
+      "tokens_seen": 2748121088
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016887337506018296,
+      "loss": 1.2538,
+      "theoretical_loss": 3.371878785078244,
+      "tokens_seen": 2748252160
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016883325308939177,
+      "loss": 1.2313,
+      "theoretical_loss": 3.371866305871355,
+      "tokens_seen": 2748383232
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016879313111860055,
+      "loss": 1.1933,
+      "theoretical_loss": 3.3718538274262237,
+      "tokens_seen": 2748514304
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016875300914780933,
+      "loss": 1.1799,
+      "theoretical_loss": 3.3718413497427657,
+      "tokens_seen": 2748645376
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016871288717701814,
+      "loss": 1.1802,
+      "theoretical_loss": 3.3718288728208994,
+      "tokens_seen": 2748776448
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016867276520622694,
+      "loss": 1.2018,
+      "theoretical_loss": 3.371816396660541,
+      "tokens_seen": 2748907520
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016863264323543572,
+      "loss": 1.1942,
+      "theoretical_loss": 3.371803921261608,
+      "tokens_seen": 2749038592
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016859252126464453,
+      "loss": 1.2828,
+      "theoretical_loss": 3.3717914466240186,
+      "tokens_seen": 2749169664
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016855239929385333,
+      "loss": 1.1977,
+      "theoretical_loss": 3.3717789727476886,
+      "tokens_seen": 2749300736
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001685122773230621,
+      "loss": 1.196,
+      "theoretical_loss": 3.3717664996325363,
+      "tokens_seen": 2749431808
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001684721553522709,
+      "loss": 1.2456,
+      "theoretical_loss": 3.3717540272784783,
+      "tokens_seen": 2749562880
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001684320333814797,
+      "loss": 1.1103,
+      "theoretical_loss": 3.3717415556854324,
+      "tokens_seen": 2749693952
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001683919114106885,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3717290848533152,
+      "tokens_seen": 2749825024
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016835178943989728,
+      "loss": 1.2345,
+      "theoretical_loss": 3.371716614782045,
+      "tokens_seen": 2749956096
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001683116674691061,
+      "loss": 1.2574,
+      "theoretical_loss": 3.371704145471538,
+      "tokens_seen": 2750087168
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001682715454983149,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3716916769217127,
+      "tokens_seen": 2750218240
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016823142352752368,
+      "loss": 1.2011,
+      "theoretical_loss": 3.371679209132486,
+      "tokens_seen": 2750349312
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016819130155673246,
+      "loss": 1.2882,
+      "theoretical_loss": 3.3716667421037743,
+      "tokens_seen": 2750480384
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016815117958594126,
+      "loss": 1.2282,
+      "theoretical_loss": 3.371654275835496,
+      "tokens_seen": 2750611456
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016811105761515004,
+      "loss": 1.1642,
+      "theoretical_loss": 3.3716418103275685,
+      "tokens_seen": 2750742528
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016807093564435885,
+      "loss": 1.2242,
+      "theoretical_loss": 3.371629345579909,
+      "tokens_seen": 2750873600
+    },
+    {
+      "epoch": 0.67,
+      "objective/train/docs_used": 923258,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2300562858581543,
+      "objective/train/theoretical_loss": 3.3716168815924346,
+      "objective/train/tokens_used": 1121463776,
+      "theoretical_loss": 3.3716168815924346,
+      "tokens_seen": 2751004672
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016803081367356765,
+      "loss": 1.1849,
+      "theoretical_loss": 3.3716168815924346,
+      "tokens_seen": 2751004672
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016799069170277646,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3716044183650635,
+      "tokens_seen": 2751135744
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016795056973198524,
+      "loss": 1.2982,
+      "theoretical_loss": 3.371591955897712,
+      "tokens_seen": 2751266816
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016791044776119405,
+      "loss": 1.2037,
+      "theoretical_loss": 3.3715794941902986,
+      "tokens_seen": 2751397888
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016787032579040282,
+      "loss": 1.2388,
+      "theoretical_loss": 3.3715670332427403,
+      "tokens_seen": 2751528960
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001678302038196116,
+      "loss": 1.1449,
+      "theoretical_loss": 3.3715545730549543,
+      "tokens_seen": 2751660032
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001677900818488204,
+      "loss": 1.1911,
+      "theoretical_loss": 3.371542113626859,
+      "tokens_seen": 2751791104
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016774995987802922,
+      "loss": 1.2409,
+      "theoretical_loss": 3.371529654958371,
+      "tokens_seen": 2751922176
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016770983790723802,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3715171970494078,
+      "tokens_seen": 2752053248
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001676697159364468,
+      "loss": 1.1459,
+      "theoretical_loss": 3.3715047398998874,
+      "tokens_seen": 2752184320
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001676295939656556,
+      "loss": 1.2685,
+      "theoretical_loss": 3.371492283509727,
+      "tokens_seen": 2752315392
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001675894719948644,
+      "loss": 1.2059,
+      "theoretical_loss": 3.371479827878845,
+      "tokens_seen": 2752446464
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016754935002407317,
+      "loss": 1.198,
+      "theoretical_loss": 3.3714673730071576,
+      "tokens_seen": 2752577536
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016750922805328197,
+      "loss": 1.2134,
+      "theoretical_loss": 3.3714549188945835,
+      "tokens_seen": 2752708608
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016746910608249078,
+      "loss": 1.2183,
+      "theoretical_loss": 3.37144246554104,
+      "tokens_seen": 2752839680
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016742898411169959,
+      "loss": 1.1733,
+      "theoretical_loss": 3.371430012946444,
+      "tokens_seen": 2752970752
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016738886214090836,
+      "loss": 1.2599,
+      "theoretical_loss": 3.371417561110714,
+      "tokens_seen": 2753101824
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016734874017011717,
+      "loss": 1.2158,
+      "theoretical_loss": 3.371405110033767,
+      "tokens_seen": 2753232896
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016730861819932598,
+      "loss": 1.1912,
+      "theoretical_loss": 3.3713926597155215,
+      "tokens_seen": 2753363968
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016726849622853473,
+      "loss": 1.1118,
+      "theoretical_loss": 3.3713802101558947,
+      "tokens_seen": 2753495040
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016722837425774354,
+      "loss": 1.1498,
+      "theoretical_loss": 3.371367761354804,
+      "tokens_seen": 2753626112
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016718825228695234,
+      "loss": 1.1803,
+      "theoretical_loss": 3.3713553133121668,
+      "tokens_seen": 2753757184
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016714813031616112,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3713428660279017,
+      "tokens_seen": 2753888256
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016710800834536993,
+      "loss": 1.1846,
+      "theoretical_loss": 3.371330419501926,
+      "tokens_seen": 2754019328
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016706788637457873,
+      "loss": 1.1895,
+      "theoretical_loss": 3.3713179737341568,
+      "tokens_seen": 2754150400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016702776440378754,
+      "loss": 1.2466,
+      "theoretical_loss": 3.371305528724513,
+      "tokens_seen": 2754281472
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001669876424329963,
+      "loss": 1.2243,
+      "theoretical_loss": 3.3712930844729114,
+      "tokens_seen": 2754412544
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001669475204622051,
+      "loss": 1.1716,
+      "theoretical_loss": 3.3712806409792706,
+      "tokens_seen": 2754543616
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001669073984914139,
+      "loss": 1.1733,
+      "theoretical_loss": 3.3712681982435075,
+      "tokens_seen": 2754674688
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016686727652062268,
+      "loss": 1.2336,
+      "theoretical_loss": 3.37125575626554,
+      "tokens_seen": 2754805760
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001668271545498315,
+      "loss": 1.1654,
+      "theoretical_loss": 3.3712433150452865,
+      "tokens_seen": 2754936832
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001667870325790403,
+      "loss": 1.1753,
+      "theoretical_loss": 3.3712308745826647,
+      "tokens_seen": 2755067904
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001667469106082491,
+      "loss": 1.1895,
+      "theoretical_loss": 3.371218434877592,
+      "tokens_seen": 2755198976
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016670678863745785,
+      "loss": 1.188,
+      "theoretical_loss": 3.3712059959299863,
+      "tokens_seen": 2755330048
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016666666666666666,
+      "loss": 1.2105,
+      "theoretical_loss": 3.3711935577397654,
+      "tokens_seen": 2755461120
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016662654469587547,
+      "loss": 1.2581,
+      "theoretical_loss": 3.371181120306848,
+      "tokens_seen": 2755592192
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016658642272508425,
+      "loss": 1.2134,
+      "theoretical_loss": 3.3711686836311507,
+      "tokens_seen": 2755723264
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016654630075429305,
+      "loss": 1.2022,
+      "theoretical_loss": 3.3711562477125923,
+      "tokens_seen": 2755854336
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016650617878350186,
+      "loss": 1.1693,
+      "theoretical_loss": 3.37114381255109,
+      "tokens_seen": 2755985408
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016646605681271067,
+      "loss": 1.2058,
+      "theoretical_loss": 3.3711313781465626,
+      "tokens_seen": 2756116480
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016642593484191944,
+      "loss": 1.243,
+      "theoretical_loss": 3.371118944498927,
+      "tokens_seen": 2756247552
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016638581287112822,
+      "loss": 1.2073,
+      "theoretical_loss": 3.371106511608102,
+      "tokens_seen": 2756378624
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016634569090033703,
+      "loss": 1.1434,
+      "theoretical_loss": 3.3710940794740054,
+      "tokens_seen": 2756509696
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001663055689295458,
+      "loss": 1.1277,
+      "theoretical_loss": 3.3710816480965544,
+      "tokens_seen": 2756640768
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016626544695875462,
+      "loss": 1.1661,
+      "theoretical_loss": 3.371069217475668,
+      "tokens_seen": 2756771840
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016622532498796342,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3710567876112636,
+      "tokens_seen": 2756902912
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016618520301717223,
+      "loss": 1.1833,
+      "theoretical_loss": 3.3710443585032595,
+      "tokens_seen": 2757033984
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.000166145081046381,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3710319301515734,
+      "tokens_seen": 2757165056
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001661049590755898,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3710195025561234,
+      "tokens_seen": 2757296128
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001660648371047986,
+      "loss": 1.2682,
+      "theoretical_loss": 3.371007075716828,
+      "tokens_seen": 2757427200
+    },
+    {
+      "epoch": 0.67,
+      "objective/train/docs_used": 925397,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1796047687530518,
+      "objective/train/theoretical_loss": 3.3709946496336043,
+      "objective/train/tokens_used": 1128017376,
+      "theoretical_loss": 3.3709946496336043,
+      "tokens_seen": 2757558272
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016602471513400737,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3709946496336043,
+      "tokens_seen": 2757558272
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016598459316321618,
+      "loss": 1.1512,
+      "theoretical_loss": 3.3709822243063714,
+      "tokens_seen": 2757689344
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016594447119242498,
+      "loss": 1.2488,
+      "theoretical_loss": 3.3709697997350467,
+      "tokens_seen": 2757820416
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016590434922163376,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3709573759195486,
+      "tokens_seen": 2757951488
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016586422725084257,
+      "loss": 1.2658,
+      "theoretical_loss": 3.3709449528597952,
+      "tokens_seen": 2758082560
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016582410528005135,
+      "loss": 1.1813,
+      "theoretical_loss": 3.370932530555704,
+      "tokens_seen": 2758213632
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016578398330926016,
+      "loss": 1.2681,
+      "theoretical_loss": 3.3709201090071943,
+      "tokens_seen": 2758344704
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016574386133846894,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3709076882141833,
+      "tokens_seen": 2758475776
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016570373936767774,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3708952681765894,
+      "tokens_seen": 2758606848
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016566361739688655,
+      "loss": 1.2009,
+      "theoretical_loss": 3.370882848894331,
+      "tokens_seen": 2758737920
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016562349542609533,
+      "loss": 1.1964,
+      "theoretical_loss": 3.370870430367326,
+      "tokens_seen": 2758868992
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016558337345530413,
+      "loss": 1.2101,
+      "theoretical_loss": 3.370858012595493,
+      "tokens_seen": 2759000064
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016554325148451294,
+      "loss": 1.2519,
+      "theoretical_loss": 3.370845595578749,
+      "tokens_seen": 2759131136
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016550312951372172,
+      "loss": 1.1478,
+      "theoretical_loss": 3.370833179317014,
+      "tokens_seen": 2759262208
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001654630075429305,
+      "loss": 1.1656,
+      "theoretical_loss": 3.370820763810205,
+      "tokens_seen": 2759393280
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001654228855721393,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3708083490582403,
+      "tokens_seen": 2759524352
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001653827636013481,
+      "loss": 1.195,
+      "theoretical_loss": 3.370795935061039,
+      "tokens_seen": 2759655424
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001653426416305569,
+      "loss": 1.1779,
+      "theoretical_loss": 3.3707835218185185,
+      "tokens_seen": 2759786496
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001653025196597657,
+      "loss": 1.1647,
+      "theoretical_loss": 3.3707711093305974,
+      "tokens_seen": 2759917568
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001652623976889745,
+      "loss": 1.2033,
+      "theoretical_loss": 3.370758697597194,
+      "tokens_seen": 2760048640
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016522227571818328,
+      "loss": 1.2051,
+      "theoretical_loss": 3.370746286618227,
+      "tokens_seen": 2760179712
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016518215374739206,
+      "loss": 1.1748,
+      "theoretical_loss": 3.3707338763936137,
+      "tokens_seen": 2760310784
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016514203177660087,
+      "loss": 1.2506,
+      "theoretical_loss": 3.3707214669232735,
+      "tokens_seen": 2760441856
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016510190980580967,
+      "loss": 1.1695,
+      "theoretical_loss": 3.370709058207124,
+      "tokens_seen": 2760572928
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016506178783501845,
+      "loss": 1.2253,
+      "theoretical_loss": 3.370696650245084,
+      "tokens_seen": 2760704000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016502166586422726,
+      "loss": 1.2254,
+      "theoretical_loss": 3.3706842430370716,
+      "tokens_seen": 2760835072
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016498154389343607,
+      "loss": 1.1482,
+      "theoretical_loss": 3.3706718365830053,
+      "tokens_seen": 2760966144
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016494142192264484,
+      "loss": 1.1682,
+      "theoretical_loss": 3.3706594308828035,
+      "tokens_seen": 2761097216
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016490129995185362,
+      "loss": 1.2067,
+      "theoretical_loss": 3.3706470259363845,
+      "tokens_seen": 2761228288
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016486117798106243,
+      "loss": 1.1647,
+      "theoretical_loss": 3.370634621743667,
+      "tokens_seen": 2761359360
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016482105601027124,
+      "loss": 1.2165,
+      "theoretical_loss": 3.3706222183045695,
+      "tokens_seen": 2761490432
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016478093403948002,
+      "loss": 1.1846,
+      "theoretical_loss": 3.3706098156190096,
+      "tokens_seen": 2761621504
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016474081206868882,
+      "loss": 1.185,
+      "theoretical_loss": 3.370597413686907,
+      "tokens_seen": 2761752576
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016470069009789763,
+      "loss": 1.2145,
+      "theoretical_loss": 3.3705850125081795,
+      "tokens_seen": 2761883648
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001646605681271064,
+      "loss": 1.2609,
+      "theoretical_loss": 3.370572612082745,
+      "tokens_seen": 2762014720
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016462044615631519,
+      "loss": 1.119,
+      "theoretical_loss": 3.370560212410523,
+      "tokens_seen": 2762145792
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.000164580324185524,
+      "loss": 1.2583,
+      "theoretical_loss": 3.3705478134914317,
+      "tokens_seen": 2762276864
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001645402022147328,
+      "loss": 1.136,
+      "theoretical_loss": 3.3705354153253895,
+      "tokens_seen": 2762407936
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016450008024394158,
+      "loss": 1.1675,
+      "theoretical_loss": 3.370523017912315,
+      "tokens_seen": 2762539008
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016445995827315038,
+      "loss": 1.3177,
+      "theoretical_loss": 3.370510621252127,
+      "tokens_seen": 2762670080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001644198363023592,
+      "loss": 1.23,
+      "theoretical_loss": 3.3704982253447433,
+      "tokens_seen": 2762801152
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016437971433156797,
+      "loss": 1.204,
+      "theoretical_loss": 3.3704858301900833,
+      "tokens_seen": 2762932224
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016433959236077675,
+      "loss": 1.2097,
+      "theoretical_loss": 3.3704734357880652,
+      "tokens_seen": 2763063296
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016429947038998556,
+      "loss": 1.1665,
+      "theoretical_loss": 3.370461042138608,
+      "tokens_seen": 2763194368
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016425934841919436,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3704486492416295,
+      "tokens_seen": 2763325440
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016421922644840314,
+      "loss": 1.0907,
+      "theoretical_loss": 3.3704362570970487,
+      "tokens_seen": 2763456512
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016417910447761195,
+      "loss": 1.1855,
+      "theoretical_loss": 3.370423865704785,
+      "tokens_seen": 2763587584
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016413898250682075,
+      "loss": 1.1701,
+      "theoretical_loss": 3.370411475064756,
+      "tokens_seen": 2763718656
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016409886053602953,
+      "loss": 1.1586,
+      "theoretical_loss": 3.3703990851768806,
+      "tokens_seen": 2763849728
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016405873856523834,
+      "loss": 1.1917,
+      "theoretical_loss": 3.3703866960410775,
+      "tokens_seen": 2763980800
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 927529,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1614798307418823,
+      "objective/train/theoretical_loss": 3.370374307657266,
+      "objective/train/tokens_used": 1134570976,
+      "theoretical_loss": 3.370374307657266,
+      "tokens_seen": 2764111872
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016401861659444712,
+      "loss": 1.22,
+      "theoretical_loss": 3.370374307657266,
+      "tokens_seen": 2764111872
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001639784946236559,
+      "loss": 1.2211,
+      "theoretical_loss": 3.370361920025364,
+      "tokens_seen": 2764242944
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001639383726528647,
+      "loss": 1.1437,
+      "theoretical_loss": 3.3703495331452906,
+      "tokens_seen": 2764374016
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001638982506820735,
+      "loss": 1.2321,
+      "theoretical_loss": 3.370337147016964,
+      "tokens_seen": 2764505088
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016385812871128232,
+      "loss": 1.1887,
+      "theoretical_loss": 3.370324761640304,
+      "tokens_seen": 2764636160
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001638180067404911,
+      "loss": 1.196,
+      "theoretical_loss": 3.3703123770152286,
+      "tokens_seen": 2764767232
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001637778847696999,
+      "loss": 1.2078,
+      "theoretical_loss": 3.3702999931416566,
+      "tokens_seen": 2764898304
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016373776279890868,
+      "loss": 1.1389,
+      "theoretical_loss": 3.370287610019507,
+      "tokens_seen": 2765029376
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016369764082811746,
+      "loss": 1.1918,
+      "theoretical_loss": 3.3702752276486985,
+      "tokens_seen": 2765160448
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016365751885732627,
+      "loss": 1.1673,
+      "theoretical_loss": 3.3702628460291497,
+      "tokens_seen": 2765291520
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016361739688653507,
+      "loss": 1.1668,
+      "theoretical_loss": 3.3702504651607796,
+      "tokens_seen": 2765422592
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016357727491574388,
+      "loss": 1.1609,
+      "theoretical_loss": 3.370238085043507,
+      "tokens_seen": 2765553664
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016353715294495266,
+      "loss": 1.2167,
+      "theoretical_loss": 3.370225705677251,
+      "tokens_seen": 2765684736
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016349703097416146,
+      "loss": 1.182,
+      "theoretical_loss": 3.37021332706193,
+      "tokens_seen": 2765815808
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016345690900337024,
+      "loss": 1.195,
+      "theoretical_loss": 3.3702009491974634,
+      "tokens_seen": 2765946880
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016341678703257902,
+      "loss": 1.2221,
+      "theoretical_loss": 3.3701885720837694,
+      "tokens_seen": 2766077952
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016337666506178783,
+      "loss": 1.2538,
+      "theoretical_loss": 3.370176195720767,
+      "tokens_seen": 2766209024
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016333654309099664,
+      "loss": 1.1811,
+      "theoretical_loss": 3.3701638201083757,
+      "tokens_seen": 2766340096
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016329642112020544,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3701514452465138,
+      "tokens_seen": 2766471168
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016325629914941422,
+      "loss": 1.1848,
+      "theoretical_loss": 3.3701390711351005,
+      "tokens_seen": 2766602240
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016321617717862303,
+      "loss": 1.1828,
+      "theoretical_loss": 3.370126697774055,
+      "tokens_seen": 2766733312
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016317605520783183,
+      "loss": 1.1662,
+      "theoretical_loss": 3.370114325163296,
+      "tokens_seen": 2766864384
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016313593323704059,
+      "loss": 1.2279,
+      "theoretical_loss": 3.370101953302742,
+      "tokens_seen": 2766995456
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001630958112662494,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3700895821923127,
+      "tokens_seen": 2767126528
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001630556892954582,
+      "loss": 1.2262,
+      "theoretical_loss": 3.370077211831927,
+      "tokens_seen": 2767257600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016301556732466698,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3700648422215034,
+      "tokens_seen": 2767388672
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016297544535387578,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3700524733609614,
+      "tokens_seen": 2767519744
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001629353233830846,
+      "loss": 1.2387,
+      "theoretical_loss": 3.3700401052502196,
+      "tokens_seen": 2767650816
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001628952014122934,
+      "loss": 1.2163,
+      "theoretical_loss": 3.370027737889197,
+      "tokens_seen": 2767781888
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016285507944150215,
+      "loss": 1.1618,
+      "theoretical_loss": 3.370015371277814,
+      "tokens_seen": 2767912960
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016281495747071096,
+      "loss": 1.2572,
+      "theoretical_loss": 3.3700030054159877,
+      "tokens_seen": 2768044032
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016277483549991976,
+      "loss": 1.1824,
+      "theoretical_loss": 3.369990640303638,
+      "tokens_seen": 2768175104
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016273471352912854,
+      "loss": 1.1576,
+      "theoretical_loss": 3.369978275940685,
+      "tokens_seen": 2768306176
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016269459155833735,
+      "loss": 1.1327,
+      "theoretical_loss": 3.3699659123270465,
+      "tokens_seen": 2768437248
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016265446958754615,
+      "loss": 1.1836,
+      "theoretical_loss": 3.3699535494626414,
+      "tokens_seen": 2768568320
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016261434761675496,
+      "loss": 1.2201,
+      "theoretical_loss": 3.36994118734739,
+      "tokens_seen": 2768699392
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016257422564596374,
+      "loss": 1.1763,
+      "theoretical_loss": 3.369928825981211,
+      "tokens_seen": 2768830464
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016253410367517252,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3699164653640232,
+      "tokens_seen": 2768961536
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016249398170438132,
+      "loss": 1.2406,
+      "theoretical_loss": 3.369904105495746,
+      "tokens_seen": 2769092608
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001624538597335901,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3698917463762985,
+      "tokens_seen": 2769223680
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001624137377627989,
+      "loss": 1.0942,
+      "theoretical_loss": 3.3698793880055997,
+      "tokens_seen": 2769354752
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016237361579200772,
+      "loss": 1.1771,
+      "theoretical_loss": 3.3698670303835696,
+      "tokens_seen": 2769485824
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016233349382121652,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3698546735101265,
+      "tokens_seen": 2769616896
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001622933718504253,
+      "loss": 1.1618,
+      "theoretical_loss": 3.3698423173851904,
+      "tokens_seen": 2769747968
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016225324987963408,
+      "loss": 1.231,
+      "theoretical_loss": 3.36982996200868,
+      "tokens_seen": 2769879040
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001622131279088429,
+      "loss": 1.2036,
+      "theoretical_loss": 3.369817607380514,
+      "tokens_seen": 2770010112
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016217300593805167,
+      "loss": 1.2018,
+      "theoretical_loss": 3.3698052535006133,
+      "tokens_seen": 2770141184
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016213288396726047,
+      "loss": 1.1883,
+      "theoretical_loss": 3.3697929003688962,
+      "tokens_seen": 2770272256
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016209276199646928,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3697805479852816,
+      "tokens_seen": 2770403328
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016205264002567809,
+      "loss": 1.2025,
+      "theoretical_loss": 3.3697681963496895,
+      "tokens_seen": 2770534400
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 929566,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2639553546905518,
+      "objective/train/theoretical_loss": 3.3697558454620387,
+      "objective/train/tokens_used": 1141124576,
+      "theoretical_loss": 3.3697558454620387,
+      "tokens_seen": 2770665472
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016201251805488686,
+      "loss": 1.2555,
+      "theoretical_loss": 3.3697558454620387,
+      "tokens_seen": 2770665472
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016197239608409564,
+      "loss": 1.1489,
+      "theoretical_loss": 3.369743495322249,
+      "tokens_seen": 2770796544
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016193227411330445,
+      "loss": 1.1941,
+      "theoretical_loss": 3.3697311459302397,
+      "tokens_seen": 2770927616
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016189215214251323,
+      "loss": 1.1845,
+      "theoretical_loss": 3.36971879728593,
+      "tokens_seen": 2771058688
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016185203017172204,
+      "loss": 1.2506,
+      "theoretical_loss": 3.369706449389239,
+      "tokens_seen": 2771189760
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016181190820093084,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3696941022400866,
+      "tokens_seen": 2771320832
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016177178623013962,
+      "loss": 1.2501,
+      "theoretical_loss": 3.369681755838392,
+      "tokens_seen": 2771451904
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016173166425934843,
+      "loss": 1.1918,
+      "theoretical_loss": 3.369669410184074,
+      "tokens_seen": 2771582976
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016169154228855723,
+      "loss": 1.2062,
+      "theoretical_loss": 3.3696570652770528,
+      "tokens_seen": 2771714048
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.000161651420317766,
+      "loss": 1.1144,
+      "theoretical_loss": 3.3696447211172478,
+      "tokens_seen": 2771845120
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001616112983469748,
+      "loss": 1.2553,
+      "theoretical_loss": 3.369632377704578,
+      "tokens_seen": 2771976192
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001615711763761836,
+      "loss": 1.2218,
+      "theoretical_loss": 3.369620035038963,
+      "tokens_seen": 2772107264
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001615310544053924,
+      "loss": 1.1688,
+      "theoretical_loss": 3.3696076931203227,
+      "tokens_seen": 2772238336
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016149093243460118,
+      "loss": 1.2463,
+      "theoretical_loss": 3.369595351948576,
+      "tokens_seen": 2772369408
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016145081046381,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3695830115236425,
+      "tokens_seen": 2772500480
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001614106884930188,
+      "loss": 1.2603,
+      "theoretical_loss": 3.369570671845442,
+      "tokens_seen": 2772631552
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016137056652222758,
+      "loss": 1.255,
+      "theoretical_loss": 3.369558332913894,
+      "tokens_seen": 2772762624
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016133044455143635,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3695459947289175,
+      "tokens_seen": 2772893696
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016129032258064516,
+      "loss": 1.2519,
+      "theoretical_loss": 3.3695336572904324,
+      "tokens_seen": 2773024768
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016125020060985397,
+      "loss": 1.1875,
+      "theoretical_loss": 3.3695213205983587,
+      "tokens_seen": 2773155840
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016121007863906275,
+      "loss": 1.1633,
+      "theoretical_loss": 3.369508984652615,
+      "tokens_seen": 2773286912
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016116995666827155,
+      "loss": 1.1227,
+      "theoretical_loss": 3.3694966494531218,
+      "tokens_seen": 2773417984
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016112983469748036,
+      "loss": 1.1809,
+      "theoretical_loss": 3.3694843149997986,
+      "tokens_seen": 2773549056
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001610897127266891,
+      "loss": 1.1474,
+      "theoretical_loss": 3.369471981292564,
+      "tokens_seen": 2773680128
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016104959075589792,
+      "loss": 1.2287,
+      "theoretical_loss": 3.369459648331339,
+      "tokens_seen": 2773811200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016100946878510672,
+      "loss": 1.2181,
+      "theoretical_loss": 3.369447316116042,
+      "tokens_seen": 2773942272
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016096934681431553,
+      "loss": 1.1932,
+      "theoretical_loss": 3.369434984646593,
+      "tokens_seen": 2774073344
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001609292248435243,
+      "loss": 1.1687,
+      "theoretical_loss": 3.3694226539229124,
+      "tokens_seen": 2774204416
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016088910287273312,
+      "loss": 1.1548,
+      "theoretical_loss": 3.369410323944919,
+      "tokens_seen": 2774335488
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016084898090194192,
+      "loss": 1.1708,
+      "theoretical_loss": 3.369397994712533,
+      "tokens_seen": 2774466560
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001608088589311507,
+      "loss": 1.0706,
+      "theoretical_loss": 3.369385666225674,
+      "tokens_seen": 2774597632
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016076873696035948,
+      "loss": 1.1674,
+      "theoretical_loss": 3.3693733384842615,
+      "tokens_seen": 2774728704
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001607286149895683,
+      "loss": 1.1529,
+      "theoretical_loss": 3.369361011488215,
+      "tokens_seen": 2774859776
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001606884930187771,
+      "loss": 1.1083,
+      "theoretical_loss": 3.369348685237455,
+      "tokens_seen": 2774990848
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016064837104798587,
+      "loss": 1.1245,
+      "theoretical_loss": 3.3693363597319,
+      "tokens_seen": 2775121920
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016060824907719468,
+      "loss": 1.1411,
+      "theoretical_loss": 3.3693240349714713,
+      "tokens_seen": 2775252992
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016056812710640348,
+      "loss": 1.236,
+      "theoretical_loss": 3.3693117109560875,
+      "tokens_seen": 2775384064
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016052800513561226,
+      "loss": 1.2452,
+      "theoretical_loss": 3.3692993876856687,
+      "tokens_seen": 2775515136
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016048788316482104,
+      "loss": 1.2233,
+      "theoretical_loss": 3.369287065160135,
+      "tokens_seen": 2775646208
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016044776119402985,
+      "loss": 1.1512,
+      "theoretical_loss": 3.369274743379406,
+      "tokens_seen": 2775777280
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016040763922323866,
+      "loss": 1.161,
+      "theoretical_loss": 3.3692624223434016,
+      "tokens_seen": 2775908352
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016036751725244744,
+      "loss": 1.1594,
+      "theoretical_loss": 3.369250102052041,
+      "tokens_seen": 2776039424
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016032739528165624,
+      "loss": 1.1487,
+      "theoretical_loss": 3.369237782505245,
+      "tokens_seen": 2776170496
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016028727331086505,
+      "loss": 1.2031,
+      "theoretical_loss": 3.369225463702933,
+      "tokens_seen": 2776301568
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016024715134007383,
+      "loss": 1.1829,
+      "theoretical_loss": 3.3692131456450247,
+      "tokens_seen": 2776432640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001602070293692826,
+      "loss": 1.2213,
+      "theoretical_loss": 3.36920082833144,
+      "tokens_seen": 2776563712
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001601669073984914,
+      "loss": 1.2351,
+      "theoretical_loss": 3.369188511762099,
+      "tokens_seen": 2776694784
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016012678542770022,
+      "loss": 1.2426,
+      "theoretical_loss": 3.369176195936922,
+      "tokens_seen": 2776825856
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.000160086663456909,
+      "loss": 1.1785,
+      "theoretical_loss": 3.369163880855828,
+      "tokens_seen": 2776956928
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001600465414861178,
+      "loss": 1.265,
+      "theoretical_loss": 3.3691515665187373,
+      "tokens_seen": 2777088000
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 931397,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.240534782409668,
+      "objective/train/theoretical_loss": 3.3691392529255704,
+      "objective/train/tokens_used": 1147678176,
+      "theoretical_loss": 3.3691392529255704,
+      "tokens_seen": 2777219072
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001600064195153266,
+      "loss": 1.187,
+      "theoretical_loss": 3.3691392529255704,
+      "tokens_seen": 2777219072
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001599662975445354,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3691269400762467,
+      "tokens_seen": 2777350144
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001599261755737442,
+      "loss": 1.1608,
+      "theoretical_loss": 3.369114627970686,
+      "tokens_seen": 2777481216
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015988605360295298,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3691023166088083,
+      "tokens_seen": 2777612288
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015984593163216175,
+      "loss": 1.2089,
+      "theoretical_loss": 3.369090005990534,
+      "tokens_seen": 2777743360
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015980580966137056,
+      "loss": 1.2163,
+      "theoretical_loss": 3.369077696115783,
+      "tokens_seen": 2777874432
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015976568769057937,
+      "loss": 1.2033,
+      "theoretical_loss": 3.369065386984475,
+      "tokens_seen": 2778005504
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015972556571978817,
+      "loss": 1.23,
+      "theoretical_loss": 3.3690530785965307,
+      "tokens_seen": 2778136576
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015968544374899695,
+      "loss": 1.2022,
+      "theoretical_loss": 3.3690407709518695,
+      "tokens_seen": 2778267648
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015964532177820576,
+      "loss": 1.2382,
+      "theoretical_loss": 3.369028464050411,
+      "tokens_seen": 2778398720
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015960519980741454,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3690161578920765,
+      "tokens_seen": 2778529792
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015956507783662332,
+      "loss": 1.1485,
+      "theoretical_loss": 3.369003852476786,
+      "tokens_seen": 2778660864
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015952495586583212,
+      "loss": 1.2019,
+      "theoretical_loss": 3.368991547804458,
+      "tokens_seen": 2778791936
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015948483389504093,
+      "loss": 1.1223,
+      "theoretical_loss": 3.3689792438750144,
+      "tokens_seen": 2778923008
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015944471192424974,
+      "loss": 1.2133,
+      "theoretical_loss": 3.368966940688374,
+      "tokens_seen": 2779054080
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015940458995345852,
+      "loss": 1.2187,
+      "theoretical_loss": 3.368954638244458,
+      "tokens_seen": 2779185152
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015936446798266732,
+      "loss": 1.1857,
+      "theoretical_loss": 3.368942336543186,
+      "tokens_seen": 2779316224
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015932434601187613,
+      "loss": 1.1487,
+      "theoretical_loss": 3.3689300355844782,
+      "tokens_seen": 2779447296
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015928422404108488,
+      "loss": 1.1441,
+      "theoretical_loss": 3.3689177353682545,
+      "tokens_seen": 2779578368
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001592441020702937,
+      "loss": 1.2016,
+      "theoretical_loss": 3.3689054358944355,
+      "tokens_seen": 2779709440
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001592039800995025,
+      "loss": 1.1874,
+      "theoretical_loss": 3.368893137162941,
+      "tokens_seen": 2779840512
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001591638581287113,
+      "loss": 1.1532,
+      "theoretical_loss": 3.3688808391736913,
+      "tokens_seen": 2779971584
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015912373615792008,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3688685419266076,
+      "tokens_seen": 2780102656
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015908361418712888,
+      "loss": 1.1514,
+      "theoretical_loss": 3.3688562454216084,
+      "tokens_seen": 2780233728
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001590434922163377,
+      "loss": 1.2344,
+      "theoretical_loss": 3.368843949658615,
+      "tokens_seen": 2780364800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015900337024554644,
+      "loss": 1.1557,
+      "theoretical_loss": 3.3688316546375474,
+      "tokens_seen": 2780495872
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015896324827475525,
+      "loss": 1.2791,
+      "theoretical_loss": 3.368819360358326,
+      "tokens_seen": 2780626944
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015892312630396406,
+      "loss": 1.1763,
+      "theoretical_loss": 3.368807066820871,
+      "tokens_seen": 2780758016
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015888300433317283,
+      "loss": 1.1541,
+      "theoretical_loss": 3.3687947740251025,
+      "tokens_seen": 2780889088
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015884288236238164,
+      "loss": 1.2344,
+      "theoretical_loss": 3.368782481970941,
+      "tokens_seen": 2781020160
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015880276039159045,
+      "loss": 1.2776,
+      "theoretical_loss": 3.368770190658307,
+      "tokens_seen": 2781151232
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015876263842079925,
+      "loss": 1.1535,
+      "theoretical_loss": 3.3687579000871204,
+      "tokens_seen": 2781282304
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.000158722516450008,
+      "loss": 1.2758,
+      "theoretical_loss": 3.3687456102573017,
+      "tokens_seen": 2781413376
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001586823944792168,
+      "loss": 1.2151,
+      "theoretical_loss": 3.3687333211687713,
+      "tokens_seen": 2781544448
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015864227250842562,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3687210328214494,
+      "tokens_seen": 2781675520
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001586021505376344,
+      "loss": 1.2541,
+      "theoretical_loss": 3.368708745215257,
+      "tokens_seen": 2781806592
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001585620285668432,
+      "loss": 1.2166,
+      "theoretical_loss": 3.3686964583501133,
+      "tokens_seen": 2781937664
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.000158521906596052,
+      "loss": 1.1776,
+      "theoretical_loss": 3.3686841722259397,
+      "tokens_seen": 2782068736
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015848178462526082,
+      "loss": 1.2342,
+      "theoretical_loss": 3.368671886842656,
+      "tokens_seen": 2782199808
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001584416626544696,
+      "loss": 1.1786,
+      "theoretical_loss": 3.3686596022001836,
+      "tokens_seen": 2782330880
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015840154068367837,
+      "loss": 1.215,
+      "theoretical_loss": 3.368647318298442,
+      "tokens_seen": 2782461952
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015836141871288718,
+      "loss": 1.2752,
+      "theoretical_loss": 3.3686350351373515,
+      "tokens_seen": 2782593024
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015832129674209596,
+      "loss": 1.2071,
+      "theoretical_loss": 3.368622752716833,
+      "tokens_seen": 2782724096
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015828117477130477,
+      "loss": 1.2204,
+      "theoretical_loss": 3.368610471036807,
+      "tokens_seen": 2782855168
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015824105280051357,
+      "loss": 1.1743,
+      "theoretical_loss": 3.3685981900971944,
+      "tokens_seen": 2782986240
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015820093082972238,
+      "loss": 1.2987,
+      "theoretical_loss": 3.3685859098979147,
+      "tokens_seen": 2783117312
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015816080885893116,
+      "loss": 1.2606,
+      "theoretical_loss": 3.368573630438889,
+      "tokens_seen": 2783248384
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015812068688813994,
+      "loss": 1.1907,
+      "theoretical_loss": 3.3685613517200377,
+      "tokens_seen": 2783379456
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015808056491734874,
+      "loss": 1.2533,
+      "theoretical_loss": 3.3685490737412818,
+      "tokens_seen": 2783510528
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015804044294655752,
+      "loss": 1.1819,
+      "theoretical_loss": 3.368536796502541,
+      "tokens_seen": 2783641600
+    },
+    {
+      "epoch": 0.69,
+      "objective/train/docs_used": 933616,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1856707334518433,
+      "objective/train/theoretical_loss": 3.3685245200037364,
+      "objective/train/tokens_used": 1154231776,
+      "theoretical_loss": 3.3685245200037364,
+      "tokens_seen": 2783772672
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015800032097576633,
+      "loss": 1.2064,
+      "theoretical_loss": 3.3685245200037364,
+      "tokens_seen": 2783772672
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015796019900497514,
+      "loss": 1.1838,
+      "theoretical_loss": 3.368512244244788,
+      "tokens_seen": 2783903744
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015792007703418394,
+      "loss": 1.2458,
+      "theoretical_loss": 3.368499969225617,
+      "tokens_seen": 2784034816
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015787995506339272,
+      "loss": 1.2606,
+      "theoretical_loss": 3.368487694946144,
+      "tokens_seen": 2784165888
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001578398330926015,
+      "loss": 1.1512,
+      "theoretical_loss": 3.3684754214062895,
+      "tokens_seen": 2784296960
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001577997111218103,
+      "loss": 1.2138,
+      "theoretical_loss": 3.368463148605974,
+      "tokens_seen": 2784428032
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015775958915101909,
+      "loss": 1.1942,
+      "theoretical_loss": 3.3684508765451175,
+      "tokens_seen": 2784559104
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001577194671802279,
+      "loss": 1.2163,
+      "theoretical_loss": 3.368438605223642,
+      "tokens_seen": 2784690176
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001576793452094367,
+      "loss": 1.2192,
+      "theoretical_loss": 3.3684263346414673,
+      "tokens_seen": 2784821248
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015763922323864548,
+      "loss": 1.2303,
+      "theoretical_loss": 3.368414064798514,
+      "tokens_seen": 2784952320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015759910126785428,
+      "loss": 1.2995,
+      "theoretical_loss": 3.3684017956947034,
+      "tokens_seen": 2785083392
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001575589792970631,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3683895273299553,
+      "tokens_seen": 2785214464
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015751885732627187,
+      "loss": 1.1837,
+      "theoretical_loss": 3.368377259704191,
+      "tokens_seen": 2785345536
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015747873535548065,
+      "loss": 1.2912,
+      "theoretical_loss": 3.3683649928173316,
+      "tokens_seen": 2785476608
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015743861338468946,
+      "loss": 1.3111,
+      "theoretical_loss": 3.368352726669297,
+      "tokens_seen": 2785607680
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015739849141389826,
+      "loss": 1.2126,
+      "theoretical_loss": 3.368340461260008,
+      "tokens_seen": 2785738752
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015735836944310704,
+      "loss": 1.2849,
+      "theoretical_loss": 3.368328196589386,
+      "tokens_seen": 2785869824
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015731824747231585,
+      "loss": 1.2685,
+      "theoretical_loss": 3.3683159326573513,
+      "tokens_seen": 2786000896
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015727812550152465,
+      "loss": 1.265,
+      "theoretical_loss": 3.368303669463825,
+      "tokens_seen": 2786131968
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015723800353073343,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3682914070087273,
+      "tokens_seen": 2786263040
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001571978815599422,
+      "loss": 1.2855,
+      "theoretical_loss": 3.368279145291979,
+      "tokens_seen": 2786394112
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015715775958915102,
+      "loss": 1.19,
+      "theoretical_loss": 3.3682668843135017,
+      "tokens_seen": 2786525184
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015711763761835982,
+      "loss": 1.2547,
+      "theoretical_loss": 3.368254624073216,
+      "tokens_seen": 2786656256
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001570775156475686,
+      "loss": 1.2813,
+      "theoretical_loss": 3.368242364571042,
+      "tokens_seen": 2786787328
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001570373936767774,
+      "loss": 1.3106,
+      "theoretical_loss": 3.368230105806902,
+      "tokens_seen": 2786918400
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015699727170598622,
+      "loss": 1.2701,
+      "theoretical_loss": 3.3682178477807154,
+      "tokens_seen": 2787049472
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015695714973519502,
+      "loss": 1.2144,
+      "theoretical_loss": 3.3682055904924035,
+      "tokens_seen": 2787180544
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015691702776440377,
+      "loss": 1.2694,
+      "theoretical_loss": 3.3681933339418872,
+      "tokens_seen": 2787311616
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015687690579361258,
+      "loss": 1.1742,
+      "theoretical_loss": 3.368181078129088,
+      "tokens_seen": 2787442688
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001568367838228214,
+      "loss": 1.2252,
+      "theoretical_loss": 3.368168823053926,
+      "tokens_seen": 2787573760
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015679666185203017,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3681565687163224,
+      "tokens_seen": 2787704832
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015675653988123897,
+      "loss": 1.2371,
+      "theoretical_loss": 3.368144315116198,
+      "tokens_seen": 2787835904
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015671641791044778,
+      "loss": 1.2723,
+      "theoretical_loss": 3.3681320622534745,
+      "tokens_seen": 2787966976
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015667629593965656,
+      "loss": 1.1542,
+      "theoretical_loss": 3.368119810128072,
+      "tokens_seen": 2788098048
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015663617396886534,
+      "loss": 1.3441,
+      "theoretical_loss": 3.368107558739912,
+      "tokens_seen": 2788229120
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015659605199807414,
+      "loss": 1.3078,
+      "theoretical_loss": 3.3680953080889147,
+      "tokens_seen": 2788360192
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015655593002728295,
+      "loss": 1.2267,
+      "theoretical_loss": 3.368083058175002,
+      "tokens_seen": 2788491264
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015651580805649173,
+      "loss": 1.163,
+      "theoretical_loss": 3.368070808998094,
+      "tokens_seen": 2788622336
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015647568608570054,
+      "loss": 1.2735,
+      "theoretical_loss": 3.3680585605581133,
+      "tokens_seen": 2788753408
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015643556411490934,
+      "loss": 1.2639,
+      "theoretical_loss": 3.368046312854979,
+      "tokens_seen": 2788884480
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015639544214411812,
+      "loss": 1.2279,
+      "theoretical_loss": 3.3680340658886134,
+      "tokens_seen": 2789015552
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001563553201733269,
+      "loss": 1.1773,
+      "theoretical_loss": 3.3680218196589373,
+      "tokens_seen": 2789146624
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001563151982025357,
+      "loss": 1.3451,
+      "theoretical_loss": 3.3680095741658715,
+      "tokens_seen": 2789277696
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001562750762317445,
+      "loss": 1.2562,
+      "theoretical_loss": 3.367997329409337,
+      "tokens_seen": 2789408768
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001562349542609533,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3679850853892557,
+      "tokens_seen": 2789539840
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001561948322901621,
+      "loss": 1.2338,
+      "theoretical_loss": 3.367972842105548,
+      "tokens_seen": 2789670912
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001561547103193709,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3679605995581348,
+      "tokens_seen": 2789801984
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015611458834857968,
+      "loss": 1.2374,
+      "theoretical_loss": 3.367948357746938,
+      "tokens_seen": 2789933056
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001560744663777885,
+      "loss": 1.2643,
+      "theoretical_loss": 3.3679361166718786,
+      "tokens_seen": 2790064128
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015603434440699727,
+      "loss": 1.2288,
+      "theoretical_loss": 3.3679238763328767,
+      "tokens_seen": 2790195200
+    },
+    {
+      "epoch": 0.69,
+      "objective/train/docs_used": 935680,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1978633403778076,
+      "objective/train/theoretical_loss": 3.367911636729855,
+      "objective/train/tokens_used": 1160785376,
+      "theoretical_loss": 3.367911636729855,
+      "tokens_seen": 2790326272
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015599422243620608,
+      "loss": 1.2261,
+      "theoretical_loss": 3.367911636729855,
+      "tokens_seen": 2790326272
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015595410046541485,
+      "loss": 1.1781,
+      "theoretical_loss": 3.3678993978627334,
+      "tokens_seen": 2790457344
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015591397849462366,
+      "loss": 1.2161,
+      "theoretical_loss": 3.367887159731434,
+      "tokens_seen": 2790588416
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015587385652383247,
+      "loss": 1.1666,
+      "theoretical_loss": 3.3678749223358775,
+      "tokens_seen": 2790719488
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015583373455304125,
+      "loss": 1.2047,
+      "theoretical_loss": 3.3678626856759855,
+      "tokens_seen": 2790850560
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015579361258225005,
+      "loss": 1.2515,
+      "theoretical_loss": 3.3678504497516784,
+      "tokens_seen": 2790981632
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015575349061145883,
+      "loss": 1.1869,
+      "theoretical_loss": 3.3678382145628785,
+      "tokens_seen": 2791112704
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001557133686406676,
+      "loss": 1.2791,
+      "theoretical_loss": 3.3678259801095063,
+      "tokens_seen": 2791243776
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015567324666987642,
+      "loss": 1.1556,
+      "theoretical_loss": 3.3678137463914837,
+      "tokens_seen": 2791374848
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015563312469908522,
+      "loss": 1.1676,
+      "theoretical_loss": 3.3678015134087316,
+      "tokens_seen": 2791505920
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015559300272829403,
+      "loss": 1.2494,
+      "theoretical_loss": 3.367789281161171,
+      "tokens_seen": 2791636992
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001555528807575028,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3677770496487236,
+      "tokens_seen": 2791768064
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015551275878671162,
+      "loss": 1.2019,
+      "theoretical_loss": 3.3677648188713105,
+      "tokens_seen": 2791899136
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001554726368159204,
+      "loss": 1.1729,
+      "theoretical_loss": 3.3677525888288535,
+      "tokens_seen": 2792030208
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015543251484512917,
+      "loss": 1.1867,
+      "theoretical_loss": 3.367740359521273,
+      "tokens_seen": 2792161280
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015539239287433798,
+      "loss": 1.1769,
+      "theoretical_loss": 3.3677281309484917,
+      "tokens_seen": 2792292352
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001553522709035468,
+      "loss": 1.2767,
+      "theoretical_loss": 3.3677159031104296,
+      "tokens_seen": 2792423424
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001553121489327556,
+      "loss": 1.2031,
+      "theoretical_loss": 3.3677036760070087,
+      "tokens_seen": 2792554496
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015527202696196437,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3676914496381505,
+      "tokens_seen": 2792685568
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015523190499117318,
+      "loss": 1.1655,
+      "theoretical_loss": 3.3676792240037763,
+      "tokens_seen": 2792816640
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015519178302038198,
+      "loss": 1.2341,
+      "theoretical_loss": 3.367666999103807,
+      "tokens_seen": 2792947712
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015515166104959074,
+      "loss": 1.2819,
+      "theoretical_loss": 3.367654774938165,
+      "tokens_seen": 2793078784
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015511153907879954,
+      "loss": 1.184,
+      "theoretical_loss": 3.367642551506771,
+      "tokens_seen": 2793209856
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015507141710800835,
+      "loss": 1.1973,
+      "theoretical_loss": 3.367630328809547,
+      "tokens_seen": 2793340928
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015503129513721716,
+      "loss": 1.2744,
+      "theoretical_loss": 3.3676181068464137,
+      "tokens_seen": 2793472000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015499117316642594,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3676058856172926,
+      "tokens_seen": 2793603072
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015495105119563474,
+      "loss": 1.199,
+      "theoretical_loss": 3.367593665122106,
+      "tokens_seen": 2793734144
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015491092922484355,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3675814453607753,
+      "tokens_seen": 2793865216
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001548708072540523,
+      "loss": 1.2537,
+      "theoretical_loss": 3.367569226333221,
+      "tokens_seen": 2793996288
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001548306852832611,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3675570080393658,
+      "tokens_seen": 2794127360
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001547905633124699,
+      "loss": 1.262,
+      "theoretical_loss": 3.3675447904791307,
+      "tokens_seen": 2794258432
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001547504413416787,
+      "loss": 1.2139,
+      "theoretical_loss": 3.367532573652437,
+      "tokens_seen": 2794389504
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001547103193708875,
+      "loss": 1.2268,
+      "theoretical_loss": 3.3675203575592065,
+      "tokens_seen": 2794520576
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001546701974000963,
+      "loss": 1.2949,
+      "theoretical_loss": 3.3675081421993607,
+      "tokens_seen": 2794651648
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001546300754293051,
+      "loss": 1.263,
+      "theoretical_loss": 3.3674959275728216,
+      "tokens_seen": 2794782720
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001545899534585139,
+      "loss": 1.2313,
+      "theoretical_loss": 3.36748371367951,
+      "tokens_seen": 2794913792
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015454983148772267,
+      "loss": 1.3299,
+      "theoretical_loss": 3.367471500519348,
+      "tokens_seen": 2795044864
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015450970951693148,
+      "loss": 1.1902,
+      "theoretical_loss": 3.3674592880922574,
+      "tokens_seen": 2795175936
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015446958754614025,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3674470763981597,
+      "tokens_seen": 2795307008
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015442946557534906,
+      "loss": 1.2635,
+      "theoretical_loss": 3.367434865436976,
+      "tokens_seen": 2795438080
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015438934360455787,
+      "loss": 1.1824,
+      "theoretical_loss": 3.3674226552086286,
+      "tokens_seen": 2795569152
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015434922163376667,
+      "loss": 1.2055,
+      "theoretical_loss": 3.367410445713039,
+      "tokens_seen": 2795700224
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015430909966297545,
+      "loss": 1.2641,
+      "theoretical_loss": 3.3673982369501285,
+      "tokens_seen": 2795831296
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015426897769218423,
+      "loss": 1.1568,
+      "theoretical_loss": 3.367386028919819,
+      "tokens_seen": 2795962368
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015422885572139304,
+      "loss": 1.1801,
+      "theoretical_loss": 3.3673738216220332,
+      "tokens_seen": 2796093440
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015418873375060182,
+      "loss": 1.1917,
+      "theoretical_loss": 3.367361615056691,
+      "tokens_seen": 2796224512
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015414861177981062,
+      "loss": 1.1806,
+      "theoretical_loss": 3.3673494092237157,
+      "tokens_seen": 2796355584
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015410848980901943,
+      "loss": 1.232,
+      "theoretical_loss": 3.367337204123028,
+      "tokens_seen": 2796486656
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015406836783822824,
+      "loss": 1.2578,
+      "theoretical_loss": 3.36732499975455,
+      "tokens_seen": 2796617728
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015402824586743702,
+      "loss": 1.2258,
+      "theoretical_loss": 3.3673127961182034,
+      "tokens_seen": 2796748800
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 937796,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1523512601852417,
+      "objective/train/theoretical_loss": 3.36730059321391,
+      "objective/train/tokens_used": 1167338976,
+      "theoretical_loss": 3.36730059321391,
+      "tokens_seen": 2796879872
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001539881238966458,
+      "loss": 1.1801,
+      "theoretical_loss": 3.36730059321391,
+      "tokens_seen": 2796879872
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001539480019258546,
+      "loss": 1.1837,
+      "theoretical_loss": 3.367288391041592,
+      "tokens_seen": 2797010944
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015390787995506338,
+      "loss": 1.2315,
+      "theoretical_loss": 3.3672761896011707,
+      "tokens_seen": 2797142016
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001538677579842722,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3672639888925677,
+      "tokens_seen": 2797273088
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.000153827636013481,
+      "loss": 1.1853,
+      "theoretical_loss": 3.367251788915705,
+      "tokens_seen": 2797404160
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001537875140426898,
+      "loss": 1.2126,
+      "theoretical_loss": 3.367239589670505,
+      "tokens_seen": 2797535232
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015374739207189858,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3672273911568893,
+      "tokens_seen": 2797666304
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015370727010110738,
+      "loss": 1.1726,
+      "theoretical_loss": 3.367215193374779,
+      "tokens_seen": 2797797376
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015366714813031616,
+      "loss": 1.2228,
+      "theoretical_loss": 3.367202996324097,
+      "tokens_seen": 2797928448
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015362702615952494,
+      "loss": 1.2407,
+      "theoretical_loss": 3.3671908000047646,
+      "tokens_seen": 2798059520
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015358690418873375,
+      "loss": 1.2254,
+      "theoretical_loss": 3.367178604416704,
+      "tokens_seen": 2798190592
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015354678221794256,
+      "loss": 1.1565,
+      "theoretical_loss": 3.367166409559837,
+      "tokens_seen": 2798321664
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015350666024715133,
+      "loss": 1.249,
+      "theoretical_loss": 3.3671542154340846,
+      "tokens_seen": 2798452736
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015346653827636014,
+      "loss": 1.2064,
+      "theoretical_loss": 3.36714202203937,
+      "tokens_seen": 2798583808
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015342641630556895,
+      "loss": 1.2386,
+      "theoretical_loss": 3.367129829375615,
+      "tokens_seen": 2798714880
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015338629433477773,
+      "loss": 1.1745,
+      "theoretical_loss": 3.3671176374427407,
+      "tokens_seen": 2798845952
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001533461723639865,
+      "loss": 1.2452,
+      "theoretical_loss": 3.36710544624067,
+      "tokens_seen": 2798977024
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001533060503931953,
+      "loss": 1.2106,
+      "theoretical_loss": 3.3670932557693245,
+      "tokens_seen": 2799108096
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015326592842240412,
+      "loss": 1.2332,
+      "theoretical_loss": 3.3670810660286263,
+      "tokens_seen": 2799239168
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001532258064516129,
+      "loss": 1.2636,
+      "theoretical_loss": 3.367068877018497,
+      "tokens_seen": 2799370240
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001531856844808217,
+      "loss": 1.1898,
+      "theoretical_loss": 3.367056688738859,
+      "tokens_seen": 2799501312
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001531455625100305,
+      "loss": 1.1664,
+      "theoretical_loss": 3.367044501189634,
+      "tokens_seen": 2799632384
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001531054405392393,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3670323143707446,
+      "tokens_seen": 2799763456
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015306531856844807,
+      "loss": 1.186,
+      "theoretical_loss": 3.367020128282112,
+      "tokens_seen": 2799894528
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015302519659765687,
+      "loss": 1.1948,
+      "theoretical_loss": 3.3670079429236592,
+      "tokens_seen": 2800025600
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015298507462686568,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3669957582953076,
+      "tokens_seen": 2800156672
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015294495265607446,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3669835743969796,
+      "tokens_seen": 2800287744
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015290483068528327,
+      "loss": 1.1958,
+      "theoretical_loss": 3.3669713912285975,
+      "tokens_seen": 2800418816
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015286470871449207,
+      "loss": 1.1935,
+      "theoretical_loss": 3.3669592087900826,
+      "tokens_seen": 2800549888
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015282458674370088,
+      "loss": 1.1875,
+      "theoretical_loss": 3.3669470270813577,
+      "tokens_seen": 2800680960
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015278446477290963,
+      "loss": 1.2315,
+      "theoretical_loss": 3.3669348461023447,
+      "tokens_seen": 2800812032
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015274434280211844,
+      "loss": 1.2202,
+      "theoretical_loss": 3.366922665852966,
+      "tokens_seen": 2800943104
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015270422083132724,
+      "loss": 1.1555,
+      "theoretical_loss": 3.3669104863331434,
+      "tokens_seen": 2801074176
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015266409886053602,
+      "loss": 1.1994,
+      "theoretical_loss": 3.366898307542799,
+      "tokens_seen": 2801205248
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015262397688974483,
+      "loss": 1.1568,
+      "theoretical_loss": 3.3668861294818555,
+      "tokens_seen": 2801336320
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015258385491895364,
+      "loss": 1.2354,
+      "theoretical_loss": 3.3668739521502347,
+      "tokens_seen": 2801467392
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015254373294816242,
+      "loss": 1.2533,
+      "theoretical_loss": 3.366861775547859,
+      "tokens_seen": 2801598464
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001525036109773712,
+      "loss": 1.2356,
+      "theoretical_loss": 3.36684959967465,
+      "tokens_seen": 2801729536
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015246348900658,
+      "loss": 1.2197,
+      "theoretical_loss": 3.366837424530531,
+      "tokens_seen": 2801860608
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001524233670357888,
+      "loss": 1.2832,
+      "theoretical_loss": 3.3668252501154234,
+      "tokens_seen": 2801991680
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015238324506499759,
+      "loss": 1.254,
+      "theoretical_loss": 3.3668130764292497,
+      "tokens_seen": 2802122752
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001523431230942064,
+      "loss": 1.1767,
+      "theoretical_loss": 3.3668009034719324,
+      "tokens_seen": 2802253824
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001523030011234152,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3667887312433935,
+      "tokens_seen": 2802384896
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015226287915262398,
+      "loss": 1.2052,
+      "theoretical_loss": 3.3667765597435553,
+      "tokens_seen": 2802515968
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015222275718183278,
+      "loss": 1.1156,
+      "theoretical_loss": 3.36676438897234,
+      "tokens_seen": 2802647040
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015218263521104156,
+      "loss": 1.1894,
+      "theoretical_loss": 3.36675221892967,
+      "tokens_seen": 2802778112
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015214251324025037,
+      "loss": 1.1624,
+      "theoretical_loss": 3.366740049615468,
+      "tokens_seen": 2802909184
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015210239126945915,
+      "loss": 1.1773,
+      "theoretical_loss": 3.3667278810296555,
+      "tokens_seen": 2803040256
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015206226929866796,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3667157131721557,
+      "tokens_seen": 2803171328
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015202214732787676,
+      "loss": 1.2083,
+      "theoretical_loss": 3.3667035460428907,
+      "tokens_seen": 2803302400
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 939819,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1146295070648193,
+      "objective/train/theoretical_loss": 3.3666913796417828,
+      "objective/train/tokens_used": 1173892576,
+      "theoretical_loss": 3.3666913796417828,
+      "tokens_seen": 2803433472
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015198202535708554,
+      "loss": 1.1824,
+      "theoretical_loss": 3.3666913796417828,
+      "tokens_seen": 2803433472
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015194190338629435,
+      "loss": 1.1748,
+      "theoretical_loss": 3.3666792139687542,
+      "tokens_seen": 2803564544
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015190178141550313,
+      "loss": 1.1892,
+      "theoretical_loss": 3.3666670490237274,
+      "tokens_seen": 2803695616
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015186165944471193,
+      "loss": 1.1541,
+      "theoretical_loss": 3.366654884806625,
+      "tokens_seen": 2803826688
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001518215374739207,
+      "loss": 1.2818,
+      "theoretical_loss": 3.366642721317369,
+      "tokens_seen": 2803957760
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015178141550312952,
+      "loss": 1.2427,
+      "theoretical_loss": 3.3666305585558822,
+      "tokens_seen": 2804088832
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015174129353233832,
+      "loss": 1.2132,
+      "theoretical_loss": 3.366618396522087,
+      "tokens_seen": 2804219904
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001517011715615471,
+      "loss": 1.1647,
+      "theoretical_loss": 3.366606235215906,
+      "tokens_seen": 2804350976
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001516610495907559,
+      "loss": 1.1917,
+      "theoretical_loss": 3.3665940746372613,
+      "tokens_seen": 2804482048
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001516209276199647,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3665819147860754,
+      "tokens_seen": 2804613120
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015158080564917347,
+      "loss": 1.1352,
+      "theoretical_loss": 3.366569755662271,
+      "tokens_seen": 2804744192
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015154068367838227,
+      "loss": 1.2074,
+      "theoretical_loss": 3.366557597265771,
+      "tokens_seen": 2804875264
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015150056170759108,
+      "loss": 1.2246,
+      "theoretical_loss": 3.3665454395964964,
+      "tokens_seen": 2805006336
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001514604397367999,
+      "loss": 1.259,
+      "theoretical_loss": 3.3665332826543715,
+      "tokens_seen": 2805137408
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015142031776600867,
+      "loss": 1.1614,
+      "theoretical_loss": 3.3665211264393182,
+      "tokens_seen": 2805268480
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015138019579521747,
+      "loss": 1.2717,
+      "theoretical_loss": 3.3665089709512586,
+      "tokens_seen": 2805399552
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015134007382442628,
+      "loss": 1.2703,
+      "theoretical_loss": 3.3664968161901156,
+      "tokens_seen": 2805530624
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015129995185363503,
+      "loss": 1.2617,
+      "theoretical_loss": 3.3664846621558118,
+      "tokens_seen": 2805661696
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015125982988284384,
+      "loss": 1.1785,
+      "theoretical_loss": 3.36647250884827,
+      "tokens_seen": 2805792768
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015121970791205264,
+      "loss": 1.2077,
+      "theoretical_loss": 3.3664603562674125,
+      "tokens_seen": 2805923840
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015117958594126145,
+      "loss": 1.2205,
+      "theoretical_loss": 3.3664482044131616,
+      "tokens_seen": 2806054912
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015113946397047023,
+      "loss": 1.2802,
+      "theoretical_loss": 3.3664360532854403,
+      "tokens_seen": 2806185984
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015109934199967904,
+      "loss": 1.1801,
+      "theoretical_loss": 3.3664239028841716,
+      "tokens_seen": 2806317056
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015105922002888784,
+      "loss": 1.1883,
+      "theoretical_loss": 3.3664117532092774,
+      "tokens_seen": 2806448128
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001510190980580966,
+      "loss": 1.2145,
+      "theoretical_loss": 3.366399604260681,
+      "tokens_seen": 2806579200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001509789760873054,
+      "loss": 1.2503,
+      "theoretical_loss": 3.366387456038305,
+      "tokens_seen": 2806710272
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001509388541165142,
+      "loss": 1.22,
+      "theoretical_loss": 3.366375308542071,
+      "tokens_seen": 2806841344
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.000150898732145723,
+      "loss": 1.2821,
+      "theoretical_loss": 3.366363161771903,
+      "tokens_seen": 2806972416
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001508586101749318,
+      "loss": 1.2933,
+      "theoretical_loss": 3.3663510157277234,
+      "tokens_seen": 2807103488
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001508184882041406,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3663388704094546,
+      "tokens_seen": 2807234560
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001507783662333494,
+      "loss": 1.1061,
+      "theoretical_loss": 3.3663267258170197,
+      "tokens_seen": 2807365632
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015073824426255816,
+      "loss": 1.1111,
+      "theoretical_loss": 3.3663145819503413,
+      "tokens_seen": 2807496704
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015069812229176696,
+      "loss": 1.2661,
+      "theoretical_loss": 3.3663024388093414,
+      "tokens_seen": 2807627776
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015065800032097577,
+      "loss": 1.155,
+      "theoretical_loss": 3.366290296393944,
+      "tokens_seen": 2807758848
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015061787835018455,
+      "loss": 1.1913,
+      "theoretical_loss": 3.366278154704071,
+      "tokens_seen": 2807889920
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015057775637939335,
+      "loss": 1.2082,
+      "theoretical_loss": 3.366266013739646,
+      "tokens_seen": 2808020992
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015053763440860216,
+      "loss": 1.1858,
+      "theoretical_loss": 3.3662538735005905,
+      "tokens_seen": 2808152064
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015049751243781097,
+      "loss": 1.1459,
+      "theoretical_loss": 3.3662417339868287,
+      "tokens_seen": 2808283136
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015045739046701975,
+      "loss": 1.258,
+      "theoretical_loss": 3.3662295951982824,
+      "tokens_seen": 2808414208
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015041726849622853,
+      "loss": 1.1694,
+      "theoretical_loss": 3.366217457134875,
+      "tokens_seen": 2808545280
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015037714652543733,
+      "loss": 1.2183,
+      "theoretical_loss": 3.3662053197965296,
+      "tokens_seen": 2808676352
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001503370245546461,
+      "loss": 1.2672,
+      "theoretical_loss": 3.366193183183168,
+      "tokens_seen": 2808807424
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015029690258385492,
+      "loss": 1.2618,
+      "theoretical_loss": 3.366181047294714,
+      "tokens_seen": 2808938496
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015025678061306372,
+      "loss": 1.1823,
+      "theoretical_loss": 3.36616891213109,
+      "tokens_seen": 2809069568
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015021665864227253,
+      "loss": 1.2662,
+      "theoretical_loss": 3.366156777692219,
+      "tokens_seen": 2809200640
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001501765366714813,
+      "loss": 1.2655,
+      "theoretical_loss": 3.3661446439780245,
+      "tokens_seen": 2809331712
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001501364147006901,
+      "loss": 1.1892,
+      "theoretical_loss": 3.366132510988429,
+      "tokens_seen": 2809462784
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001500962927298989,
+      "loss": 1.207,
+      "theoretical_loss": 3.3661203787233545,
+      "tokens_seen": 2809593856
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015005617075910767,
+      "loss": 1.2399,
+      "theoretical_loss": 3.3661082471827255,
+      "tokens_seen": 2809724928
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015001604878831648,
+      "loss": 1.1788,
+      "theoretical_loss": 3.3660961163664638,
+      "tokens_seen": 2809856000
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 942090,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.216341257095337,
+      "objective/train/theoretical_loss": 3.366083986274493,
+      "objective/train/tokens_used": 1180446176,
+      "theoretical_loss": 3.366083986274493,
+      "tokens_seen": 2809987072
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001499759268175253,
+      "loss": 1.2128,
+      "theoretical_loss": 3.366083986274493,
+      "tokens_seen": 2809987072
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001499358048467341,
+      "loss": 1.1497,
+      "theoretical_loss": 3.3660718569067356,
+      "tokens_seen": 2810118144
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014989568287594287,
+      "loss": 1.279,
+      "theoretical_loss": 3.3660597282631146,
+      "tokens_seen": 2810249216
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014985556090515168,
+      "loss": 1.2656,
+      "theoretical_loss": 3.366047600343554,
+      "tokens_seen": 2810380288
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014981543893436046,
+      "loss": 1.2514,
+      "theoretical_loss": 3.3660354731479756,
+      "tokens_seen": 2810511360
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014977531696356924,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3660233466763025,
+      "tokens_seen": 2810642432
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014973519499277804,
+      "loss": 1.2971,
+      "theoretical_loss": 3.3660112209284585,
+      "tokens_seen": 2810773504
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014969507302198685,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3659990959043666,
+      "tokens_seen": 2810904576
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014965495105119563,
+      "loss": 1.2197,
+      "theoretical_loss": 3.3659869716039488,
+      "tokens_seen": 2811035648
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014961482908040444,
+      "loss": 1.2967,
+      "theoretical_loss": 3.365974848027129,
+      "tokens_seen": 2811166720
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014957470710961324,
+      "loss": 1.2227,
+      "theoretical_loss": 3.3659627251738304,
+      "tokens_seen": 2811297792
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014953458513882202,
+      "loss": 1.2355,
+      "theoretical_loss": 3.365950603043976,
+      "tokens_seen": 2811428864
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001494944631680308,
+      "loss": 1.2398,
+      "theoretical_loss": 3.365938481637488,
+      "tokens_seen": 2811559936
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001494543411972396,
+      "loss": 1.1612,
+      "theoretical_loss": 3.365926360954291,
+      "tokens_seen": 2811691008
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001494142192264484,
+      "loss": 1.208,
+      "theoretical_loss": 3.365914240994307,
+      "tokens_seen": 2811822080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001493740972556572,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3659021217574594,
+      "tokens_seen": 2811953152
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.000149333975284866,
+      "loss": 1.1825,
+      "theoretical_loss": 3.365890003243672,
+      "tokens_seen": 2812084224
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001492938533140748,
+      "loss": 1.2292,
+      "theoretical_loss": 3.365877885452867,
+      "tokens_seen": 2812215296
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014925373134328358,
+      "loss": 1.1665,
+      "theoretical_loss": 3.3658657683849675,
+      "tokens_seen": 2812346368
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014921360937249236,
+      "loss": 1.2,
+      "theoretical_loss": 3.365853652039898,
+      "tokens_seen": 2812477440
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014917348740170117,
+      "loss": 1.2193,
+      "theoretical_loss": 3.3658415364175807,
+      "tokens_seen": 2812608512
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014913336543090998,
+      "loss": 1.1219,
+      "theoretical_loss": 3.365829421517939,
+      "tokens_seen": 2812739584
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014909324346011875,
+      "loss": 1.2598,
+      "theoretical_loss": 3.3658173073408957,
+      "tokens_seen": 2812870656
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014905312148932756,
+      "loss": 1.2369,
+      "theoretical_loss": 3.365805193886375,
+      "tokens_seen": 2813001728
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014901299951853637,
+      "loss": 1.2646,
+      "theoretical_loss": 3.3657930811542993,
+      "tokens_seen": 2813132800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014897287754774517,
+      "loss": 1.2206,
+      "theoretical_loss": 3.365780969144592,
+      "tokens_seen": 2813263872
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014893275557695393,
+      "loss": 1.1942,
+      "theoretical_loss": 3.3657688578571765,
+      "tokens_seen": 2813394944
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014889263360616273,
+      "loss": 1.2262,
+      "theoretical_loss": 3.3657567472919765,
+      "tokens_seen": 2813526016
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014885251163537154,
+      "loss": 1.2315,
+      "theoretical_loss": 3.3657446374489144,
+      "tokens_seen": 2813657088
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014881238966458032,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3657325283279143,
+      "tokens_seen": 2813788160
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014877226769378912,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3657204199288993,
+      "tokens_seen": 2813919232
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014873214572299793,
+      "loss": 1.1986,
+      "theoretical_loss": 3.365708312251792,
+      "tokens_seen": 2814050304
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014869202375220674,
+      "loss": 1.188,
+      "theoretical_loss": 3.365696205296517,
+      "tokens_seen": 2814181376
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001486519017814155,
+      "loss": 1.1453,
+      "theoretical_loss": 3.365684099062997,
+      "tokens_seen": 2814312448
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001486117798106243,
+      "loss": 1.2894,
+      "theoretical_loss": 3.365671993551155,
+      "tokens_seen": 2814443520
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001485716578398331,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3656598887609146,
+      "tokens_seen": 2814574592
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014853153586904188,
+      "loss": 1.2702,
+      "theoretical_loss": 3.3656477846921997,
+      "tokens_seen": 2814705664
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001484914138982507,
+      "loss": 1.2683,
+      "theoretical_loss": 3.3656356813449335,
+      "tokens_seen": 2814836736
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001484512919274595,
+      "loss": 1.24,
+      "theoretical_loss": 3.3656235787190387,
+      "tokens_seen": 2814967808
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014841116995666827,
+      "loss": 1.2246,
+      "theoretical_loss": 3.3656114768144394,
+      "tokens_seen": 2815098880
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014837104798587705,
+      "loss": 1.1898,
+      "theoretical_loss": 3.365599375631059,
+      "tokens_seen": 2815229952
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014833092601508586,
+      "loss": 1.2264,
+      "theoretical_loss": 3.3655872751688207,
+      "tokens_seen": 2815361024
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014829080404429466,
+      "loss": 1.1919,
+      "theoretical_loss": 3.365575175427648,
+      "tokens_seen": 2815492096
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014825068207350344,
+      "loss": 1.2596,
+      "theoretical_loss": 3.365563076407465,
+      "tokens_seen": 2815623168
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014821056010271225,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3655509781081943,
+      "tokens_seen": 2815754240
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014817043813192106,
+      "loss": 1.3614,
+      "theoretical_loss": 3.3655388805297592,
+      "tokens_seen": 2815885312
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014813031616112983,
+      "loss": 1.2194,
+      "theoretical_loss": 3.3655267836720846,
+      "tokens_seen": 2816016384
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014809019419033864,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3655146875350925,
+      "tokens_seen": 2816147456
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014805007221954742,
+      "loss": 1.2194,
+      "theoretical_loss": 3.365502592118707,
+      "tokens_seen": 2816278528
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014800995024875623,
+      "loss": 1.2544,
+      "theoretical_loss": 3.365490497422852,
+      "tokens_seen": 2816409600
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 944328,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2352261543273926,
+      "objective/train/theoretical_loss": 3.3654784034474505,
+      "objective/train/tokens_used": 1186999776,
+      "theoretical_loss": 3.3654784034474505,
+      "tokens_seen": 2816540672
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.000147969828277965,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3654784034474505,
+      "tokens_seen": 2816540672
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001479297063071738,
+      "loss": 1.2888,
+      "theoretical_loss": 3.3654663101924265,
+      "tokens_seen": 2816671744
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014788958433638262,
+      "loss": 1.2588,
+      "theoretical_loss": 3.365454217657703,
+      "tokens_seen": 2816802816
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001478494623655914,
+      "loss": 1.198,
+      "theoretical_loss": 3.3654421258432046,
+      "tokens_seen": 2816933888
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001478093403948002,
+      "loss": 1.2775,
+      "theoretical_loss": 3.365430034748854,
+      "tokens_seen": 2817064960
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014776921842400898,
+      "loss": 1.2524,
+      "theoretical_loss": 3.365417944374575,
+      "tokens_seen": 2817196032
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001477290964532178,
+      "loss": 1.1849,
+      "theoretical_loss": 3.365405854720291,
+      "tokens_seen": 2817327104
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014768897448242657,
+      "loss": 1.2606,
+      "theoretical_loss": 3.3653937657859263,
+      "tokens_seen": 2817458176
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014764885251163538,
+      "loss": 1.2882,
+      "theoretical_loss": 3.3653816775714036,
+      "tokens_seen": 2817589248
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014760873054084418,
+      "loss": 1.2391,
+      "theoretical_loss": 3.3653695900766474,
+      "tokens_seen": 2817720320
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014756860857005296,
+      "loss": 1.2418,
+      "theoretical_loss": 3.3653575033015812,
+      "tokens_seen": 2817851392
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014752848659926177,
+      "loss": 1.2913,
+      "theoretical_loss": 3.3653454172461283,
+      "tokens_seen": 2817982464
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014748836462847057,
+      "loss": 1.2964,
+      "theoretical_loss": 3.3653333319102128,
+      "tokens_seen": 2818113536
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014744824265767933,
+      "loss": 1.1812,
+      "theoretical_loss": 3.365321247293758,
+      "tokens_seen": 2818244608
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014740812068688813,
+      "loss": 1.2784,
+      "theoretical_loss": 3.365309163396688,
+      "tokens_seen": 2818375680
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014736799871609694,
+      "loss": 1.3197,
+      "theoretical_loss": 3.3652970802189266,
+      "tokens_seen": 2818506752
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014732787674530574,
+      "loss": 1.253,
+      "theoretical_loss": 3.365284997760397,
+      "tokens_seen": 2818637824
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014728775477451452,
+      "loss": 1.2251,
+      "theoretical_loss": 3.3652729160210235,
+      "tokens_seen": 2818768896
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014724763280372333,
+      "loss": 1.2981,
+      "theoretical_loss": 3.365260835000729,
+      "tokens_seen": 2818899968
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014720751083293214,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3652487546994383,
+      "tokens_seen": 2819031040
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001471673888621409,
+      "loss": 1.2662,
+      "theoretical_loss": 3.3652366751170746,
+      "tokens_seen": 2819162112
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001471272668913497,
+      "loss": 1.2641,
+      "theoretical_loss": 3.365224596253562,
+      "tokens_seen": 2819293184
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001470871449205585,
+      "loss": 1.218,
+      "theoretical_loss": 3.365212518108824,
+      "tokens_seen": 2819424256
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001470470229497673,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3652004406827847,
+      "tokens_seen": 2819555328
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014700690097897609,
+      "loss": 1.1822,
+      "theoretical_loss": 3.365188363975368,
+      "tokens_seen": 2819686400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001469667790081849,
+      "loss": 1.2872,
+      "theoretical_loss": 3.365176287986497,
+      "tokens_seen": 2819817472
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001469266570373937,
+      "loss": 1.2132,
+      "theoretical_loss": 3.3651642127160963,
+      "tokens_seen": 2819948544
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014688653506660245,
+      "loss": 1.1727,
+      "theoretical_loss": 3.3651521381640896,
+      "tokens_seen": 2820079616
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014684641309581126,
+      "loss": 1.2862,
+      "theoretical_loss": 3.365140064330401,
+      "tokens_seen": 2820210688
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014680629112502006,
+      "loss": 1.2237,
+      "theoretical_loss": 3.365127991214953,
+      "tokens_seen": 2820341760
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014676616915422887,
+      "loss": 1.2089,
+      "theoretical_loss": 3.3651159188176716,
+      "tokens_seen": 2820472832
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014672604718343765,
+      "loss": 1.2101,
+      "theoretical_loss": 3.3651038471384793,
+      "tokens_seen": 2820603904
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014668592521264646,
+      "loss": 1.2668,
+      "theoretical_loss": 3.3650917761773007,
+      "tokens_seen": 2820734976
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014664580324185526,
+      "loss": 1.2212,
+      "theoretical_loss": 3.3650797059340594,
+      "tokens_seen": 2820866048
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014660568127106404,
+      "loss": 1.2666,
+      "theoretical_loss": 3.3650676364086793,
+      "tokens_seen": 2820997120
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014656555930027282,
+      "loss": 1.215,
+      "theoretical_loss": 3.3650555676010843,
+      "tokens_seen": 2821128192
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014652543732948163,
+      "loss": 1.2559,
+      "theoretical_loss": 3.3650434995111986,
+      "tokens_seen": 2821259264
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001464853153586904,
+      "loss": 1.291,
+      "theoretical_loss": 3.365031432138946,
+      "tokens_seen": 2821390336
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001464451933878992,
+      "loss": 1.1512,
+      "theoretical_loss": 3.365019365484251,
+      "tokens_seen": 2821521408
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014640507141710802,
+      "loss": 1.2123,
+      "theoretical_loss": 3.365007299547037,
+      "tokens_seen": 2821652480
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014636494944631682,
+      "loss": 1.2531,
+      "theoretical_loss": 3.3649952343272282,
+      "tokens_seen": 2821783552
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001463248274755256,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3649831698247485,
+      "tokens_seen": 2821914624
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014628470550473438,
+      "loss": 1.2055,
+      "theoretical_loss": 3.364971106039522,
+      "tokens_seen": 2822045696
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001462445835339432,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3649590429714733,
+      "tokens_seen": 2822176768
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014620446156315197,
+      "loss": 1.1577,
+      "theoretical_loss": 3.3649469806205254,
+      "tokens_seen": 2822307840
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014616433959236077,
+      "loss": 1.2193,
+      "theoretical_loss": 3.364934918986603,
+      "tokens_seen": 2822438912
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014612421762156958,
+      "loss": 1.2437,
+      "theoretical_loss": 3.3649228580696304,
+      "tokens_seen": 2822569984
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001460840956507784,
+      "loss": 1.2193,
+      "theoretical_loss": 3.3649107978695314,
+      "tokens_seen": 2822701056
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014604397367998717,
+      "loss": 1.1695,
+      "theoretical_loss": 3.36489873838623,
+      "tokens_seen": 2822832128
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014600385170919595,
+      "loss": 1.3286,
+      "theoretical_loss": 3.3648866796196506,
+      "tokens_seen": 2822963200
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 946526,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.250018835067749,
+      "objective/train/theoretical_loss": 3.364874621569717,
+      "objective/train/tokens_used": 1193553376,
+      "theoretical_loss": 3.364874621569717,
+      "tokens_seen": 2823094272
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014596372973840475,
+      "loss": 1.2664,
+      "theoretical_loss": 3.364874621569717,
+      "tokens_seen": 2823094272
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014592360776761353,
+      "loss": 1.2315,
+      "theoretical_loss": 3.364862564236353,
+      "tokens_seen": 2823225344
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014588348579682234,
+      "loss": 1.2415,
+      "theoretical_loss": 3.3648505076194843,
+      "tokens_seen": 2823356416
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014584336382603114,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3648384517190335,
+      "tokens_seen": 2823487488
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014580324185523995,
+      "loss": 1.2753,
+      "theoretical_loss": 3.3648263965349257,
+      "tokens_seen": 2823618560
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014576311988444873,
+      "loss": 1.2208,
+      "theoretical_loss": 3.364814342067084,
+      "tokens_seen": 2823749632
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014572299791365754,
+      "loss": 1.2631,
+      "theoretical_loss": 3.364802288315434,
+      "tokens_seen": 2823880704
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014568287594286631,
+      "loss": 1.177,
+      "theoretical_loss": 3.3647902352798984,
+      "tokens_seen": 2824011776
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001456427539720751,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3647781829604027,
+      "tokens_seen": 2824142848
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001456026320012839,
+      "loss": 1.3182,
+      "theoretical_loss": 3.3647661313568706,
+      "tokens_seen": 2824273920
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001455625100304927,
+      "loss": 1.2697,
+      "theoretical_loss": 3.3647540804692264,
+      "tokens_seen": 2824404992
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014552238805970149,
+      "loss": 1.2655,
+      "theoretical_loss": 3.364742030297394,
+      "tokens_seen": 2824536064
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001454822660889103,
+      "loss": 1.2337,
+      "theoretical_loss": 3.3647299808412985,
+      "tokens_seen": 2824667136
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001454421441181191,
+      "loss": 1.2949,
+      "theoretical_loss": 3.3647179321008633,
+      "tokens_seen": 2824798208
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014540202214732788,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3647058840760136,
+      "tokens_seen": 2824929280
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014536190017653666,
+      "loss": 1.2217,
+      "theoretical_loss": 3.3646938367666728,
+      "tokens_seen": 2825060352
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014532177820574546,
+      "loss": 1.2153,
+      "theoretical_loss": 3.364681790172766,
+      "tokens_seen": 2825191424
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014528165623495427,
+      "loss": 1.2444,
+      "theoretical_loss": 3.3646697442942166,
+      "tokens_seen": 2825322496
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014524153426416305,
+      "loss": 1.2472,
+      "theoretical_loss": 3.36465769913095,
+      "tokens_seen": 2825453568
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014520141229337185,
+      "loss": 1.2924,
+      "theoretical_loss": 3.3646456546828896,
+      "tokens_seen": 2825584640
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014516129032258066,
+      "loss": 1.2618,
+      "theoretical_loss": 3.36463361094996,
+      "tokens_seen": 2825715712
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014512116835178947,
+      "loss": 1.2011,
+      "theoretical_loss": 3.364621567932086,
+      "tokens_seen": 2825846784
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014508104638099822,
+      "loss": 1.1783,
+      "theoretical_loss": 3.3646095256291915,
+      "tokens_seen": 2825977856
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014504092441020703,
+      "loss": 1.2726,
+      "theoretical_loss": 3.3645974840412016,
+      "tokens_seen": 2826108928
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014500080243941583,
+      "loss": 1.2406,
+      "theoretical_loss": 3.36458544316804,
+      "tokens_seen": 2826240000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001449606804686246,
+      "loss": 1.2486,
+      "theoretical_loss": 3.364573403009631,
+      "tokens_seen": 2826371072
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014492055849783342,
+      "loss": 1.2281,
+      "theoretical_loss": 3.3645613635658997,
+      "tokens_seen": 2826502144
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014488043652704222,
+      "loss": 1.2593,
+      "theoretical_loss": 3.36454932483677,
+      "tokens_seen": 2826633216
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014484031455625103,
+      "loss": 1.2295,
+      "theoretical_loss": 3.364537286822167,
+      "tokens_seen": 2826764288
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014480019258545978,
+      "loss": 1.2277,
+      "theoretical_loss": 3.364525249522014,
+      "tokens_seen": 2826895360
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001447600706146686,
+      "loss": 1.1804,
+      "theoretical_loss": 3.364513212936237,
+      "tokens_seen": 2827026432
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001447199486438774,
+      "loss": 1.241,
+      "theoretical_loss": 3.364501177064759,
+      "tokens_seen": 2827157504
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014467982667308617,
+      "loss": 1.2307,
+      "theoretical_loss": 3.364489141907506,
+      "tokens_seen": 2827288576
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014463970470229498,
+      "loss": 1.2323,
+      "theoretical_loss": 3.364477107464401,
+      "tokens_seen": 2827419648
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001445995827315038,
+      "loss": 1.2977,
+      "theoretical_loss": 3.3644650737353694,
+      "tokens_seen": 2827550720
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001445594607607126,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3644530407203352,
+      "tokens_seen": 2827681792
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014451933878992135,
+      "loss": 1.2155,
+      "theoretical_loss": 3.3644410084192238,
+      "tokens_seen": 2827812864
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014447921681913015,
+      "loss": 1.3194,
+      "theoretical_loss": 3.364428976831959,
+      "tokens_seen": 2827943936
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014443909484833896,
+      "loss": 1.1933,
+      "theoretical_loss": 3.3644169459584656,
+      "tokens_seen": 2828075008
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014439897287754774,
+      "loss": 1.1594,
+      "theoretical_loss": 3.3644049157986684,
+      "tokens_seen": 2828206080
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014435885090675654,
+      "loss": 1.2805,
+      "theoretical_loss": 3.3643928863524915,
+      "tokens_seen": 2828337152
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014431872893596535,
+      "loss": 1.1346,
+      "theoretical_loss": 3.3643808576198597,
+      "tokens_seen": 2828468224
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014427860696517413,
+      "loss": 1.2613,
+      "theoretical_loss": 3.364368829600698,
+      "tokens_seen": 2828599296
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014423848499438294,
+      "loss": 1.2637,
+      "theoretical_loss": 3.3643568022949304,
+      "tokens_seen": 2828730368
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014419836302359171,
+      "loss": 1.17,
+      "theoretical_loss": 3.364344775702482,
+      "tokens_seen": 2828861440
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014415824105280052,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3643327498232773,
+      "tokens_seen": 2828992512
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001441181190820093,
+      "loss": 1.2624,
+      "theoretical_loss": 3.364320724657241,
+      "tokens_seen": 2829123584
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001440779971112181,
+      "loss": 1.2225,
+      "theoretical_loss": 3.3643087002042975,
+      "tokens_seen": 2829254656
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001440378751404269,
+      "loss": 1.2713,
+      "theoretical_loss": 3.364296676464372,
+      "tokens_seen": 2829385728
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001439977531696357,
+      "loss": 1.214,
+      "theoretical_loss": 3.3642846534373887,
+      "tokens_seen": 2829516800
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 948624,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2875678539276123,
+      "objective/train/theoretical_loss": 3.3642726311232725,
+      "objective/train/tokens_used": 1200106976,
+      "theoretical_loss": 3.3642726311232725,
+      "tokens_seen": 2829647872
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001439576311988445,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3642726311232725,
+      "tokens_seen": 2829647872
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014391750922805328,
+      "loss": 1.2202,
+      "theoretical_loss": 3.364260609521948,
+      "tokens_seen": 2829778944
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014387738725726208,
+      "loss": 1.2559,
+      "theoretical_loss": 3.36424858863334,
+      "tokens_seen": 2829910016
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014383726528647086,
+      "loss": 1.2561,
+      "theoretical_loss": 3.3642365684573736,
+      "tokens_seen": 2830041088
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014379714331567967,
+      "loss": 1.2873,
+      "theoretical_loss": 3.364224548993973,
+      "tokens_seen": 2830172160
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014375702134488848,
+      "loss": 1.2414,
+      "theoretical_loss": 3.364212530243063,
+      "tokens_seen": 2830303232
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014371689937409725,
+      "loss": 1.1774,
+      "theoretical_loss": 3.364200512204569,
+      "tokens_seen": 2830434304
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014367677740330606,
+      "loss": 1.2423,
+      "theoretical_loss": 3.364188494878415,
+      "tokens_seen": 2830565376
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014363665543251484,
+      "loss": 1.2755,
+      "theoretical_loss": 3.3641764782645263,
+      "tokens_seen": 2830696448
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014359653346172365,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3641644623628273,
+      "tokens_seen": 2830827520
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014355641149093243,
+      "loss": 1.2218,
+      "theoretical_loss": 3.364152447173243,
+      "tokens_seen": 2830958592
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014351628952014123,
+      "loss": 1.2201,
+      "theoretical_loss": 3.3641404326956987,
+      "tokens_seen": 2831089664
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014347616754935004,
+      "loss": 1.2836,
+      "theoretical_loss": 3.3641284189301186,
+      "tokens_seen": 2831220736
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014343604557855882,
+      "loss": 1.2754,
+      "theoretical_loss": 3.3641164058764277,
+      "tokens_seen": 2831351808
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014339592360776762,
+      "loss": 1.2824,
+      "theoretical_loss": 3.364104393534551,
+      "tokens_seen": 2831482880
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014335580163697643,
+      "loss": 1.194,
+      "theoretical_loss": 3.3640923819044133,
+      "tokens_seen": 2831613952
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014331567966618518,
+      "loss": 1.1752,
+      "theoretical_loss": 3.3640803709859397,
+      "tokens_seen": 2831745024
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.000143275557695394,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3640683607790547,
+      "tokens_seen": 2831876096
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001432354357246028,
+      "loss": 1.2678,
+      "theoretical_loss": 3.3640563512836836,
+      "tokens_seen": 2832007168
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001431953137538116,
+      "loss": 1.1424,
+      "theoretical_loss": 3.364044342499751,
+      "tokens_seen": 2832138240
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014315519178302038,
+      "loss": 1.2428,
+      "theoretical_loss": 3.364032334427182,
+      "tokens_seen": 2832269312
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001431150698122292,
+      "loss": 1.2461,
+      "theoretical_loss": 3.3640203270659015,
+      "tokens_seen": 2832400384
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.000143074947841438,
+      "loss": 1.2376,
+      "theoretical_loss": 3.3640083204158344,
+      "tokens_seen": 2832531456
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014303482587064675,
+      "loss": 1.1641,
+      "theoretical_loss": 3.3639963144769056,
+      "tokens_seen": 2832662528
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014299470389985555,
+      "loss": 1.2121,
+      "theoretical_loss": 3.3639843092490405,
+      "tokens_seen": 2832793600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014295458192906436,
+      "loss": 1.1894,
+      "theoretical_loss": 3.3639723047321635,
+      "tokens_seen": 2832924672
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014291445995827316,
+      "loss": 1.2059,
+      "theoretical_loss": 3.3639603009262,
+      "tokens_seen": 2833055744
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014287433798748194,
+      "loss": 1.2266,
+      "theoretical_loss": 3.363948297831075,
+      "tokens_seen": 2833186816
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014283421601669075,
+      "loss": 1.258,
+      "theoretical_loss": 3.363936295446713,
+      "tokens_seen": 2833317888
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014279409404589956,
+      "loss": 1.1675,
+      "theoretical_loss": 3.36392429377304,
+      "tokens_seen": 2833448960
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014275397207510833,
+      "loss": 1.2022,
+      "theoretical_loss": 3.3639122928099807,
+      "tokens_seen": 2833580032
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014271385010431711,
+      "loss": 1.2138,
+      "theoretical_loss": 3.3639002925574593,
+      "tokens_seen": 2833711104
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014267372813352592,
+      "loss": 1.2446,
+      "theoretical_loss": 3.363888293015402,
+      "tokens_seen": 2833842176
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014263360616273473,
+      "loss": 1.1984,
+      "theoretical_loss": 3.3638762941837332,
+      "tokens_seen": 2833973248
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001425934841919435,
+      "loss": 1.229,
+      "theoretical_loss": 3.3638642960623786,
+      "tokens_seen": 2834104320
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001425533622211523,
+      "loss": 1.2644,
+      "theoretical_loss": 3.3638522986512625,
+      "tokens_seen": 2834235392
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014251324025036112,
+      "loss": 1.1294,
+      "theoretical_loss": 3.3638403019503103,
+      "tokens_seen": 2834366464
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001424731182795699,
+      "loss": 1.2394,
+      "theoretical_loss": 3.3638283059594474,
+      "tokens_seen": 2834497536
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014243299630877868,
+      "loss": 1.2145,
+      "theoretical_loss": 3.3638163106785988,
+      "tokens_seen": 2834628608
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014239287433798748,
+      "loss": 1.2749,
+      "theoretical_loss": 3.36380431610769,
+      "tokens_seen": 2834759680
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014235275236719626,
+      "loss": 1.2239,
+      "theoretical_loss": 3.363792322246645,
+      "tokens_seen": 2834890752
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014231263039640507,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3637803290953903,
+      "tokens_seen": 2835021824
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014227250842561388,
+      "loss": 1.2679,
+      "theoretical_loss": 3.3637683366538504,
+      "tokens_seen": 2835152896
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014223238645482268,
+      "loss": 1.2778,
+      "theoretical_loss": 3.363756344921951,
+      "tokens_seen": 2835283968
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014219226448403146,
+      "loss": 1.1992,
+      "theoretical_loss": 3.3637443538996163,
+      "tokens_seen": 2835415040
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014215214251324024,
+      "loss": 1.2309,
+      "theoretical_loss": 3.363732363586773,
+      "tokens_seen": 2835546112
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014211202054244905,
+      "loss": 1.2518,
+      "theoretical_loss": 3.3637203739833446,
+      "tokens_seen": 2835677184
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014207189857165783,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3637083850892577,
+      "tokens_seen": 2835808256
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014203177660086663,
+      "loss": 1.2294,
+      "theoretical_loss": 3.363696396904437,
+      "tokens_seen": 2835939328
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014199165463007544,
+      "loss": 1.2623,
+      "theoretical_loss": 3.3636844094288074,
+      "tokens_seen": 2836070400
+    },
+    {
+      "epoch": 0.72,
+      "objective/train/docs_used": 950760,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.174678087234497,
+      "objective/train/theoretical_loss": 3.3636724226622956,
+      "objective/train/tokens_used": 1206660576,
+      "theoretical_loss": 3.3636724226622956,
+      "tokens_seen": 2836201472
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014195153265928424,
+      "loss": 1.1832,
+      "theoretical_loss": 3.3636724226622956,
+      "tokens_seen": 2836201472
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014191141068849302,
+      "loss": 1.2275,
+      "theoretical_loss": 3.363660436604825,
+      "tokens_seen": 2836332544
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014187128871770183,
+      "loss": 1.1993,
+      "theoretical_loss": 3.363648451256322,
+      "tokens_seen": 2836463616
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001418311667469106,
+      "loss": 1.2477,
+      "theoretical_loss": 3.363636466616712,
+      "tokens_seen": 2836594688
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001417910447761194,
+      "loss": 1.1949,
+      "theoretical_loss": 3.36362448268592,
+      "tokens_seen": 2836725760
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001417509228053282,
+      "loss": 1.2493,
+      "theoretical_loss": 3.363612499463871,
+      "tokens_seen": 2836856832
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.000141710800834537,
+      "loss": 1.1764,
+      "theoretical_loss": 3.363600516950491,
+      "tokens_seen": 2836987904
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001416706788637458,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3635885351457047,
+      "tokens_seen": 2837118976
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014163055689295459,
+      "loss": 1.2375,
+      "theoretical_loss": 3.363576554049438,
+      "tokens_seen": 2837250048
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001415904349221634,
+      "loss": 1.1897,
+      "theoretical_loss": 3.363564573661616,
+      "tokens_seen": 2837381120
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014155031295137217,
+      "loss": 1.1735,
+      "theoretical_loss": 3.363552593982164,
+      "tokens_seen": 2837512192
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014151019098058095,
+      "loss": 1.2326,
+      "theoretical_loss": 3.363540615011008,
+      "tokens_seen": 2837643264
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014147006900978976,
+      "loss": 1.1763,
+      "theoretical_loss": 3.3635286367480726,
+      "tokens_seen": 2837774336
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014142994703899856,
+      "loss": 1.2605,
+      "theoretical_loss": 3.3635166591932837,
+      "tokens_seen": 2837905408
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014138982506820734,
+      "loss": 1.2661,
+      "theoretical_loss": 3.3635046823465666,
+      "tokens_seen": 2838036480
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014134970309741615,
+      "loss": 1.245,
+      "theoretical_loss": 3.3634927062078472,
+      "tokens_seen": 2838167552
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014130958112662496,
+      "loss": 1.2491,
+      "theoretical_loss": 3.36348073077705,
+      "tokens_seen": 2838298624
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014126945915583373,
+      "loss": 1.2773,
+      "theoretical_loss": 3.3634687560541012,
+      "tokens_seen": 2838429696
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014122933718504251,
+      "loss": 1.237,
+      "theoretical_loss": 3.363456782038926,
+      "tokens_seen": 2838560768
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014118921521425132,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3634448087314497,
+      "tokens_seen": 2838691840
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014114909324346013,
+      "loss": 1.2023,
+      "theoretical_loss": 3.363432836131598,
+      "tokens_seen": 2838822912
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001411089712726689,
+      "loss": 1.2176,
+      "theoretical_loss": 3.363420864239297,
+      "tokens_seen": 2838953984
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001410688493018777,
+      "loss": 1.1798,
+      "theoretical_loss": 3.3634088930544714,
+      "tokens_seen": 2839085056
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014102872733108652,
+      "loss": 1.1666,
+      "theoretical_loss": 3.363396922577047,
+      "tokens_seen": 2839216128
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014098860536029532,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3633849528069493,
+      "tokens_seen": 2839347200
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014094848338950408,
+      "loss": 1.2749,
+      "theoretical_loss": 3.363372983744104,
+      "tokens_seen": 2839478272
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014090836141871288,
+      "loss": 1.2326,
+      "theoretical_loss": 3.3633610153884366,
+      "tokens_seen": 2839609344
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001408682394479217,
+      "loss": 1.1927,
+      "theoretical_loss": 3.3633490477398724,
+      "tokens_seen": 2839740416
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014082811747713047,
+      "loss": 1.2136,
+      "theoretical_loss": 3.3633370807983374,
+      "tokens_seen": 2839871488
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014078799550633927,
+      "loss": 1.178,
+      "theoretical_loss": 3.3633251145637573,
+      "tokens_seen": 2840002560
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014074787353554808,
+      "loss": 1.2131,
+      "theoretical_loss": 3.363313149036057,
+      "tokens_seen": 2840133632
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001407077515647569,
+      "loss": 1.2634,
+      "theoretical_loss": 3.363301184215163,
+      "tokens_seen": 2840264704
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014066762959396564,
+      "loss": 1.2699,
+      "theoretical_loss": 3.363289220101,
+      "tokens_seen": 2840395776
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014062750762317445,
+      "loss": 1.2767,
+      "theoretical_loss": 3.3632772566934945,
+      "tokens_seen": 2840526848
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014058738565238325,
+      "loss": 1.2075,
+      "theoretical_loss": 3.363265293992572,
+      "tokens_seen": 2840657920
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014054726368159203,
+      "loss": 1.1992,
+      "theoretical_loss": 3.3632533319981572,
+      "tokens_seen": 2840788992
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014050714171080084,
+      "loss": 1.2797,
+      "theoretical_loss": 3.3632413707101776,
+      "tokens_seen": 2840920064
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014046701974000964,
+      "loss": 1.2587,
+      "theoretical_loss": 3.363229410128557,
+      "tokens_seen": 2841051136
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014042689776921845,
+      "loss": 1.3269,
+      "theoretical_loss": 3.3632174502532224,
+      "tokens_seen": 2841182208
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014038677579842723,
+      "loss": 1.1584,
+      "theoretical_loss": 3.363205491084099,
+      "tokens_seen": 2841313280
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.000140346653827636,
+      "loss": 1.1568,
+      "theoretical_loss": 3.3631935326211124,
+      "tokens_seen": 2841444352
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014030653185684481,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3631815748641887,
+      "tokens_seen": 2841575424
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001402664098860536,
+      "loss": 1.2816,
+      "theoretical_loss": 3.3631696178132535,
+      "tokens_seen": 2841706496
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001402262879152624,
+      "loss": 1.2976,
+      "theoretical_loss": 3.3631576614682324,
+      "tokens_seen": 2841837568
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001401861659444712,
+      "loss": 1.1971,
+      "theoretical_loss": 3.3631457058290515,
+      "tokens_seen": 2841968640
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014014604397367999,
+      "loss": 1.2233,
+      "theoretical_loss": 3.363133750895636,
+      "tokens_seen": 2842099712
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001401059220028888,
+      "loss": 1.2078,
+      "theoretical_loss": 3.3631217966679126,
+      "tokens_seen": 2842230784
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014006580003209757,
+      "loss": 1.2395,
+      "theoretical_loss": 3.363109843145806,
+      "tokens_seen": 2842361856
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014002567806130638,
+      "loss": 1.1989,
+      "theoretical_loss": 3.363097890329243,
+      "tokens_seen": 2842492928
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013998555609051516,
+      "loss": 1.2187,
+      "theoretical_loss": 3.363085938218149,
+      "tokens_seen": 2842624000
+    },
+    {
+      "epoch": 0.72,
+      "objective/train/docs_used": 952946,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3542383909225464,
+      "objective/train/theoretical_loss": 3.3630739868124495,
+      "objective/train/tokens_used": 1213214176,
+      "theoretical_loss": 3.3630739868124495,
+      "tokens_seen": 2842755072
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013994543411972396,
+      "loss": 1.2807,
+      "theoretical_loss": 3.3630739868124495,
+      "tokens_seen": 2842755072
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013990531214893277,
+      "loss": 1.2511,
+      "theoretical_loss": 3.363062036112071,
+      "tokens_seen": 2842886144
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013986519017814155,
+      "loss": 1.2113,
+      "theoretical_loss": 3.363050086116939,
+      "tokens_seen": 2843017216
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013982506820735036,
+      "loss": 1.1477,
+      "theoretical_loss": 3.3630381368269795,
+      "tokens_seen": 2843148288
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013978494623655913,
+      "loss": 1.1701,
+      "theoretical_loss": 3.363026188242118,
+      "tokens_seen": 2843279360
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013974482426576794,
+      "loss": 1.1945,
+      "theoretical_loss": 3.363014240362281,
+      "tokens_seen": 2843410432
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013970470229497672,
+      "loss": 1.2594,
+      "theoretical_loss": 3.3630022931873937,
+      "tokens_seen": 2843541504
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013966458032418553,
+      "loss": 1.1925,
+      "theoretical_loss": 3.3629903467173827,
+      "tokens_seen": 2843672576
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013962445835339433,
+      "loss": 1.1999,
+      "theoretical_loss": 3.3629784009521737,
+      "tokens_seen": 2843803648
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001395843363826031,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3629664558916925,
+      "tokens_seen": 2843934720
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013954421441181192,
+      "loss": 1.1938,
+      "theoretical_loss": 3.3629545115358654,
+      "tokens_seen": 2844065792
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013950409244102072,
+      "loss": 1.2847,
+      "theoretical_loss": 3.362942567884618,
+      "tokens_seen": 2844196864
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013946397047022948,
+      "loss": 1.2281,
+      "theoretical_loss": 3.362930624937876,
+      "tokens_seen": 2844327936
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013942384849943828,
+      "loss": 1.1872,
+      "theoretical_loss": 3.362918682695566,
+      "tokens_seen": 2844459008
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001393837265286471,
+      "loss": 1.181,
+      "theoretical_loss": 3.3629067411576137,
+      "tokens_seen": 2844590080
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001393436045578559,
+      "loss": 1.2459,
+      "theoretical_loss": 3.362894800323945,
+      "tokens_seen": 2844721152
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013930348258706467,
+      "loss": 1.2258,
+      "theoretical_loss": 3.3628828601944862,
+      "tokens_seen": 2844852224
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013926336061627348,
+      "loss": 1.2358,
+      "theoretical_loss": 3.362870920769163,
+      "tokens_seen": 2844983296
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001392232386454823,
+      "loss": 1.1907,
+      "theoretical_loss": 3.362858982047902,
+      "tokens_seen": 2845114368
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013918311667469104,
+      "loss": 1.1993,
+      "theoretical_loss": 3.3628470440306284,
+      "tokens_seen": 2845245440
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013914299470389985,
+      "loss": 1.1975,
+      "theoretical_loss": 3.3628351067172693,
+      "tokens_seen": 2845376512
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013910287273310865,
+      "loss": 1.2011,
+      "theoretical_loss": 3.36282317010775,
+      "tokens_seen": 2845507584
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013906275076231746,
+      "loss": 1.254,
+      "theoretical_loss": 3.3628112342019962,
+      "tokens_seen": 2845638656
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013902262879152624,
+      "loss": 1.2717,
+      "theoretical_loss": 3.3627992989999353,
+      "tokens_seen": 2845769728
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013898250682073504,
+      "loss": 1.2067,
+      "theoretical_loss": 3.3627873645014925,
+      "tokens_seen": 2845900800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013894238484994385,
+      "loss": 1.1738,
+      "theoretical_loss": 3.362775430706594,
+      "tokens_seen": 2846031872
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001389022628791526,
+      "loss": 1.2138,
+      "theoretical_loss": 3.362763497615166,
+      "tokens_seen": 2846162944
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001388621409083614,
+      "loss": 1.2465,
+      "theoretical_loss": 3.3627515652271347,
+      "tokens_seen": 2846294016
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013882201893757021,
+      "loss": 1.233,
+      "theoretical_loss": 3.362739633542426,
+      "tokens_seen": 2846425088
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013878189696677902,
+      "loss": 1.1845,
+      "theoretical_loss": 3.3627277025609668,
+      "tokens_seen": 2846556160
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001387417749959878,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3627157722826824,
+      "tokens_seen": 2846687232
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001387016530251966,
+      "loss": 1.2174,
+      "theoretical_loss": 3.362703842707499,
+      "tokens_seen": 2846818304
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001386615310544054,
+      "loss": 1.1936,
+      "theoretical_loss": 3.3626919138353437,
+      "tokens_seen": 2846949376
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001386214090836142,
+      "loss": 1.2234,
+      "theoretical_loss": 3.362679985666142,
+      "tokens_seen": 2847080448
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013858128711282297,
+      "loss": 1.1749,
+      "theoretical_loss": 3.3626680581998203,
+      "tokens_seen": 2847211520
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013854116514203178,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3626561314363044,
+      "tokens_seen": 2847342592
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013850104317124058,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3626442053755214,
+      "tokens_seen": 2847473664
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013846092120044936,
+      "loss": 1.259,
+      "theoretical_loss": 3.3626322800173964,
+      "tokens_seen": 2847604736
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013842079922965817,
+      "loss": 1.1832,
+      "theoretical_loss": 3.362620355361857,
+      "tokens_seen": 2847735808
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013838067725886698,
+      "loss": 1.2747,
+      "theoretical_loss": 3.3626084314088285,
+      "tokens_seen": 2847866880
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013834055528807575,
+      "loss": 1.1888,
+      "theoretical_loss": 3.3625965081582376,
+      "tokens_seen": 2847997952
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013830043331728453,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3625845856100103,
+      "tokens_seen": 2848129024
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013826031134649334,
+      "loss": 1.2529,
+      "theoretical_loss": 3.362572663764073,
+      "tokens_seen": 2848260096
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013822018937570212,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3625607426203525,
+      "tokens_seen": 2848391168
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013818006740491093,
+      "loss": 1.2243,
+      "theoretical_loss": 3.3625488221787743,
+      "tokens_seen": 2848522240
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013813994543411973,
+      "loss": 1.2837,
+      "theoretical_loss": 3.3625369024392655,
+      "tokens_seen": 2848653312
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013809982346332854,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3625249834017517,
+      "tokens_seen": 2848784384
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013805970149253732,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3625130650661603,
+      "tokens_seen": 2848915456
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013801957952174612,
+      "loss": 1.1848,
+      "theoretical_loss": 3.3625011474324165,
+      "tokens_seen": 2849046528
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001379794575509549,
+      "loss": 1.1612,
+      "theoretical_loss": 3.3624892305004472,
+      "tokens_seen": 2849177600
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 955085,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3430936336517334,
+      "objective/train/theoretical_loss": 3.3624773142701794,
+      "objective/train/tokens_used": 1219767776,
+      "theoretical_loss": 3.3624773142701794,
+      "tokens_seen": 2849308672
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013793933558016368,
+      "loss": 1.2814,
+      "theoretical_loss": 3.3624773142701794,
+      "tokens_seen": 2849308672
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001378992136093725,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3624653987415383,
+      "tokens_seen": 2849439744
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001378590916385813,
+      "loss": 1.2466,
+      "theoretical_loss": 3.3624534839144515,
+      "tokens_seen": 2849570816
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001378189696677901,
+      "loss": 1.2514,
+      "theoretical_loss": 3.3624415697888446,
+      "tokens_seen": 2849701888
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013777884769699888,
+      "loss": 1.1661,
+      "theoretical_loss": 3.362429656364644,
+      "tokens_seen": 2849832960
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001377387257262077,
+      "loss": 1.2239,
+      "theoretical_loss": 3.362417743641777,
+      "tokens_seen": 2849964032
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013769860375541647,
+      "loss": 1.2172,
+      "theoretical_loss": 3.362405831620169,
+      "tokens_seen": 2850095104
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013765848178462525,
+      "loss": 1.1936,
+      "theoretical_loss": 3.3623939202997475,
+      "tokens_seen": 2850226176
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013761835981383405,
+      "loss": 1.2605,
+      "theoretical_loss": 3.362382009680438,
+      "tokens_seen": 2850357248
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013757823784304286,
+      "loss": 1.2584,
+      "theoretical_loss": 3.3623700997621677,
+      "tokens_seen": 2850488320
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013753811587225166,
+      "loss": 1.179,
+      "theoretical_loss": 3.362358190544863,
+      "tokens_seen": 2850619392
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013749799390146044,
+      "loss": 1.2783,
+      "theoretical_loss": 3.3623462820284504,
+      "tokens_seen": 2850750464
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013745787193066925,
+      "loss": 1.1776,
+      "theoretical_loss": 3.362334374212856,
+      "tokens_seen": 2850881536
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013741774995987803,
+      "loss": 1.2422,
+      "theoretical_loss": 3.3623224670980068,
+      "tokens_seen": 2851012608
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001373776279890868,
+      "loss": 1.2086,
+      "theoretical_loss": 3.362310560683829,
+      "tokens_seen": 2851143680
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013733750601829561,
+      "loss": 1.2903,
+      "theoretical_loss": 3.36229865497025,
+      "tokens_seen": 2851274752
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013729738404750442,
+      "loss": 1.221,
+      "theoretical_loss": 3.362286749957195,
+      "tokens_seen": 2851405824
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001372572620767132,
+      "loss": 1.2953,
+      "theoretical_loss": 3.362274845644592,
+      "tokens_seen": 2851536896
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.000137217140105922,
+      "loss": 1.2145,
+      "theoretical_loss": 3.362262942032366,
+      "tokens_seen": 2851667968
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001371770181351308,
+      "loss": 1.1887,
+      "theoretical_loss": 3.3622510391204457,
+      "tokens_seen": 2851799040
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013713689616433962,
+      "loss": 1.1848,
+      "theoretical_loss": 3.3622391369087556,
+      "tokens_seen": 2851930112
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013709677419354837,
+      "loss": 1.2585,
+      "theoretical_loss": 3.362227235397224,
+      "tokens_seen": 2852061184
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013705665222275718,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3622153345857764,
+      "tokens_seen": 2852192256
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013701653025196598,
+      "loss": 1.1422,
+      "theoretical_loss": 3.3622034344743397,
+      "tokens_seen": 2852323328
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013697640828117476,
+      "loss": 1.2183,
+      "theoretical_loss": 3.362191535062841,
+      "tokens_seen": 2852454400
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013693628631038357,
+      "loss": 1.2653,
+      "theoretical_loss": 3.3621796363512066,
+      "tokens_seen": 2852585472
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013689616433959238,
+      "loss": 1.1208,
+      "theoretical_loss": 3.3621677383393633,
+      "tokens_seen": 2852716544
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013685604236880118,
+      "loss": 1.2423,
+      "theoretical_loss": 3.3621558410272376,
+      "tokens_seen": 2852847616
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013681592039800993,
+      "loss": 1.1837,
+      "theoretical_loss": 3.3621439444147567,
+      "tokens_seen": 2852978688
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013677579842721874,
+      "loss": 1.1717,
+      "theoretical_loss": 3.362132048501847,
+      "tokens_seen": 2853109760
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013673567645642755,
+      "loss": 1.2492,
+      "theoretical_loss": 3.362120153288435,
+      "tokens_seen": 2853240832
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013669555448563633,
+      "loss": 1.247,
+      "theoretical_loss": 3.3621082587744473,
+      "tokens_seen": 2853371904
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013665543251484513,
+      "loss": 1.1815,
+      "theoretical_loss": 3.3620963649598115,
+      "tokens_seen": 2853502976
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013661531054405394,
+      "loss": 1.2084,
+      "theoretical_loss": 3.3620844718444536,
+      "tokens_seen": 2853634048
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013657518857326274,
+      "loss": 1.2006,
+      "theoretical_loss": 3.362072579428301,
+      "tokens_seen": 2853765120
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001365350666024715,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3620606877112795,
+      "tokens_seen": 2853896192
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001364949446316803,
+      "loss": 1.206,
+      "theoretical_loss": 3.3620487966933172,
+      "tokens_seen": 2854027264
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001364548226608891,
+      "loss": 1.2851,
+      "theoretical_loss": 3.36203690637434,
+      "tokens_seen": 2854158336
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001364147006900979,
+      "loss": 1.235,
+      "theoretical_loss": 3.3620250167542745,
+      "tokens_seen": 2854289408
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001363745787193067,
+      "loss": 1.1848,
+      "theoretical_loss": 3.3620131278330483,
+      "tokens_seen": 2854420480
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001363344567485155,
+      "loss": 1.1555,
+      "theoretical_loss": 3.362001239610588,
+      "tokens_seen": 2854551552
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001362943347777243,
+      "loss": 1.1198,
+      "theoretical_loss": 3.36198935208682,
+      "tokens_seen": 2854682624
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013625421280693309,
+      "loss": 1.2084,
+      "theoretical_loss": 3.361977465261672,
+      "tokens_seen": 2854813696
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013621409083614187,
+      "loss": 1.1808,
+      "theoretical_loss": 3.3619655791350698,
+      "tokens_seen": 2854944768
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013617396886535067,
+      "loss": 1.1714,
+      "theoretical_loss": 3.3619536937069414,
+      "tokens_seen": 2855075840
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013613384689455945,
+      "loss": 1.1538,
+      "theoretical_loss": 3.3619418089772126,
+      "tokens_seen": 2855206912
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013609372492376826,
+      "loss": 1.2484,
+      "theoretical_loss": 3.361929924945811,
+      "tokens_seen": 2855337984
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013605360295297706,
+      "loss": 1.1828,
+      "theoretical_loss": 3.3619180416126637,
+      "tokens_seen": 2855469056
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013601348098218584,
+      "loss": 1.2493,
+      "theoretical_loss": 3.361906158977697,
+      "tokens_seen": 2855600128
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013597335901139465,
+      "loss": 1.2423,
+      "theoretical_loss": 3.361894277040838,
+      "tokens_seen": 2855731200
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 957114,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2372342348098755,
+      "objective/train/theoretical_loss": 3.361882395802014,
+      "objective/train/tokens_used": 1226321376,
+      "theoretical_loss": 3.361882395802014,
+      "tokens_seen": 2855862272
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013593323704060343,
+      "loss": 1.214,
+      "theoretical_loss": 3.361882395802014,
+      "tokens_seen": 2855862272
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013589311506981223,
+      "loss": 1.1999,
+      "theoretical_loss": 3.3618705152611517,
+      "tokens_seen": 2855993344
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013585299309902101,
+      "loss": 1.203,
+      "theoretical_loss": 3.361858635418178,
+      "tokens_seen": 2856124416
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013581287112822982,
+      "loss": 1.2179,
+      "theoretical_loss": 3.36184675627302,
+      "tokens_seen": 2856255488
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013577274915743863,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3618348778256046,
+      "tokens_seen": 2856386560
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001357326271866474,
+      "loss": 1.2503,
+      "theoretical_loss": 3.3618230000758587,
+      "tokens_seen": 2856517632
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001356925052158562,
+      "loss": 1.3011,
+      "theoretical_loss": 3.3618111230237098,
+      "tokens_seen": 2856648704
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013565238324506502,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3617992466690847,
+      "tokens_seen": 2856779776
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001356122612742738,
+      "loss": 1.2281,
+      "theoretical_loss": 3.36178737101191,
+      "tokens_seen": 2856910848
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013557213930348258,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3617754960521133,
+      "tokens_seen": 2857041920
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013553201733269138,
+      "loss": 1.2406,
+      "theoretical_loss": 3.3617636217896214,
+      "tokens_seen": 2857172992
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001354918953619002,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3617517482243615,
+      "tokens_seen": 2857304064
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013545177339110897,
+      "loss": 1.2115,
+      "theoretical_loss": 3.3617398753562604,
+      "tokens_seen": 2857435136
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013541165142031777,
+      "loss": 1.2024,
+      "theoretical_loss": 3.3617280031852452,
+      "tokens_seen": 2857566208
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013537152944952658,
+      "loss": 1.1936,
+      "theoretical_loss": 3.3617161317112436,
+      "tokens_seen": 2857697280
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013533140747873533,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3617042609341823,
+      "tokens_seen": 2857828352
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013529128550794414,
+      "loss": 1.1941,
+      "theoretical_loss": 3.361692390853988,
+      "tokens_seen": 2857959424
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013525116353715295,
+      "loss": 1.2477,
+      "theoretical_loss": 3.3616805214705883,
+      "tokens_seen": 2858090496
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013521104156636175,
+      "loss": 1.2136,
+      "theoretical_loss": 3.361668652783911,
+      "tokens_seen": 2858221568
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013517091959557053,
+      "loss": 1.1706,
+      "theoretical_loss": 3.3616567847938814,
+      "tokens_seen": 2858352640
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013513079762477934,
+      "loss": 1.198,
+      "theoretical_loss": 3.3616449175004286,
+      "tokens_seen": 2858483712
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013509067565398814,
+      "loss": 1.151,
+      "theoretical_loss": 3.361633050903479,
+      "tokens_seen": 2858614784
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001350505536831969,
+      "loss": 1.215,
+      "theoretical_loss": 3.3616211850029596,
+      "tokens_seen": 2858745856
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001350104317124057,
+      "loss": 1.1501,
+      "theoretical_loss": 3.3616093197987977,
+      "tokens_seen": 2858876928
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001349703097416145,
+      "loss": 1.156,
+      "theoretical_loss": 3.3615974552909207,
+      "tokens_seen": 2859008000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013493018777082331,
+      "loss": 1.2135,
+      "theoretical_loss": 3.3615855914792556,
+      "tokens_seen": 2859139072
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001348900658000321,
+      "loss": 1.2305,
+      "theoretical_loss": 3.36157372836373,
+      "tokens_seen": 2859270144
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001348499438292409,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3615618659442705,
+      "tokens_seen": 2859401216
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001348098218584497,
+      "loss": 1.2131,
+      "theoretical_loss": 3.361550004220805,
+      "tokens_seen": 2859532288
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013476969988765849,
+      "loss": 1.1555,
+      "theoretical_loss": 3.3615381431932603,
+      "tokens_seen": 2859663360
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013472957791686727,
+      "loss": 1.1577,
+      "theoretical_loss": 3.361526282861564,
+      "tokens_seen": 2859794432
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013468945594607607,
+      "loss": 1.1636,
+      "theoretical_loss": 3.3615144232256435,
+      "tokens_seen": 2859925504
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013464933397528488,
+      "loss": 1.2615,
+      "theoretical_loss": 3.3615025642854257,
+      "tokens_seen": 2860056576
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013460921200449366,
+      "loss": 1.1779,
+      "theoretical_loss": 3.361490706040838,
+      "tokens_seen": 2860187648
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013456909003370246,
+      "loss": 1.2131,
+      "theoretical_loss": 3.361478848491808,
+      "tokens_seen": 2860318720
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013452896806291127,
+      "loss": 1.2047,
+      "theoretical_loss": 3.3614669916382627,
+      "tokens_seen": 2860449792
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013448884609212005,
+      "loss": 1.2066,
+      "theoretical_loss": 3.3614551354801296,
+      "tokens_seen": 2860580864
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013444872412132883,
+      "loss": 1.2633,
+      "theoretical_loss": 3.361443280017336,
+      "tokens_seen": 2860711936
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013440860215053763,
+      "loss": 1.1686,
+      "theoretical_loss": 3.361431425249809,
+      "tokens_seen": 2860843008
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013436848017974644,
+      "loss": 1.1719,
+      "theoretical_loss": 3.3614195711774766,
+      "tokens_seen": 2860974080
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013432835820895522,
+      "loss": 1.1722,
+      "theoretical_loss": 3.361407717800266,
+      "tokens_seen": 2861105152
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013428823623816403,
+      "loss": 1.1701,
+      "theoretical_loss": 3.361395865118104,
+      "tokens_seen": 2861236224
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013424811426737283,
+      "loss": 1.1783,
+      "theoretical_loss": 3.3613840131309187,
+      "tokens_seen": 2861367296
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001342079922965816,
+      "loss": 1.2915,
+      "theoretical_loss": 3.361372161838637,
+      "tokens_seen": 2861498368
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001341678703257904,
+      "loss": 1.1593,
+      "theoretical_loss": 3.361360311241187,
+      "tokens_seen": 2861629440
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001341277483549992,
+      "loss": 1.2243,
+      "theoretical_loss": 3.361348461338496,
+      "tokens_seen": 2861760512
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013408762638420798,
+      "loss": 1.2557,
+      "theoretical_loss": 3.3613366121304904,
+      "tokens_seen": 2861891584
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013404750441341678,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3613247636170986,
+      "tokens_seen": 2862022656
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001340073824426256,
+      "loss": 1.1873,
+      "theoretical_loss": 3.361312915798248,
+      "tokens_seen": 2862153728
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001339672604718344,
+      "loss": 1.2067,
+      "theoretical_loss": 3.3613010686738662,
+      "tokens_seen": 2862284800
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 959126,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.245158076286316,
+      "objective/train/theoretical_loss": 3.3612892222438804,
+      "objective/train/tokens_used": 1232874976,
+      "theoretical_loss": 3.3612892222438804,
+      "tokens_seen": 2862415872
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013392713850104317,
+      "loss": 1.2095,
+      "theoretical_loss": 3.3612892222438804,
+      "tokens_seen": 2862415872
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013388701653025198,
+      "loss": 1.2039,
+      "theoretical_loss": 3.3612773765082187,
+      "tokens_seen": 2862546944
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013384689455946076,
+      "loss": 1.2746,
+      "theoretical_loss": 3.3612655314668074,
+      "tokens_seen": 2862678016
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013380677258866954,
+      "loss": 1.154,
+      "theoretical_loss": 3.361253687119575,
+      "tokens_seen": 2862809088
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013376665061787835,
+      "loss": 1.2814,
+      "theoretical_loss": 3.361241843466449,
+      "tokens_seen": 2862940160
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013372652864708715,
+      "loss": 1.3125,
+      "theoretical_loss": 3.3612300005073563,
+      "tokens_seen": 2863071232
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013368640667629596,
+      "loss": 1.1724,
+      "theoretical_loss": 3.3612181582422247,
+      "tokens_seen": 2863202304
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013364628470550474,
+      "loss": 1.1745,
+      "theoretical_loss": 3.3612063166709825,
+      "tokens_seen": 2863333376
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013360616273471354,
+      "loss": 1.1917,
+      "theoretical_loss": 3.361194475793557,
+      "tokens_seen": 2863464448
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013356604076392232,
+      "loss": 1.2144,
+      "theoretical_loss": 3.361182635609875,
+      "tokens_seen": 2863595520
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001335259187931311,
+      "loss": 1.1981,
+      "theoretical_loss": 3.3611707961198647,
+      "tokens_seen": 2863726592
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001334857968223399,
+      "loss": 1.1624,
+      "theoretical_loss": 3.361158957323454,
+      "tokens_seen": 2863857664
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013344567485154871,
+      "loss": 1.1544,
+      "theoretical_loss": 3.3611471192205697,
+      "tokens_seen": 2863988736
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013340555288075752,
+      "loss": 1.1763,
+      "theoretical_loss": 3.36113528181114,
+      "tokens_seen": 2864119808
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001333654309099663,
+      "loss": 1.2666,
+      "theoretical_loss": 3.3611234450950924,
+      "tokens_seen": 2864250880
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001333253089391751,
+      "loss": 1.1909,
+      "theoretical_loss": 3.361111609072355,
+      "tokens_seen": 2864381952
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001332851869683839,
+      "loss": 1.1819,
+      "theoretical_loss": 3.361099773742855,
+      "tokens_seen": 2864513024
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013324506499759266,
+      "loss": 1.1927,
+      "theoretical_loss": 3.36108793910652,
+      "tokens_seen": 2864644096
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013320494302680147,
+      "loss": 1.1318,
+      "theoretical_loss": 3.361076105163278,
+      "tokens_seen": 2864775168
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013316482105601028,
+      "loss": 1.1366,
+      "theoretical_loss": 3.3610642719130563,
+      "tokens_seen": 2864906240
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013312469908521906,
+      "loss": 1.1688,
+      "theoretical_loss": 3.3610524393557832,
+      "tokens_seen": 2865037312
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013308457711442786,
+      "loss": 1.2153,
+      "theoretical_loss": 3.361040607491386,
+      "tokens_seen": 2865168384
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013304445514363667,
+      "loss": 1.1121,
+      "theoretical_loss": 3.3610287763197926,
+      "tokens_seen": 2865299456
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013300433317284548,
+      "loss": 1.1295,
+      "theoretical_loss": 3.3610169458409302,
+      "tokens_seen": 2865430528
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013296421120205423,
+      "loss": 1.1873,
+      "theoretical_loss": 3.3610051160547276,
+      "tokens_seen": 2865561600
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013292408923126303,
+      "loss": 1.1353,
+      "theoretical_loss": 3.3609932869611114,
+      "tokens_seen": 2865692672
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013288396726047184,
+      "loss": 1.2059,
+      "theoretical_loss": 3.3609814585600106,
+      "tokens_seen": 2865823744
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013284384528968062,
+      "loss": 1.1788,
+      "theoretical_loss": 3.3609696308513524,
+      "tokens_seen": 2865954816
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013280372331888943,
+      "loss": 1.2032,
+      "theoretical_loss": 3.360957803835064,
+      "tokens_seen": 2866085888
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013276360134809823,
+      "loss": 1.1781,
+      "theoretical_loss": 3.360945977511074,
+      "tokens_seen": 2866216960
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013272347937730704,
+      "loss": 1.2145,
+      "theoretical_loss": 3.36093415187931,
+      "tokens_seen": 2866348032
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001326833574065158,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3609223269396997,
+      "tokens_seen": 2866479104
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001326432354357246,
+      "loss": 1.1848,
+      "theoretical_loss": 3.360910502692171,
+      "tokens_seen": 2866610176
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001326031134649334,
+      "loss": 1.2189,
+      "theoretical_loss": 3.3608986791366524,
+      "tokens_seen": 2866741248
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013256299149414218,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3608868562730705,
+      "tokens_seen": 2866872320
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.000132522869523351,
+      "loss": 1.1641,
+      "theoretical_loss": 3.360875034101354,
+      "tokens_seen": 2867003392
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001324827475525598,
+      "loss": 1.1093,
+      "theoretical_loss": 3.360863212621431,
+      "tokens_seen": 2867134464
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001324426255817686,
+      "loss": 1.1311,
+      "theoretical_loss": 3.3608513918332283,
+      "tokens_seen": 2867265536
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013240250361097738,
+      "loss": 1.2793,
+      "theoretical_loss": 3.3608395717366752,
+      "tokens_seen": 2867396608
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013236238164018616,
+      "loss": 1.2247,
+      "theoretical_loss": 3.3608277523316987,
+      "tokens_seen": 2867527680
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013232225966939497,
+      "loss": 1.1762,
+      "theoretical_loss": 3.360815933618227,
+      "tokens_seen": 2867658752
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013228213769860375,
+      "loss": 1.2087,
+      "theoretical_loss": 3.3608041155961876,
+      "tokens_seen": 2867789824
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013224201572781255,
+      "loss": 1.2661,
+      "theoretical_loss": 3.3607922982655096,
+      "tokens_seen": 2867920896
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013220189375702136,
+      "loss": 1.294,
+      "theoretical_loss": 3.36078048162612,
+      "tokens_seen": 2868051968
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013216177178623016,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3607686656779467,
+      "tokens_seen": 2868183040
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013212164981543894,
+      "loss": 1.2016,
+      "theoretical_loss": 3.3607568504209184,
+      "tokens_seen": 2868314112
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013208152784464772,
+      "loss": 1.2627,
+      "theoretical_loss": 3.360745035854962,
+      "tokens_seen": 2868445184
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013204140587385653,
+      "loss": 1.1295,
+      "theoretical_loss": 3.360733221980007,
+      "tokens_seen": 2868576256
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001320012839030653,
+      "loss": 1.1858,
+      "theoretical_loss": 3.36072140879598,
+      "tokens_seen": 2868707328
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013196116193227411,
+      "loss": 1.0959,
+      "theoretical_loss": 3.36070959630281,
+      "tokens_seen": 2868838400
+    },
+    {
+      "epoch": 0.74,
+      "objective/train/docs_used": 961045,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1604371070861816,
+      "objective/train/theoretical_loss": 3.3606977845004247,
+      "objective/train/tokens_used": 1239428576,
+      "theoretical_loss": 3.3606977845004247,
+      "tokens_seen": 2868969472
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013192103996148292,
+      "loss": 1.2246,
+      "theoretical_loss": 3.3606977845004247,
+      "tokens_seen": 2868969472
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001318809179906917,
+      "loss": 1.1763,
+      "theoretical_loss": 3.360685973388752,
+      "tokens_seen": 2869100544
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001318407960199005,
+      "loss": 1.1359,
+      "theoretical_loss": 3.3606741629677197,
+      "tokens_seen": 2869231616
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013180067404910929,
+      "loss": 1.1911,
+      "theoretical_loss": 3.360662353237257,
+      "tokens_seen": 2869362688
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001317605520783181,
+      "loss": 1.2231,
+      "theoretical_loss": 3.3606505441972905,
+      "tokens_seen": 2869493760
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013172043010752687,
+      "loss": 1.1735,
+      "theoretical_loss": 3.360638735847749,
+      "tokens_seen": 2869624832
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013168030813673568,
+      "loss": 1.2079,
+      "theoretical_loss": 3.360626928188561,
+      "tokens_seen": 2869755904
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013164018616594448,
+      "loss": 1.2196,
+      "theoretical_loss": 3.360615121219654,
+      "tokens_seen": 2869886976
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013160006419515326,
+      "loss": 1.1028,
+      "theoretical_loss": 3.360603314940956,
+      "tokens_seen": 2870018048
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013155994222436207,
+      "loss": 1.1494,
+      "theoretical_loss": 3.360591509352396,
+      "tokens_seen": 2870149120
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013151982025357088,
+      "loss": 1.2154,
+      "theoretical_loss": 3.360579704453902,
+      "tokens_seen": 2870280192
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013147969828277965,
+      "loss": 1.274,
+      "theoretical_loss": 3.360567900245401,
+      "tokens_seen": 2870411264
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013143957631198843,
+      "loss": 1.1097,
+      "theoretical_loss": 3.360556096726822,
+      "tokens_seen": 2870542336
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013139945434119724,
+      "loss": 1.1474,
+      "theoretical_loss": 3.3605442938980934,
+      "tokens_seen": 2870673408
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013135933237040605,
+      "loss": 1.2329,
+      "theoretical_loss": 3.360532491759143,
+      "tokens_seen": 2870804480
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013131921039961483,
+      "loss": 1.1986,
+      "theoretical_loss": 3.3605206903098988,
+      "tokens_seen": 2870935552
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013127908842882363,
+      "loss": 1.1724,
+      "theoretical_loss": 3.3605088895502897,
+      "tokens_seen": 2871066624
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013123896645803244,
+      "loss": 1.1699,
+      "theoretical_loss": 3.3604970894802433,
+      "tokens_seen": 2871197696
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001311988444872412,
+      "loss": 1.1528,
+      "theoretical_loss": 3.360485290099688,
+      "tokens_seen": 2871328768
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013115872251645,
+      "loss": 1.1891,
+      "theoretical_loss": 3.3604734914085523,
+      "tokens_seen": 2871459840
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001311186005456588,
+      "loss": 1.1624,
+      "theoretical_loss": 3.360461693406764,
+      "tokens_seen": 2871590912
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001310784785748676,
+      "loss": 1.2004,
+      "theoretical_loss": 3.360449896094252,
+      "tokens_seen": 2871721984
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001310383566040764,
+      "loss": 1.1312,
+      "theoretical_loss": 3.3604380994709437,
+      "tokens_seen": 2871853056
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001309982346332852,
+      "loss": 1.1269,
+      "theoretical_loss": 3.360426303536768,
+      "tokens_seen": 2871984128
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.000130958112662494,
+      "loss": 1.1846,
+      "theoretical_loss": 3.3604145082916532,
+      "tokens_seen": 2872115200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013091799069170278,
+      "loss": 1.1852,
+      "theoretical_loss": 3.3604027137355272,
+      "tokens_seen": 2872246272
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013087786872091156,
+      "loss": 1.2258,
+      "theoretical_loss": 3.360390919868319,
+      "tokens_seen": 2872377344
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013083774675012037,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3603791266899563,
+      "tokens_seen": 2872508416
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013079762477932917,
+      "loss": 1.183,
+      "theoretical_loss": 3.360367334200368,
+      "tokens_seen": 2872639488
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013075750280853795,
+      "loss": 1.2134,
+      "theoretical_loss": 3.3603555423994815,
+      "tokens_seen": 2872770560
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013071738083774676,
+      "loss": 1.1862,
+      "theoretical_loss": 3.3603437512872256,
+      "tokens_seen": 2872901632
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013067725886695556,
+      "loss": 1.1524,
+      "theoretical_loss": 3.3603319608635296,
+      "tokens_seen": 2873032704
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013063713689616434,
+      "loss": 1.2545,
+      "theoretical_loss": 3.3603201711283206,
+      "tokens_seen": 2873163776
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013059701492537312,
+      "loss": 1.1945,
+      "theoretical_loss": 3.3603083820815276,
+      "tokens_seen": 2873294848
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013055689295458193,
+      "loss": 1.1892,
+      "theoretical_loss": 3.3602965937230787,
+      "tokens_seen": 2873425920
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013051677098379073,
+      "loss": 1.1326,
+      "theoretical_loss": 3.3602848060529027,
+      "tokens_seen": 2873556992
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013047664901299951,
+      "loss": 1.1302,
+      "theoretical_loss": 3.360273019070928,
+      "tokens_seen": 2873688064
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013043652704220832,
+      "loss": 1.2128,
+      "theoretical_loss": 3.3602612327770824,
+      "tokens_seen": 2873819136
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013039640507141713,
+      "loss": 1.174,
+      "theoretical_loss": 3.360249447171295,
+      "tokens_seen": 2873950208
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001303562831006259,
+      "loss": 1.1839,
+      "theoretical_loss": 3.3602376622534944,
+      "tokens_seen": 2874081280
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013031616112983468,
+      "loss": 1.2252,
+      "theoretical_loss": 3.3602258780236083,
+      "tokens_seen": 2874212352
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001302760391590435,
+      "loss": 1.149,
+      "theoretical_loss": 3.3602140944815657,
+      "tokens_seen": 2874343424
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001302359171882523,
+      "loss": 1.1659,
+      "theoretical_loss": 3.3602023116272948,
+      "tokens_seen": 2874474496
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013019579521746108,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3601905294607244,
+      "tokens_seen": 2874605568
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013015567324666988,
+      "loss": 1.1817,
+      "theoretical_loss": 3.3601787479817835,
+      "tokens_seen": 2874736640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001301155512758787,
+      "loss": 1.1459,
+      "theoretical_loss": 3.3601669671903993,
+      "tokens_seen": 2874867712
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013007542930508747,
+      "loss": 1.1495,
+      "theoretical_loss": 3.360155187086501,
+      "tokens_seen": 2874998784
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013003530733429627,
+      "loss": 1.1161,
+      "theoretical_loss": 3.3601434076700176,
+      "tokens_seen": 2875129856
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012999518536350505,
+      "loss": 1.1628,
+      "theoretical_loss": 3.360131628940877,
+      "tokens_seen": 2875260928
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012995506339271383,
+      "loss": 1.2083,
+      "theoretical_loss": 3.360119850899008,
+      "tokens_seen": 2875392000
+    },
+    {
+      "epoch": 0.74,
+      "objective/train/docs_used": 963256,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3161914348602295,
+      "objective/train/theoretical_loss": 3.36010807354434,
+      "objective/train/tokens_used": 1245982176,
+      "theoretical_loss": 3.36010807354434,
+      "tokens_seen": 2875523072
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012991494142192264,
+      "loss": 1.2536,
+      "theoretical_loss": 3.36010807354434,
+      "tokens_seen": 2875523072
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012987481945113145,
+      "loss": 1.1784,
+      "theoretical_loss": 3.3600962968767996,
+      "tokens_seen": 2875654144
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012983469748034025,
+      "loss": 1.1662,
+      "theoretical_loss": 3.360084520896317,
+      "tokens_seen": 2875785216
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012979457550954903,
+      "loss": 1.2376,
+      "theoretical_loss": 3.36007274560282,
+      "tokens_seen": 2875916288
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012975445353875784,
+      "loss": 1.1525,
+      "theoretical_loss": 3.3600609709962383,
+      "tokens_seen": 2876047360
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012971433156796662,
+      "loss": 1.1582,
+      "theoretical_loss": 3.360049197076499,
+      "tokens_seen": 2876178432
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001296742095971754,
+      "loss": 1.1623,
+      "theoretical_loss": 3.360037423843532,
+      "tokens_seen": 2876309504
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001296340876263842,
+      "loss": 1.1149,
+      "theoretical_loss": 3.3600256512972657,
+      "tokens_seen": 2876440576
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.000129593965655593,
+      "loss": 1.2041,
+      "theoretical_loss": 3.360013879437628,
+      "tokens_seen": 2876571648
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012955384368480181,
+      "loss": 1.1509,
+      "theoretical_loss": 3.3600021082645486,
+      "tokens_seen": 2876702720
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001295137217140106,
+      "loss": 1.1576,
+      "theoretical_loss": 3.3599903377779556,
+      "tokens_seen": 2876833792
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001294735997432194,
+      "loss": 1.174,
+      "theoretical_loss": 3.3599785679777776,
+      "tokens_seen": 2876964864
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012943347777242818,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3599667988639434,
+      "tokens_seen": 2877095936
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012939335580163696,
+      "loss": 1.1772,
+      "theoretical_loss": 3.3599550304363817,
+      "tokens_seen": 2877227008
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012935323383084577,
+      "loss": 1.1309,
+      "theoretical_loss": 3.3599432626950216,
+      "tokens_seen": 2877358080
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012931311186005457,
+      "loss": 1.1809,
+      "theoretical_loss": 3.3599314956397914,
+      "tokens_seen": 2877489152
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012927298988926338,
+      "loss": 1.1369,
+      "theoretical_loss": 3.35991972927062,
+      "tokens_seen": 2877620224
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012923286791847216,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3599079635874363,
+      "tokens_seen": 2877751296
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012919274594768096,
+      "loss": 1.2539,
+      "theoretical_loss": 3.3598961985901687,
+      "tokens_seen": 2877882368
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012915262397688977,
+      "loss": 1.1797,
+      "theoretical_loss": 3.3598844342787464,
+      "tokens_seen": 2878013440
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012911250200609852,
+      "loss": 1.2944,
+      "theoretical_loss": 3.3598726706530977,
+      "tokens_seen": 2878144512
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012907238003530733,
+      "loss": 1.1863,
+      "theoretical_loss": 3.359860907713152,
+      "tokens_seen": 2878275584
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012903225806451613,
+      "loss": 1.2866,
+      "theoretical_loss": 3.3598491454588375,
+      "tokens_seen": 2878406656
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001289921360937249,
+      "loss": 1.1373,
+      "theoretical_loss": 3.3598373838900835,
+      "tokens_seen": 2878537728
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012895201412293372,
+      "loss": 1.1931,
+      "theoretical_loss": 3.3598256230068184,
+      "tokens_seen": 2878668800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012891189215214253,
+      "loss": 1.132,
+      "theoretical_loss": 3.3598138628089713,
+      "tokens_seen": 2878799872
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012887177018135133,
+      "loss": 1.2246,
+      "theoretical_loss": 3.359802103296471,
+      "tokens_seen": 2878930944
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012883164821056008,
+      "loss": 1.2205,
+      "theoretical_loss": 3.3597903444692463,
+      "tokens_seen": 2879062016
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001287915262397689,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3597785863272263,
+      "tokens_seen": 2879193088
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001287514042689777,
+      "loss": 1.2593,
+      "theoretical_loss": 3.35976682887034,
+      "tokens_seen": 2879324160
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012871128229818648,
+      "loss": 1.319,
+      "theoretical_loss": 3.3597550720985154,
+      "tokens_seen": 2879455232
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012867116032739528,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3597433160116825,
+      "tokens_seen": 2879586304
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001286310383566041,
+      "loss": 1.2018,
+      "theoretical_loss": 3.3597315606097693,
+      "tokens_seen": 2879717376
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001285909163858129,
+      "loss": 1.1696,
+      "theoretical_loss": 3.3597198058927056,
+      "tokens_seen": 2879848448
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012855079441502167,
+      "loss": 1.2193,
+      "theoretical_loss": 3.3597080518604194,
+      "tokens_seen": 2879979520
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012851067244423045,
+      "loss": 1.2381,
+      "theoretical_loss": 3.3596962985128407,
+      "tokens_seen": 2880110592
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012847055047343926,
+      "loss": 1.2329,
+      "theoretical_loss": 3.3596845458498974,
+      "tokens_seen": 2880241664
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012843042850264804,
+      "loss": 1.2573,
+      "theoretical_loss": 3.3596727938715194,
+      "tokens_seen": 2880372736
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012839030653185685,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3596610425776348,
+      "tokens_seen": 2880503808
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012835018456106565,
+      "loss": 1.1787,
+      "theoretical_loss": 3.3596492919681733,
+      "tokens_seen": 2880634880
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012831006259027446,
+      "loss": 1.1978,
+      "theoretical_loss": 3.359637542043064,
+      "tokens_seen": 2880765952
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012826994061948324,
+      "loss": 1.2155,
+      "theoretical_loss": 3.359625792802235,
+      "tokens_seen": 2880897024
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012822981864869202,
+      "loss": 1.2158,
+      "theoretical_loss": 3.3596140442456157,
+      "tokens_seen": 2881028096
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012818969667790082,
+      "loss": 1.2261,
+      "theoretical_loss": 3.3596022963731356,
+      "tokens_seen": 2881159168
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001281495747071096,
+      "loss": 1.2133,
+      "theoretical_loss": 3.3595905491847233,
+      "tokens_seen": 2881290240
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001281094527363184,
+      "loss": 1.2502,
+      "theoretical_loss": 3.359578802680308,
+      "tokens_seen": 2881421312
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012806933076552721,
+      "loss": 1.1713,
+      "theoretical_loss": 3.359567056859819,
+      "tokens_seen": 2881552384
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.000128029208794736,
+      "loss": 1.2329,
+      "theoretical_loss": 3.3595553117231844,
+      "tokens_seen": 2881683456
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001279890868239448,
+      "loss": 1.2555,
+      "theoretical_loss": 3.3595435672703347,
+      "tokens_seen": 2881814528
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012794896485315358,
+      "loss": 1.2616,
+      "theoretical_loss": 3.359531823501198,
+      "tokens_seen": 2881945600
+    },
+    {
+      "epoch": 0.75,
+      "objective/train/docs_used": 965415,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1705117225646973,
+      "objective/train/theoretical_loss": 3.3595200804157033,
+      "objective/train/tokens_used": 1252535776,
+      "theoretical_loss": 3.3595200804157033,
+      "tokens_seen": 2882076672
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012790884288236239,
+      "loss": 1.1895,
+      "theoretical_loss": 3.3595200804157033,
+      "tokens_seen": 2882076672
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012786872091157116,
+      "loss": 1.1833,
+      "theoretical_loss": 3.359508338013781,
+      "tokens_seen": 2882207744
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012782859894077997,
+      "loss": 1.2853,
+      "theoretical_loss": 3.3594965962953585,
+      "tokens_seen": 2882338816
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012778847696998878,
+      "loss": 1.2646,
+      "theoretical_loss": 3.359484855260366,
+      "tokens_seen": 2882469888
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012774835499919756,
+      "loss": 1.2233,
+      "theoretical_loss": 3.3594731149087327,
+      "tokens_seen": 2882600960
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012770823302840636,
+      "loss": 1.1823,
+      "theoretical_loss": 3.3594613752403872,
+      "tokens_seen": 2882732032
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012766811105761517,
+      "loss": 1.2413,
+      "theoretical_loss": 3.359449636255259,
+      "tokens_seen": 2882863104
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012762798908682395,
+      "loss": 1.141,
+      "theoretical_loss": 3.359437897953277,
+      "tokens_seen": 2882994176
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012758786711603273,
+      "loss": 1.1607,
+      "theoretical_loss": 3.359426160334371,
+      "tokens_seen": 2883125248
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012754774514524153,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3594144233984697,
+      "tokens_seen": 2883256320
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012750762317445034,
+      "loss": 1.2141,
+      "theoretical_loss": 3.359402687145502,
+      "tokens_seen": 2883387392
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012746750120365912,
+      "loss": 1.1884,
+      "theoretical_loss": 3.359390951575398,
+      "tokens_seen": 2883518464
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012742737923286793,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3593792166880863,
+      "tokens_seen": 2883649536
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012738725726207673,
+      "loss": 1.2357,
+      "theoretical_loss": 3.3593674824834965,
+      "tokens_seen": 2883780608
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001273471352912855,
+      "loss": 1.2318,
+      "theoretical_loss": 3.359355748961557,
+      "tokens_seen": 2883911680
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001273070133204943,
+      "loss": 1.2343,
+      "theoretical_loss": 3.359344016122199,
+      "tokens_seen": 2884042752
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001272668913497031,
+      "loss": 1.3111,
+      "theoretical_loss": 3.359332283965349,
+      "tokens_seen": 2884173824
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001272267693789119,
+      "loss": 1.256,
+      "theoretical_loss": 3.3593205524909386,
+      "tokens_seen": 2884304896
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012718664740812068,
+      "loss": 1.1366,
+      "theoretical_loss": 3.359308821698896,
+      "tokens_seen": 2884435968
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001271465254373295,
+      "loss": 1.1751,
+      "theoretical_loss": 3.359297091589151,
+      "tokens_seen": 2884567040
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001271064034665383,
+      "loss": 1.1678,
+      "theoretical_loss": 3.3592853621616325,
+      "tokens_seen": 2884698112
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012706628149574705,
+      "loss": 1.2986,
+      "theoretical_loss": 3.35927363341627,
+      "tokens_seen": 2884829184
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012702615952495585,
+      "loss": 1.2447,
+      "theoretical_loss": 3.359261905352993,
+      "tokens_seen": 2884960256
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012698603755416466,
+      "loss": 1.2349,
+      "theoretical_loss": 3.359250177971731,
+      "tokens_seen": 2885091328
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012694591558337347,
+      "loss": 1.2611,
+      "theoretical_loss": 3.3592384512724123,
+      "tokens_seen": 2885222400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012690579361258225,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3592267252549677,
+      "tokens_seen": 2885353472
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012686567164179105,
+      "loss": 1.1659,
+      "theoretical_loss": 3.3592149999193257,
+      "tokens_seen": 2885484544
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012682554967099986,
+      "loss": 1.1642,
+      "theoretical_loss": 3.3592032752654157,
+      "tokens_seen": 2885615616
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012678542770020864,
+      "loss": 1.1471,
+      "theoretical_loss": 3.359191551293167,
+      "tokens_seen": 2885746688
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012674530572941742,
+      "loss": 1.2703,
+      "theoretical_loss": 3.3591798280025102,
+      "tokens_seen": 2885877760
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012670518375862622,
+      "loss": 1.2026,
+      "theoretical_loss": 3.359168105393373,
+      "tokens_seen": 2886008832
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012666506178783503,
+      "loss": 1.3071,
+      "theoretical_loss": 3.359156383465686,
+      "tokens_seen": 2886139904
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001266249398170438,
+      "loss": 1.1187,
+      "theoretical_loss": 3.3591446622193786,
+      "tokens_seen": 2886270976
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012658481784625261,
+      "loss": 1.2296,
+      "theoretical_loss": 3.359132941654379,
+      "tokens_seen": 2886402048
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012654469587546142,
+      "loss": 1.2496,
+      "theoretical_loss": 3.359121221770619,
+      "tokens_seen": 2886533120
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001265045739046702,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3591095025680255,
+      "tokens_seen": 2886664192
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012646445193387898,
+      "loss": 1.2193,
+      "theoretical_loss": 3.3590977840465297,
+      "tokens_seen": 2886795264
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012642432996308779,
+      "loss": 1.2291,
+      "theoretical_loss": 3.35908606620606,
+      "tokens_seen": 2886926336
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001263842079922966,
+      "loss": 1.2497,
+      "theoretical_loss": 3.359074349046547,
+      "tokens_seen": 2887057408
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012634408602150537,
+      "loss": 1.2029,
+      "theoretical_loss": 3.3590626325679196,
+      "tokens_seen": 2887188480
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012630396405071418,
+      "loss": 1.1684,
+      "theoretical_loss": 3.3590509167701077,
+      "tokens_seen": 2887319552
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012626384207992298,
+      "loss": 1.1896,
+      "theoretical_loss": 3.35903920165304,
+      "tokens_seen": 2887450624
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012622372010913176,
+      "loss": 1.1926,
+      "theoretical_loss": 3.359027487216647,
+      "tokens_seen": 2887581696
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012618359813834057,
+      "loss": 1.1945,
+      "theoretical_loss": 3.3590157734608574,
+      "tokens_seen": 2887712768
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012614347616754935,
+      "loss": 1.1522,
+      "theoretical_loss": 3.3590040603856015,
+      "tokens_seen": 2887843840
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012610335419675815,
+      "loss": 1.1825,
+      "theoretical_loss": 3.3589923479908084,
+      "tokens_seen": 2887974912
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012606323222596693,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3589806362764083,
+      "tokens_seen": 2888105984
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012602311025517574,
+      "loss": 1.29,
+      "theoretical_loss": 3.3589689252423303,
+      "tokens_seen": 2888237056
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012598298828438455,
+      "loss": 1.2194,
+      "theoretical_loss": 3.3589572148885036,
+      "tokens_seen": 2888368128
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012594286631359333,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3589455052148587,
+      "tokens_seen": 2888499200
+    },
+    {
+      "epoch": 0.75,
+      "objective/train/docs_used": 967898,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2726324796676636,
+      "objective/train/theoretical_loss": 3.358933796221325,
+      "objective/train/tokens_used": 1259089376,
+      "theoretical_loss": 3.358933796221325,
+      "tokens_seen": 2888630272
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012590274434280213,
+      "loss": 1.2586,
+      "theoretical_loss": 3.358933796221325,
+      "tokens_seen": 2888630272
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001258626223720109,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3589220879078314,
+      "tokens_seen": 2888761344
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001258225004012197,
+      "loss": 1.2607,
+      "theoretical_loss": 3.3589103802743083,
+      "tokens_seen": 2888892416
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001257823784304285,
+      "loss": 1.2387,
+      "theoretical_loss": 3.3588986733206854,
+      "tokens_seen": 2889023488
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001257422564596373,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3588869670468924,
+      "tokens_seen": 2889154560
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001257021344888461,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3588752614528583,
+      "tokens_seen": 2889285632
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001256620125180549,
+      "loss": 1.1764,
+      "theoretical_loss": 3.3588635565385134,
+      "tokens_seen": 2889416704
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001256218905472637,
+      "loss": 1.2475,
+      "theoretical_loss": 3.358851852303787,
+      "tokens_seen": 2889547776
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012558176857647247,
+      "loss": 1.2761,
+      "theoretical_loss": 3.3588401487486097,
+      "tokens_seen": 2889678848
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012554164660568125,
+      "loss": 1.2709,
+      "theoretical_loss": 3.35882844587291,
+      "tokens_seen": 2889809920
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012550152463489006,
+      "loss": 1.2757,
+      "theoretical_loss": 3.3588167436766185,
+      "tokens_seen": 2889940992
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012546140266409887,
+      "loss": 1.2009,
+      "theoretical_loss": 3.3588050421596645,
+      "tokens_seen": 2890072064
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012542128069330767,
+      "loss": 1.2586,
+      "theoretical_loss": 3.358793341321978,
+      "tokens_seen": 2890203136
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012538115872251645,
+      "loss": 1.2097,
+      "theoretical_loss": 3.358781641163489,
+      "tokens_seen": 2890334208
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012534103675172526,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3587699416841263,
+      "tokens_seen": 2890465280
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012530091478093406,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3587582428838205,
+      "tokens_seen": 2890596352
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012526079281014282,
+      "loss": 1.2332,
+      "theoretical_loss": 3.358746544762502,
+      "tokens_seen": 2890727424
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012522067083935162,
+      "loss": 1.2097,
+      "theoretical_loss": 3.358734847320099,
+      "tokens_seen": 2890858496
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012518054886856043,
+      "loss": 1.2373,
+      "theoretical_loss": 3.3587231505565427,
+      "tokens_seen": 2890989568
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012514042689776923,
+      "loss": 1.3199,
+      "theoretical_loss": 3.358711454471762,
+      "tokens_seen": 2891120640
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012510030492697801,
+      "loss": 1.2288,
+      "theoretical_loss": 3.3586997590656873,
+      "tokens_seen": 2891251712
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012506018295618682,
+      "loss": 1.1475,
+      "theoretical_loss": 3.3586880643382484,
+      "tokens_seen": 2891382784
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012502006098539563,
+      "loss": 1.1258,
+      "theoretical_loss": 3.3586763702893747,
+      "tokens_seen": 2891513856
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001249799390146044,
+      "loss": 1.1508,
+      "theoretical_loss": 3.3586646769189965,
+      "tokens_seen": 2891644928
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001249398170438132,
+      "loss": 1.2505,
+      "theoretical_loss": 3.358652984227044,
+      "tokens_seen": 2891776000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.000124899695073022,
+      "loss": 1.1614,
+      "theoretical_loss": 3.3586412922134468,
+      "tokens_seen": 2891907072
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012485957310223077,
+      "loss": 1.1776,
+      "theoretical_loss": 3.3586296008781344,
+      "tokens_seen": 2892038144
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012481945113143958,
+      "loss": 1.1804,
+      "theoretical_loss": 3.358617910221037,
+      "tokens_seen": 2892169216
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012477932916064838,
+      "loss": 1.1863,
+      "theoretical_loss": 3.358606220242084,
+      "tokens_seen": 2892300288
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012473920718985716,
+      "loss": 1.2077,
+      "theoretical_loss": 3.3585945309412066,
+      "tokens_seen": 2892431360
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012469908521906597,
+      "loss": 1.2501,
+      "theoretical_loss": 3.358582842318334,
+      "tokens_seen": 2892562432
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012465896324827477,
+      "loss": 1.1929,
+      "theoretical_loss": 3.358571154373396,
+      "tokens_seen": 2892693504
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012461884127748355,
+      "loss": 1.2561,
+      "theoretical_loss": 3.3585594671063226,
+      "tokens_seen": 2892824576
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012457871930669233,
+      "loss": 1.2423,
+      "theoretical_loss": 3.358547780517044,
+      "tokens_seen": 2892955648
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012453859733590114,
+      "loss": 1.2288,
+      "theoretical_loss": 3.35853609460549,
+      "tokens_seen": 2893086720
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012449847536510995,
+      "loss": 1.2665,
+      "theoretical_loss": 3.358524409371591,
+      "tokens_seen": 2893217792
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012445835339431873,
+      "loss": 1.2388,
+      "theoretical_loss": 3.3585127248152764,
+      "tokens_seen": 2893348864
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012441823142352753,
+      "loss": 1.1383,
+      "theoretical_loss": 3.358501040936477,
+      "tokens_seen": 2893479936
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001243781094527363,
+      "loss": 1.144,
+      "theoretical_loss": 3.3584893577351216,
+      "tokens_seen": 2893611008
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012433798748194512,
+      "loss": 1.1635,
+      "theoretical_loss": 3.3584776752111414,
+      "tokens_seen": 2893742080
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001242978655111539,
+      "loss": 1.1846,
+      "theoretical_loss": 3.358465993364466,
+      "tokens_seen": 2893873152
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001242577435403627,
+      "loss": 1.1925,
+      "theoretical_loss": 3.3584543121950254,
+      "tokens_seen": 2894004224
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001242176215695715,
+      "loss": 1.1383,
+      "theoretical_loss": 3.35844263170275,
+      "tokens_seen": 2894135296
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001241774995987803,
+      "loss": 1.16,
+      "theoretical_loss": 3.358430951887569,
+      "tokens_seen": 2894266368
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001241373776279891,
+      "loss": 1.1849,
+      "theoretical_loss": 3.3584192727494138,
+      "tokens_seen": 2894397440
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012409725565719787,
+      "loss": 1.1319,
+      "theoretical_loss": 3.3584075942882134,
+      "tokens_seen": 2894528512
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012405713368640668,
+      "loss": 1.1924,
+      "theoretical_loss": 3.3583959165038983,
+      "tokens_seen": 2894659584
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012401701171561546,
+      "loss": 1.1828,
+      "theoretical_loss": 3.3583842393963987,
+      "tokens_seen": 2894790656
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012397688974482427,
+      "loss": 1.3239,
+      "theoretical_loss": 3.358372562965645,
+      "tokens_seen": 2894921728
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012393676777403307,
+      "loss": 1.1902,
+      "theoretical_loss": 3.3583608872115667,
+      "tokens_seen": 2895052800
+    },
+    {
+      "epoch": 0.75,
+      "objective/train/docs_used": 970103,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2479755878448486,
+      "objective/train/theoretical_loss": 3.3583492121340948,
+      "objective/train/tokens_used": 1265642976,
+      "theoretical_loss": 3.3583492121340948,
+      "tokens_seen": 2895183872
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012389664580324185,
+      "loss": 1.282,
+      "theoretical_loss": 3.3583492121340948,
+      "tokens_seen": 2895183872
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012385652383245066,
+      "loss": 1.2071,
+      "theoretical_loss": 3.358337537733158,
+      "tokens_seen": 2895314944
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012381640186165944,
+      "loss": 1.1472,
+      "theoretical_loss": 3.3583258640086884,
+      "tokens_seen": 2895446016
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012377627989086824,
+      "loss": 1.1673,
+      "theoretical_loss": 3.3583141909606145,
+      "tokens_seen": 2895577088
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012373615792007702,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3583025185888675,
+      "tokens_seen": 2895708160
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012369603594928583,
+      "loss": 1.1903,
+      "theoretical_loss": 3.358290846893377,
+      "tokens_seen": 2895839232
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012365591397849463,
+      "loss": 1.1592,
+      "theoretical_loss": 3.358279175874074,
+      "tokens_seen": 2895970304
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012361579200770341,
+      "loss": 1.232,
+      "theoretical_loss": 3.358267505530888,
+      "tokens_seen": 2896101376
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012357567003691222,
+      "loss": 1.1856,
+      "theoretical_loss": 3.358255835863749,
+      "tokens_seen": 2896232448
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.000123535548066121,
+      "loss": 1.1616,
+      "theoretical_loss": 3.358244166872588,
+      "tokens_seen": 2896363520
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001234954260953298,
+      "loss": 1.1671,
+      "theoretical_loss": 3.358232498557335,
+      "tokens_seen": 2896494592
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001234553041245386,
+      "loss": 1.2687,
+      "theoretical_loss": 3.3582208309179205,
+      "tokens_seen": 2896625664
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001234151821537474,
+      "loss": 1.1983,
+      "theoretical_loss": 3.358209163954274,
+      "tokens_seen": 2896756736
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001233750601829562,
+      "loss": 1.1749,
+      "theoretical_loss": 3.358197497666327,
+      "tokens_seen": 2896887808
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012333493821216498,
+      "loss": 1.189,
+      "theoretical_loss": 3.3581858320540086,
+      "tokens_seen": 2897018880
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012329481624137378,
+      "loss": 1.1537,
+      "theoretical_loss": 3.3581741671172494,
+      "tokens_seen": 2897149952
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012325469427058256,
+      "loss": 1.1667,
+      "theoretical_loss": 3.35816250285598,
+      "tokens_seen": 2897281024
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012321457229979137,
+      "loss": 1.2106,
+      "theoretical_loss": 3.358150839270131,
+      "tokens_seen": 2897412096
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012317445032900017,
+      "loss": 1.1486,
+      "theoretical_loss": 3.3581391763596318,
+      "tokens_seen": 2897543168
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012313432835820895,
+      "loss": 1.1992,
+      "theoretical_loss": 3.358127514124414,
+      "tokens_seen": 2897674240
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012309420638741776,
+      "loss": 1.2326,
+      "theoretical_loss": 3.358115852564407,
+      "tokens_seen": 2897805312
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012305408441662654,
+      "loss": 1.193,
+      "theoretical_loss": 3.358104191679541,
+      "tokens_seen": 2897936384
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012301396244583535,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3580925314697474,
+      "tokens_seen": 2898067456
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012297384047504412,
+      "loss": 1.1992,
+      "theoretical_loss": 3.3580808719349555,
+      "tokens_seen": 2898198528
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012293371850425293,
+      "loss": 1.2056,
+      "theoretical_loss": 3.3580692130750966,
+      "tokens_seen": 2898329600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012289359653346174,
+      "loss": 1.1887,
+      "theoretical_loss": 3.3580575548901006,
+      "tokens_seen": 2898460672
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012285347456267052,
+      "loss": 1.254,
+      "theoretical_loss": 3.358045897379898,
+      "tokens_seen": 2898591744
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012281335259187932,
+      "loss": 1.1587,
+      "theoretical_loss": 3.35803424054442,
+      "tokens_seen": 2898722816
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001227732306210881,
+      "loss": 1.1905,
+      "theoretical_loss": 3.3580225843835954,
+      "tokens_seen": 2898853888
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001227331086502969,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3580109288973556,
+      "tokens_seen": 2898984960
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001226929866795057,
+      "loss": 1.1823,
+      "theoretical_loss": 3.357999274085632,
+      "tokens_seen": 2899116032
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001226528647087145,
+      "loss": 1.2083,
+      "theoretical_loss": 3.357987619948353,
+      "tokens_seen": 2899247104
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001226127427379233,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3579759664854505,
+      "tokens_seen": 2899378176
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012257262076713208,
+      "loss": 1.2396,
+      "theoretical_loss": 3.357964313696855,
+      "tokens_seen": 2899509248
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012253249879634089,
+      "loss": 1.1814,
+      "theoretical_loss": 3.3579526615824964,
+      "tokens_seen": 2899640320
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012249237682554966,
+      "loss": 1.152,
+      "theoretical_loss": 3.357941010142306,
+      "tokens_seen": 2899771392
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012245225485475847,
+      "loss": 1.1896,
+      "theoretical_loss": 3.357929359376213,
+      "tokens_seen": 2899902464
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012241213288396728,
+      "loss": 1.296,
+      "theoretical_loss": 3.3579177092841492,
+      "tokens_seen": 2900033536
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012237201091317606,
+      "loss": 1.18,
+      "theoretical_loss": 3.357906059866045,
+      "tokens_seen": 2900164608
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012233188894238486,
+      "loss": 1.2121,
+      "theoretical_loss": 3.35789441112183,
+      "tokens_seen": 2900295680
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012229176697159364,
+      "loss": 1.222,
+      "theoretical_loss": 3.357882763051436,
+      "tokens_seen": 2900426752
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012225164500080245,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3578711156547927,
+      "tokens_seen": 2900557824
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012221152303001123,
+      "loss": 1.2029,
+      "theoretical_loss": 3.3578594689318306,
+      "tokens_seen": 2900688896
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012217140105922003,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3578478228824813,
+      "tokens_seen": 2900819968
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012213127908842884,
+      "loss": 1.1677,
+      "theoretical_loss": 3.3578361775066745,
+      "tokens_seen": 2900951040
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012209115711763762,
+      "loss": 1.2228,
+      "theoretical_loss": 3.3578245328043415,
+      "tokens_seen": 2901082112
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012205103514684641,
+      "loss": 1.1971,
+      "theoretical_loss": 3.357812888775412,
+      "tokens_seen": 2901213184
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012201091317605522,
+      "loss": 1.1992,
+      "theoretical_loss": 3.357801245419817,
+      "tokens_seen": 2901344256
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012197079120526401,
+      "loss": 1.1818,
+      "theoretical_loss": 3.357789602737488,
+      "tokens_seen": 2901475328
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012193066923447279,
+      "loss": 1.2229,
+      "theoretical_loss": 3.3577779607283547,
+      "tokens_seen": 2901606400
+    },
+    {
+      "epoch": 0.76,
+      "objective/train/docs_used": 972373,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2256826162338257,
+      "objective/train/theoretical_loss": 3.357766319392348,
+      "objective/train/tokens_used": 1272196576,
+      "theoretical_loss": 3.357766319392348,
+      "tokens_seen": 2901737472
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001218905472636816,
+      "loss": 1.2408,
+      "theoretical_loss": 3.357766319392348,
+      "tokens_seen": 2901737472
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012185042529289039,
+      "loss": 1.2481,
+      "theoretical_loss": 3.357754678729399,
+      "tokens_seen": 2901868544
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012181030332209918,
+      "loss": 1.192,
+      "theoretical_loss": 3.357743038739437,
+      "tokens_seen": 2901999616
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012177018135130798,
+      "loss": 1.1296,
+      "theoretical_loss": 3.3577313994223945,
+      "tokens_seen": 2902130688
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012173005938051678,
+      "loss": 1.2355,
+      "theoretical_loss": 3.3577197607782012,
+      "tokens_seen": 2902261760
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012168993740972557,
+      "loss": 1.1912,
+      "theoretical_loss": 3.357708122806788,
+      "tokens_seen": 2902392832
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012164981543893435,
+      "loss": 1.176,
+      "theoretical_loss": 3.357696485508086,
+      "tokens_seen": 2902523904
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012160969346814316,
+      "loss": 1.2,
+      "theoretical_loss": 3.357684848882025,
+      "tokens_seen": 2902654976
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012156957149735195,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3576732129285367,
+      "tokens_seen": 2902786048
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012152944952656076,
+      "loss": 1.2492,
+      "theoretical_loss": 3.3576615776475514,
+      "tokens_seen": 2902917120
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012148932755576954,
+      "loss": 1.2296,
+      "theoretical_loss": 3.357649943039,
+      "tokens_seen": 2903048192
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012144920558497833,
+      "loss": 1.2146,
+      "theoretical_loss": 3.3576383091028132,
+      "tokens_seen": 2903179264
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012140908361418714,
+      "loss": 1.1672,
+      "theoretical_loss": 3.357626675838922,
+      "tokens_seen": 2903310336
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012136896164339592,
+      "loss": 1.2215,
+      "theoretical_loss": 3.357615043247257,
+      "tokens_seen": 2903441408
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012132883967260472,
+      "loss": 1.2387,
+      "theoretical_loss": 3.357603411327749,
+      "tokens_seen": 2903572480
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012128871770181352,
+      "loss": 1.2034,
+      "theoretical_loss": 3.357591780080329,
+      "tokens_seen": 2903703552
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012124859573102232,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3575801495049276,
+      "tokens_seen": 2903834624
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001212084737602311,
+      "loss": 1.3097,
+      "theoretical_loss": 3.3575685196014757,
+      "tokens_seen": 2903965696
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001211683517894399,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3575568903699047,
+      "tokens_seen": 2904096768
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001211282298186487,
+      "loss": 1.1761,
+      "theoretical_loss": 3.3575452618101442,
+      "tokens_seen": 2904227840
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012108810784785749,
+      "loss": 1.1618,
+      "theoretical_loss": 3.3575336339221264,
+      "tokens_seen": 2904358912
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012104798587706629,
+      "loss": 1.1741,
+      "theoretical_loss": 3.357522006705781,
+      "tokens_seen": 2904489984
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012100786390627508,
+      "loss": 1.178,
+      "theoretical_loss": 3.3575103801610404,
+      "tokens_seen": 2904621056
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012096774193548387,
+      "loss": 1.2477,
+      "theoretical_loss": 3.357498754287834,
+      "tokens_seen": 2904752128
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012092761996469266,
+      "loss": 1.2723,
+      "theoretical_loss": 3.3574871290860937,
+      "tokens_seen": 2904883200
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012088749799390146,
+      "loss": 1.1691,
+      "theoretical_loss": 3.35747550455575,
+      "tokens_seen": 2905014272
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012084737602311026,
+      "loss": 1.1703,
+      "theoretical_loss": 3.357463880696734,
+      "tokens_seen": 2905145344
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012080725405231906,
+      "loss": 1.237,
+      "theoretical_loss": 3.3574522575089762,
+      "tokens_seen": 2905276416
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012076713208152785,
+      "loss": 1.2698,
+      "theoretical_loss": 3.357440634992408,
+      "tokens_seen": 2905407488
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012072701011073664,
+      "loss": 1.2661,
+      "theoretical_loss": 3.35742901314696,
+      "tokens_seen": 2905538560
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012068688813994543,
+      "loss": 1.1834,
+      "theoretical_loss": 3.357417391972564,
+      "tokens_seen": 2905669632
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012064676616915424,
+      "loss": 1.1997,
+      "theoretical_loss": 3.35740577146915,
+      "tokens_seen": 2905800704
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012060664419836302,
+      "loss": 1.1753,
+      "theoretical_loss": 3.3573941516366497,
+      "tokens_seen": 2905931776
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012056652222757183,
+      "loss": 1.2049,
+      "theoretical_loss": 3.3573825324749937,
+      "tokens_seen": 2906062848
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012052640025678062,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3573709139841132,
+      "tokens_seen": 2906193920
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001204862782859894,
+      "loss": 1.2182,
+      "theoretical_loss": 3.357359296163939,
+      "tokens_seen": 2906324992
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001204461563151982,
+      "loss": 1.1224,
+      "theoretical_loss": 3.3573476790144023,
+      "tokens_seen": 2906456064
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.000120406034344407,
+      "loss": 1.2408,
+      "theoretical_loss": 3.3573360625354343,
+      "tokens_seen": 2906587136
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001203659123736158,
+      "loss": 1.2442,
+      "theoretical_loss": 3.357324446726966,
+      "tokens_seen": 2906718208
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012032579040282458,
+      "loss": 1.2369,
+      "theoretical_loss": 3.357312831588928,
+      "tokens_seen": 2906849280
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012028566843203339,
+      "loss": 1.2303,
+      "theoretical_loss": 3.357301217121252,
+      "tokens_seen": 2906980352
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012024554646124218,
+      "loss": 1.2449,
+      "theoretical_loss": 3.357289603323869,
+      "tokens_seen": 2907111424
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012020542449045097,
+      "loss": 1.1662,
+      "theoretical_loss": 3.3572779901967094,
+      "tokens_seen": 2907242496
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012016530251965977,
+      "loss": 1.2265,
+      "theoretical_loss": 3.3572663777397054,
+      "tokens_seen": 2907373568
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012012518054886856,
+      "loss": 1.1649,
+      "theoretical_loss": 3.357254765952787,
+      "tokens_seen": 2907504640
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012008505857807737,
+      "loss": 1.1906,
+      "theoretical_loss": 3.3572431548358868,
+      "tokens_seen": 2907635712
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012004493660728616,
+      "loss": 1.2253,
+      "theoretical_loss": 3.357231544388934,
+      "tokens_seen": 2907766784
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012000481463649494,
+      "loss": 1.1695,
+      "theoretical_loss": 3.3572199346118614,
+      "tokens_seen": 2907897856
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011996469266570374,
+      "loss": 1.1904,
+      "theoretical_loss": 3.3572083255045992,
+      "tokens_seen": 2908028928
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011992457069491254,
+      "loss": 1.161,
+      "theoretical_loss": 3.357196717067079,
+      "tokens_seen": 2908160000
+    },
+    {
+      "epoch": 0.76,
+      "objective/train/docs_used": 974455,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2455153465270996,
+      "objective/train/theoretical_loss": 3.357185109299232,
+      "objective/train/tokens_used": 1278750176,
+      "theoretical_loss": 3.357185109299232,
+      "tokens_seen": 2908291072
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011988444872412133,
+      "loss": 1.2108,
+      "theoretical_loss": 3.357185109299232,
+      "tokens_seen": 2908291072
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011984432675333012,
+      "loss": 1.1201,
+      "theoretical_loss": 3.357173502200989,
+      "tokens_seen": 2908422144
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011980420478253893,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3571618957722817,
+      "tokens_seen": 2908553216
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011976408281174772,
+      "loss": 1.2478,
+      "theoretical_loss": 3.357150290013041,
+      "tokens_seen": 2908684288
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001197239608409565,
+      "loss": 1.1958,
+      "theoretical_loss": 3.3571386849231986,
+      "tokens_seen": 2908815360
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001196838388701653,
+      "loss": 1.2588,
+      "theoretical_loss": 3.3571270805026847,
+      "tokens_seen": 2908946432
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001196437168993741,
+      "loss": 1.1743,
+      "theoretical_loss": 3.3571154767514315,
+      "tokens_seen": 2909077504
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001196035949285829,
+      "loss": 1.2327,
+      "theoretical_loss": 3.35710387366937,
+      "tokens_seen": 2909208576
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011956347295779169,
+      "loss": 1.1638,
+      "theoretical_loss": 3.3570922712564313,
+      "tokens_seen": 2909339648
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011952335098700048,
+      "loss": 1.2181,
+      "theoretical_loss": 3.3570806695125466,
+      "tokens_seen": 2909470720
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011948322901620928,
+      "loss": 1.2118,
+      "theoretical_loss": 3.357069068437647,
+      "tokens_seen": 2909601792
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011944310704541806,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3570574680316647,
+      "tokens_seen": 2909732864
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011940298507462687,
+      "loss": 1.228,
+      "theoretical_loss": 3.3570458682945303,
+      "tokens_seen": 2909863936
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011936286310383566,
+      "loss": 1.2551,
+      "theoretical_loss": 3.357034269226175,
+      "tokens_seen": 2909995008
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011932274113304447,
+      "loss": 1.2321,
+      "theoretical_loss": 3.357022670826531,
+      "tokens_seen": 2910126080
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011928261916225325,
+      "loss": 1.2746,
+      "theoretical_loss": 3.3570110730955283,
+      "tokens_seen": 2910257152
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011924249719146204,
+      "loss": 1.1434,
+      "theoretical_loss": 3.3569994760330992,
+      "tokens_seen": 2910388224
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011920237522067085,
+      "loss": 1.1521,
+      "theoretical_loss": 3.3569878796391746,
+      "tokens_seen": 2910519296
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011916225324987964,
+      "loss": 1.2438,
+      "theoretical_loss": 3.356976283913686,
+      "tokens_seen": 2910650368
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011912213127908843,
+      "loss": 1.2576,
+      "theoretical_loss": 3.356964688856565,
+      "tokens_seen": 2910781440
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011908200930829723,
+      "loss": 1.2222,
+      "theoretical_loss": 3.356953094467743,
+      "tokens_seen": 2910912512
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011904188733750602,
+      "loss": 1.242,
+      "theoretical_loss": 3.3569415007471504,
+      "tokens_seen": 2911043584
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011900176536671481,
+      "loss": 1.225,
+      "theoretical_loss": 3.3569299076947203,
+      "tokens_seen": 2911174656
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001189616433959236,
+      "loss": 1.1497,
+      "theoretical_loss": 3.3569183153103825,
+      "tokens_seen": 2911305728
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011892152142513241,
+      "loss": 1.2193,
+      "theoretical_loss": 3.3569067235940695,
+      "tokens_seen": 2911436800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001188813994543412,
+      "loss": 1.2314,
+      "theoretical_loss": 3.3568951325457124,
+      "tokens_seen": 2911567872
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011884127748355,
+      "loss": 1.1834,
+      "theoretical_loss": 3.3568835421652423,
+      "tokens_seen": 2911698944
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011880115551275879,
+      "loss": 1.2656,
+      "theoretical_loss": 3.3568719524525914,
+      "tokens_seen": 2911830016
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011876103354196758,
+      "loss": 1.1398,
+      "theoretical_loss": 3.3568603634076903,
+      "tokens_seen": 2911961088
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011872091157117639,
+      "loss": 1.2163,
+      "theoretical_loss": 3.356848775030471,
+      "tokens_seen": 2912092160
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011868078960038517,
+      "loss": 1.1681,
+      "theoretical_loss": 3.3568371873208647,
+      "tokens_seen": 2912223232
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011864066762959397,
+      "loss": 1.2766,
+      "theoretical_loss": 3.3568256002788033,
+      "tokens_seen": 2912354304
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011860054565880277,
+      "loss": 1.1451,
+      "theoretical_loss": 3.356814013904218,
+      "tokens_seen": 2912485376
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011856042368801154,
+      "loss": 1.1994,
+      "theoretical_loss": 3.3568024281970406,
+      "tokens_seen": 2912616448
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011852030171722035,
+      "loss": 1.2203,
+      "theoretical_loss": 3.356790843157202,
+      "tokens_seen": 2912747520
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011848017974642914,
+      "loss": 1.0858,
+      "theoretical_loss": 3.3567792587846346,
+      "tokens_seen": 2912878592
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011844005777563795,
+      "loss": 1.2202,
+      "theoretical_loss": 3.356767675079269,
+      "tokens_seen": 2913009664
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011839993580484673,
+      "loss": 1.1452,
+      "theoretical_loss": 3.3567560920410373,
+      "tokens_seen": 2913140736
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011835981383405554,
+      "loss": 1.183,
+      "theoretical_loss": 3.3567445096698716,
+      "tokens_seen": 2913271808
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011831969186326433,
+      "loss": 1.1771,
+      "theoretical_loss": 3.356732927965702,
+      "tokens_seen": 2913402880
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011827956989247312,
+      "loss": 1.2394,
+      "theoretical_loss": 3.3567213469284614,
+      "tokens_seen": 2913533952
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011823944792168191,
+      "loss": 1.1865,
+      "theoretical_loss": 3.356709766558081,
+      "tokens_seen": 2913665024
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001181993259508907,
+      "loss": 1.1851,
+      "theoretical_loss": 3.3566981868544925,
+      "tokens_seen": 2913796096
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011815920398009951,
+      "loss": 1.1545,
+      "theoretical_loss": 3.356686607817627,
+      "tokens_seen": 2913927168
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011811908200930829,
+      "loss": 1.2933,
+      "theoretical_loss": 3.3566750294474166,
+      "tokens_seen": 2914058240
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001180789600385171,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3566634517437928,
+      "tokens_seen": 2914189312
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011803883806772589,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3566518747066874,
+      "tokens_seen": 2914320384
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011799871609693468,
+      "loss": 1.22,
+      "theoretical_loss": 3.356640298336032,
+      "tokens_seen": 2914451456
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011795859412614348,
+      "loss": 1.1922,
+      "theoretical_loss": 3.356628722631758,
+      "tokens_seen": 2914582528
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011791847215535227,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3566171475937976,
+      "tokens_seen": 2914713600
+    },
+    {
+      "epoch": 0.77,
+      "objective/train/docs_used": 976720,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.243255615234375,
+      "objective/train/theoretical_loss": 3.3566055732220814,
+      "objective/train/tokens_used": 1285303776,
+      "theoretical_loss": 3.3566055732220814,
+      "tokens_seen": 2914844672
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011787835018456108,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3566055732220814,
+      "tokens_seen": 2914844672
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011783822821376987,
+      "loss": 1.2836,
+      "theoretical_loss": 3.3565939995165426,
+      "tokens_seen": 2914975744
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011779810624297865,
+      "loss": 1.2171,
+      "theoretical_loss": 3.356582426477112,
+      "tokens_seen": 2915106816
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011775798427218745,
+      "loss": 1.3042,
+      "theoretical_loss": 3.356570854103721,
+      "tokens_seen": 2915237888
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011771786230139625,
+      "loss": 1.131,
+      "theoretical_loss": 3.3565592823963026,
+      "tokens_seen": 2915368960
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011767774033060505,
+      "loss": 1.2842,
+      "theoretical_loss": 3.3565477113547875,
+      "tokens_seen": 2915500032
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011763761835981383,
+      "loss": 1.175,
+      "theoretical_loss": 3.3565361409791072,
+      "tokens_seen": 2915631104
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011759749638902264,
+      "loss": 1.2105,
+      "theoretical_loss": 3.356524571269194,
+      "tokens_seen": 2915762176
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011755737441823143,
+      "loss": 1.1685,
+      "theoretical_loss": 3.35651300222498,
+      "tokens_seen": 2915893248
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011751725244744021,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3565014338463963,
+      "tokens_seen": 2916024320
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011747713047664902,
+      "loss": 1.1786,
+      "theoretical_loss": 3.356489866133375,
+      "tokens_seen": 2916155392
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011743700850585781,
+      "loss": 1.2317,
+      "theoretical_loss": 3.356478299085848,
+      "tokens_seen": 2916286464
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011739688653506662,
+      "loss": 1.2032,
+      "theoretical_loss": 3.356466732703747,
+      "tokens_seen": 2916417536
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001173567645642754,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3564551669870037,
+      "tokens_seen": 2916548608
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011731664259348419,
+      "loss": 1.2143,
+      "theoretical_loss": 3.35644360193555,
+      "tokens_seen": 2916679680
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.000117276520622693,
+      "loss": 1.2194,
+      "theoretical_loss": 3.3564320375493177,
+      "tokens_seen": 2916810752
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011723639865190179,
+      "loss": 1.2723,
+      "theoretical_loss": 3.356420473828239,
+      "tokens_seen": 2916941824
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011719627668111058,
+      "loss": 1.2192,
+      "theoretical_loss": 3.356408910772245,
+      "tokens_seen": 2917072896
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011715615471031937,
+      "loss": 1.2447,
+      "theoretical_loss": 3.356397348381268,
+      "tokens_seen": 2917203968
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011711603273952818,
+      "loss": 1.2304,
+      "theoretical_loss": 3.35638578665524,
+      "tokens_seen": 2917335040
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011707591076873696,
+      "loss": 1.2715,
+      "theoretical_loss": 3.3563742255940925,
+      "tokens_seen": 2917466112
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011703578879794575,
+      "loss": 1.201,
+      "theoretical_loss": 3.356362665197758,
+      "tokens_seen": 2917597184
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011699566682715456,
+      "loss": 1.2293,
+      "theoretical_loss": 3.3563511054661683,
+      "tokens_seen": 2917728256
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011695554485636335,
+      "loss": 1.2183,
+      "theoretical_loss": 3.356339546399255,
+      "tokens_seen": 2917859328
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011691542288557214,
+      "loss": 1.1548,
+      "theoretical_loss": 3.35632798799695,
+      "tokens_seen": 2917990400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011687530091478094,
+      "loss": 1.1734,
+      "theoretical_loss": 3.3563164302591852,
+      "tokens_seen": 2918121472
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011683517894398973,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3563048731858927,
+      "tokens_seen": 2918252544
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011679505697319853,
+      "loss": 1.1559,
+      "theoretical_loss": 3.3562933167770046,
+      "tokens_seen": 2918383616
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011675493500240731,
+      "loss": 1.1938,
+      "theoretical_loss": 3.3562817610324527,
+      "tokens_seen": 2918514688
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011671481303161612,
+      "loss": 1.1945,
+      "theoretical_loss": 3.356270205952169,
+      "tokens_seen": 2918645760
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011667469106082491,
+      "loss": 1.2172,
+      "theoretical_loss": 3.3562586515360855,
+      "tokens_seen": 2918776832
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001166345690900337,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3562470977841343,
+      "tokens_seen": 2918907904
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001165944471192425,
+      "loss": 1.1923,
+      "theoretical_loss": 3.356235544696247,
+      "tokens_seen": 2919038976
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011655432514845129,
+      "loss": 1.2198,
+      "theoretical_loss": 3.356223992272356,
+      "tokens_seen": 2919170048
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001165142031776601,
+      "loss": 1.1951,
+      "theoretical_loss": 3.3562124405123934,
+      "tokens_seen": 2919301120
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011647408120686888,
+      "loss": 1.1148,
+      "theoretical_loss": 3.356200889416291,
+      "tokens_seen": 2919432192
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011643395923607768,
+      "loss": 1.1638,
+      "theoretical_loss": 3.356189338983981,
+      "tokens_seen": 2919563264
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011639383726528648,
+      "loss": 1.2243,
+      "theoretical_loss": 3.356177789215395,
+      "tokens_seen": 2919694336
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011635371529449527,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3561662401104657,
+      "tokens_seen": 2919825408
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011631359332370406,
+      "loss": 1.1552,
+      "theoretical_loss": 3.356154691669125,
+      "tokens_seen": 2919956480
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011627347135291285,
+      "loss": 1.2139,
+      "theoretical_loss": 3.356143143891305,
+      "tokens_seen": 2920087552
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011623334938212166,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3561315967769376,
+      "tokens_seen": 2920218624
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011619322741133044,
+      "loss": 1.1391,
+      "theoretical_loss": 3.3561200503259547,
+      "tokens_seen": 2920349696
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011615310544053925,
+      "loss": 1.1996,
+      "theoretical_loss": 3.3561085045382892,
+      "tokens_seen": 2920480768
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011611298346974804,
+      "loss": 1.1904,
+      "theoretical_loss": 3.3560969594138723,
+      "tokens_seen": 2920611840
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011607286149895683,
+      "loss": 1.2107,
+      "theoretical_loss": 3.356085414952637,
+      "tokens_seen": 2920742912
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011603273952816562,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3560738711545146,
+      "tokens_seen": 2920873984
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011599261755737442,
+      "loss": 1.2013,
+      "theoretical_loss": 3.356062328019438,
+      "tokens_seen": 2921005056
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011595249558658322,
+      "loss": 1.1624,
+      "theoretical_loss": 3.356050785547339,
+      "tokens_seen": 2921136128
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011591237361579202,
+      "loss": 1.1701,
+      "theoretical_loss": 3.3560392437381497,
+      "tokens_seen": 2921267200
+    },
+    {
+      "epoch": 0.77,
+      "objective/train/docs_used": 978874,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1926000118255615,
+      "objective/train/theoretical_loss": 3.3560277025918026,
+      "objective/train/tokens_used": 1291857376,
+      "theoretical_loss": 3.3560277025918026,
+      "tokens_seen": 2921398272
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001158722516450008,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3560277025918026,
+      "tokens_seen": 2921398272
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001158321296742096,
+      "loss": 1.1867,
+      "theoretical_loss": 3.35601616210823,
+      "tokens_seen": 2921529344
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001157920077034184,
+      "loss": 1.2518,
+      "theoretical_loss": 3.3560046222873634,
+      "tokens_seen": 2921660416
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011575188573262719,
+      "loss": 1.206,
+      "theoretical_loss": 3.3559930831291354,
+      "tokens_seen": 2921791488
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011571176376183598,
+      "loss": 1.1817,
+      "theoretical_loss": 3.3559815446334786,
+      "tokens_seen": 2921922560
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011567164179104479,
+      "loss": 1.1567,
+      "theoretical_loss": 3.3559700068003244,
+      "tokens_seen": 2922053632
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011563151982025358,
+      "loss": 1.2207,
+      "theoretical_loss": 3.3559584696296056,
+      "tokens_seen": 2922184704
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011559139784946236,
+      "loss": 1.1774,
+      "theoretical_loss": 3.3559469331212552,
+      "tokens_seen": 2922315776
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011555127587867116,
+      "loss": 1.2066,
+      "theoretical_loss": 3.355935397275204,
+      "tokens_seen": 2922446848
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011551115390787996,
+      "loss": 1.2157,
+      "theoretical_loss": 3.355923862091385,
+      "tokens_seen": 2922577920
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011547103193708876,
+      "loss": 1.2188,
+      "theoretical_loss": 3.3559123275697305,
+      "tokens_seen": 2922708992
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011543090996629754,
+      "loss": 1.1624,
+      "theoretical_loss": 3.355900793710173,
+      "tokens_seen": 2922840064
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011539078799550633,
+      "loss": 1.2307,
+      "theoretical_loss": 3.355889260512644,
+      "tokens_seen": 2922971136
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011535066602471514,
+      "loss": 1.1789,
+      "theoretical_loss": 3.3558777279770764,
+      "tokens_seen": 2923102208
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011531054405392392,
+      "loss": 1.1333,
+      "theoretical_loss": 3.355866196103403,
+      "tokens_seen": 2923233280
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011527042208313273,
+      "loss": 1.1032,
+      "theoretical_loss": 3.355854664891555,
+      "tokens_seen": 2923364352
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011523030011234152,
+      "loss": 1.1894,
+      "theoretical_loss": 3.355843134341466,
+      "tokens_seen": 2923495424
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011519017814155033,
+      "loss": 1.2561,
+      "theoretical_loss": 3.3558316044530674,
+      "tokens_seen": 2923626496
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001151500561707591,
+      "loss": 1.25,
+      "theoretical_loss": 3.355820075226292,
+      "tokens_seen": 2923757568
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001151099341999679,
+      "loss": 1.1831,
+      "theoretical_loss": 3.3558085466610716,
+      "tokens_seen": 2923888640
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001150698122291767,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3557970187573396,
+      "tokens_seen": 2924019712
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001150296902583855,
+      "loss": 1.2222,
+      "theoretical_loss": 3.355785491515028,
+      "tokens_seen": 2924150784
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011498956828759429,
+      "loss": 1.1122,
+      "theoretical_loss": 3.3557739649340683,
+      "tokens_seen": 2924281856
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011494944631680308,
+      "loss": 1.1722,
+      "theoretical_loss": 3.355762439014394,
+      "tokens_seen": 2924412928
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011490932434601187,
+      "loss": 1.2064,
+      "theoretical_loss": 3.355750913755937,
+      "tokens_seen": 2924544000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011486920237522068,
+      "loss": 1.2476,
+      "theoretical_loss": 3.3557393891586305,
+      "tokens_seen": 2924675072
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011482908040442946,
+      "loss": 1.1556,
+      "theoretical_loss": 3.355727865222406,
+      "tokens_seen": 2924806144
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011478895843363827,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3557163419471965,
+      "tokens_seen": 2924937216
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011474883646284706,
+      "loss": 1.2847,
+      "theoretical_loss": 3.355704819332934,
+      "tokens_seen": 2925068288
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011470871449205585,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3556932973795517,
+      "tokens_seen": 2925199360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011466859252126464,
+      "loss": 1.2577,
+      "theoretical_loss": 3.355681776086981,
+      "tokens_seen": 2925330432
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011462847055047344,
+      "loss": 1.1867,
+      "theoretical_loss": 3.3556702554551556,
+      "tokens_seen": 2925461504
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011458834857968224,
+      "loss": 1.234,
+      "theoretical_loss": 3.3556587354840075,
+      "tokens_seen": 2925592576
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011454822660889102,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3556472161734687,
+      "tokens_seen": 2925723648
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011450810463809983,
+      "loss": 1.1823,
+      "theoretical_loss": 3.3556356975234727,
+      "tokens_seen": 2925854720
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011446798266730862,
+      "loss": 1.2231,
+      "theoretical_loss": 3.355624179533951,
+      "tokens_seen": 2925985792
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011442786069651743,
+      "loss": 1.193,
+      "theoretical_loss": 3.355612662204837,
+      "tokens_seen": 2926116864
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011438773872572621,
+      "loss": 1.2136,
+      "theoretical_loss": 3.355601145536063,
+      "tokens_seen": 2926247936
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.000114347616754935,
+      "loss": 1.1749,
+      "theoretical_loss": 3.3555896295275613,
+      "tokens_seen": 2926379008
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011430749478414381,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3555781141792647,
+      "tokens_seen": 2926510080
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011426737281335259,
+      "loss": 1.308,
+      "theoretical_loss": 3.3555665994911057,
+      "tokens_seen": 2926641152
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011422725084256139,
+      "loss": 1.2195,
+      "theoretical_loss": 3.355555085463017,
+      "tokens_seen": 2926772224
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011418712887177019,
+      "loss": 1.2513,
+      "theoretical_loss": 3.3555435720949314,
+      "tokens_seen": 2926903296
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011414700690097898,
+      "loss": 1.239,
+      "theoretical_loss": 3.355532059386781,
+      "tokens_seen": 2927034368
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011410688493018777,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3555205473384984,
+      "tokens_seen": 2927165440
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011406676295939656,
+      "loss": 1.1959,
+      "theoretical_loss": 3.355509035950017,
+      "tokens_seen": 2927296512
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011402664098860537,
+      "loss": 1.2179,
+      "theoretical_loss": 3.355497525221269,
+      "tokens_seen": 2927427584
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011398651901781416,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3554860151521866,
+      "tokens_seen": 2927558656
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011394639704702294,
+      "loss": 1.174,
+      "theoretical_loss": 3.355474505742703,
+      "tokens_seen": 2927689728
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011390627507623175,
+      "loss": 1.2588,
+      "theoretical_loss": 3.3554629969927507,
+      "tokens_seen": 2927820800
+    },
+    {
+      "epoch": 0.77,
+      "objective/train/docs_used": 981126,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1865119934082031,
+      "objective/train/theoretical_loss": 3.3554514889022626,
+      "objective/train/tokens_used": 1298410976,
+      "theoretical_loss": 3.3554514889022626,
+      "tokens_seen": 2927951872
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011386615310544054,
+      "loss": 1.2114,
+      "theoretical_loss": 3.3554514889022626,
+      "tokens_seen": 2927951872
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011382603113464933,
+      "loss": 1.1983,
+      "theoretical_loss": 3.355439981471171,
+      "tokens_seen": 2928082944
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011378590916385813,
+      "loss": 1.22,
+      "theoretical_loss": 3.3554284746994094,
+      "tokens_seen": 2928214016
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011374578719306693,
+      "loss": 1.1816,
+      "theoretical_loss": 3.3554169685869093,
+      "tokens_seen": 2928345088
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011370566522227573,
+      "loss": 1.0824,
+      "theoretical_loss": 3.3554054631336045,
+      "tokens_seen": 2928476160
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001136655432514845,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3553939583394268,
+      "tokens_seen": 2928607232
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011362542128069331,
+      "loss": 1.3158,
+      "theoretical_loss": 3.35538245420431,
+      "tokens_seen": 2928738304
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001135852993099021,
+      "loss": 1.1894,
+      "theoretical_loss": 3.355370950728186,
+      "tokens_seen": 2928869376
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011354517733911091,
+      "loss": 1.228,
+      "theoretical_loss": 3.355359447910988,
+      "tokens_seen": 2929000448
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011350505536831969,
+      "loss": 1.2303,
+      "theoretical_loss": 3.3553479457526487,
+      "tokens_seen": 2929131520
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001134649333975285,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3553364442531004,
+      "tokens_seen": 2929262592
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011342481142673729,
+      "loss": 1.2176,
+      "theoretical_loss": 3.3553249434122767,
+      "tokens_seen": 2929393664
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011338468945594607,
+      "loss": 1.1907,
+      "theoretical_loss": 3.3553134432301097,
+      "tokens_seen": 2929524736
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011334456748515487,
+      "loss": 1.2199,
+      "theoretical_loss": 3.355301943706533,
+      "tokens_seen": 2929655808
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011330444551436367,
+      "loss": 1.1931,
+      "theoretical_loss": 3.3552904448414784,
+      "tokens_seen": 2929786880
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011326432354357247,
+      "loss": 1.2451,
+      "theoretical_loss": 3.355278946634879,
+      "tokens_seen": 2929917952
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011322420157278125,
+      "loss": 1.1771,
+      "theoretical_loss": 3.3552674490866687,
+      "tokens_seen": 2930049024
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011318407960199004,
+      "loss": 1.2656,
+      "theoretical_loss": 3.3552559521967793,
+      "tokens_seen": 2930180096
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011314395763119885,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3552444559651438,
+      "tokens_seen": 2930311168
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011310383566040764,
+      "loss": 1.1833,
+      "theoretical_loss": 3.355232960391695,
+      "tokens_seen": 2930442240
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011306371368961644,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3552214654763666,
+      "tokens_seen": 2930573312
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011302359171882523,
+      "loss": 1.2061,
+      "theoretical_loss": 3.3552099712190904,
+      "tokens_seen": 2930704384
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011298346974803404,
+      "loss": 1.1814,
+      "theoretical_loss": 3.3551984776198,
+      "tokens_seen": 2930835456
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011294334777724281,
+      "loss": 1.2196,
+      "theoretical_loss": 3.355186984678428,
+      "tokens_seen": 2930966528
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011290322580645161,
+      "loss": 1.1714,
+      "theoretical_loss": 3.355175492394907,
+      "tokens_seen": 2931097600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011286310383566041,
+      "loss": 1.1974,
+      "theoretical_loss": 3.3551640007691708,
+      "tokens_seen": 2931228672
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001128229818648692,
+      "loss": 1.2411,
+      "theoretical_loss": 3.355152509801152,
+      "tokens_seen": 2931359744
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.000112782859894078,
+      "loss": 1.1794,
+      "theoretical_loss": 3.355141019490783,
+      "tokens_seen": 2931490816
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011274273792328679,
+      "loss": 1.2128,
+      "theoretical_loss": 3.3551295298379973,
+      "tokens_seen": 2931621888
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011270261595249558,
+      "loss": 1.2303,
+      "theoretical_loss": 3.3551180408427275,
+      "tokens_seen": 2931752960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011266249398170439,
+      "loss": 1.2176,
+      "theoretical_loss": 3.3551065525049073,
+      "tokens_seen": 2931884032
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011262237201091317,
+      "loss": 1.2222,
+      "theoretical_loss": 3.355095064824469,
+      "tokens_seen": 2932015104
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011258225004012198,
+      "loss": 1.1822,
+      "theoretical_loss": 3.355083577801346,
+      "tokens_seen": 2932146176
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011254212806933077,
+      "loss": 1.1504,
+      "theoretical_loss": 3.3550720914354706,
+      "tokens_seen": 2932277248
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011250200609853958,
+      "loss": 1.2328,
+      "theoretical_loss": 3.3550606057267767,
+      "tokens_seen": 2932408320
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011246188412774835,
+      "loss": 1.2708,
+      "theoretical_loss": 3.3550491206751967,
+      "tokens_seen": 2932539392
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011242176215695715,
+      "loss": 1.2475,
+      "theoretical_loss": 3.355037636280664,
+      "tokens_seen": 2932670464
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011238164018616595,
+      "loss": 1.2014,
+      "theoretical_loss": 3.355026152543112,
+      "tokens_seen": 2932801536
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011234151821537473,
+      "loss": 1.1551,
+      "theoretical_loss": 3.3550146694624727,
+      "tokens_seen": 2932932608
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011230139624458354,
+      "loss": 1.2778,
+      "theoretical_loss": 3.35500318703868,
+      "tokens_seen": 2933063680
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011226127427379233,
+      "loss": 1.1723,
+      "theoretical_loss": 3.354991705271667,
+      "tokens_seen": 2933194752
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011222115230300112,
+      "loss": 1.2213,
+      "theoretical_loss": 3.354980224161366,
+      "tokens_seen": 2933325824
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011218103033220992,
+      "loss": 1.1844,
+      "theoretical_loss": 3.354968743707711,
+      "tokens_seen": 2933456896
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011214090836141871,
+      "loss": 1.2761,
+      "theoretical_loss": 3.3549572639106344,
+      "tokens_seen": 2933587968
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011210078639062752,
+      "loss": 1.1598,
+      "theoretical_loss": 3.35494578477007,
+      "tokens_seen": 2933719040
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011206066441983631,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3549343062859505,
+      "tokens_seen": 2933850112
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001120205424490451,
+      "loss": 1.1693,
+      "theoretical_loss": 3.3549228284582093,
+      "tokens_seen": 2933981184
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001119804204782539,
+      "loss": 1.1906,
+      "theoretical_loss": 3.354911351286779,
+      "tokens_seen": 2934112256
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011194029850746269,
+      "loss": 1.3079,
+      "theoretical_loss": 3.3548998747715935,
+      "tokens_seen": 2934243328
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011190017653667148,
+      "loss": 1.2516,
+      "theoretical_loss": 3.3548883989125855,
+      "tokens_seen": 2934374400
+    },
+    {
+      "epoch": 0.78,
+      "objective/train/docs_used": 983352,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.105314016342163,
+      "objective/train/theoretical_loss": 3.3548769237096883,
+      "objective/train/tokens_used": 1304964576,
+      "theoretical_loss": 3.3548769237096883,
+      "tokens_seen": 2934505472
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011186005456588027,
+      "loss": 1.1422,
+      "theoretical_loss": 3.3548769237096883,
+      "tokens_seen": 2934505472
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011181993259508908,
+      "loss": 1.1839,
+      "theoretical_loss": 3.3548654491628347,
+      "tokens_seen": 2934636544
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011177981062429787,
+      "loss": 1.1698,
+      "theoretical_loss": 3.3548539752719586,
+      "tokens_seen": 2934767616
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011173968865350665,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3548425020369925,
+      "tokens_seen": 2934898688
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011169956668271546,
+      "loss": 1.2824,
+      "theoretical_loss": 3.3548310294578703,
+      "tokens_seen": 2935029760
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011165944471192425,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3548195575345243,
+      "tokens_seen": 2935160832
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011161932274113306,
+      "loss": 1.3179,
+      "theoretical_loss": 3.354808086266889,
+      "tokens_seen": 2935291904
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011157920077034184,
+      "loss": 1.3239,
+      "theoretical_loss": 3.3547966156548967,
+      "tokens_seen": 2935422976
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011153907879955064,
+      "loss": 1.2142,
+      "theoretical_loss": 3.354785145698481,
+      "tokens_seen": 2935554048
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011149895682875944,
+      "loss": 1.2781,
+      "theoretical_loss": 3.354773676397575,
+      "tokens_seen": 2935685120
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011145883485796821,
+      "loss": 1.269,
+      "theoretical_loss": 3.354762207752112,
+      "tokens_seen": 2935816192
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011141871288717702,
+      "loss": 1.2929,
+      "theoretical_loss": 3.354750739762025,
+      "tokens_seen": 2935947264
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011137859091638581,
+      "loss": 1.1985,
+      "theoretical_loss": 3.3547392724272482,
+      "tokens_seen": 2936078336
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011133846894559462,
+      "loss": 1.1879,
+      "theoretical_loss": 3.354727805747714,
+      "tokens_seen": 2936209408
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001112983469748034,
+      "loss": 1.2523,
+      "theoretical_loss": 3.3547163397233564,
+      "tokens_seen": 2936340480
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011125822500401219,
+      "loss": 1.1912,
+      "theoretical_loss": 3.354704874354108,
+      "tokens_seen": 2936471552
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.000111218103033221,
+      "loss": 1.251,
+      "theoretical_loss": 3.3546934096399026,
+      "tokens_seen": 2936602624
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011117798106242979,
+      "loss": 1.3168,
+      "theoretical_loss": 3.3546819455806736,
+      "tokens_seen": 2936733696
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011113785909163858,
+      "loss": 1.3331,
+      "theoretical_loss": 3.3546704821763536,
+      "tokens_seen": 2936864768
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011109773712084738,
+      "loss": 1.2092,
+      "theoretical_loss": 3.354659019426877,
+      "tokens_seen": 2936995840
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011105761515005618,
+      "loss": 1.3028,
+      "theoretical_loss": 3.354647557332177,
+      "tokens_seen": 2937126912
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011101749317926496,
+      "loss": 1.2877,
+      "theoretical_loss": 3.354636095892186,
+      "tokens_seen": 2937257984
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011097737120847375,
+      "loss": 1.2892,
+      "theoretical_loss": 3.3546246351068385,
+      "tokens_seen": 2937389056
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011093724923768256,
+      "loss": 1.297,
+      "theoretical_loss": 3.3546131749760675,
+      "tokens_seen": 2937520128
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011089712726689135,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3546017154998067,
+      "tokens_seen": 2937651200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011085700529610015,
+      "loss": 1.2829,
+      "theoretical_loss": 3.3545902566779886,
+      "tokens_seen": 2937782272
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011081688332530894,
+      "loss": 1.1959,
+      "theoretical_loss": 3.3545787985105475,
+      "tokens_seen": 2937913344
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011077676135451773,
+      "loss": 1.1775,
+      "theoretical_loss": 3.3545673409974164,
+      "tokens_seen": 2938044416
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011073663938372654,
+      "loss": 1.243,
+      "theoretical_loss": 3.354555884138529,
+      "tokens_seen": 2938175488
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011069651741293532,
+      "loss": 1.1982,
+      "theoretical_loss": 3.354544427933819,
+      "tokens_seen": 2938306560
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011065639544214412,
+      "loss": 1.3092,
+      "theoretical_loss": 3.354532972383219,
+      "tokens_seen": 2938437632
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011061627347135292,
+      "loss": 1.2887,
+      "theoretical_loss": 3.354521517486664,
+      "tokens_seen": 2938568704
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011057615150056171,
+      "loss": 1.3045,
+      "theoretical_loss": 3.354510063244086,
+      "tokens_seen": 2938699776
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001105360295297705,
+      "loss": 1.2953,
+      "theoretical_loss": 3.3544986096554186,
+      "tokens_seen": 2938830848
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001104959075589793,
+      "loss": 1.2788,
+      "theoretical_loss": 3.354487156720596,
+      "tokens_seen": 2938961920
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001104557855881881,
+      "loss": 1.3053,
+      "theoretical_loss": 3.3544757044395515,
+      "tokens_seen": 2939092992
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011041566361739688,
+      "loss": 1.2957,
+      "theoretical_loss": 3.3544642528122184,
+      "tokens_seen": 2939224064
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011037554164660569,
+      "loss": 1.3958,
+      "theoretical_loss": 3.354452801838531,
+      "tokens_seen": 2939355136
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011033541967581448,
+      "loss": 1.2427,
+      "theoretical_loss": 3.3544413515184215,
+      "tokens_seen": 2939486208
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011029529770502327,
+      "loss": 1.2242,
+      "theoretical_loss": 3.3544299018518244,
+      "tokens_seen": 2939617280
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011025517573423206,
+      "loss": 1.2289,
+      "theoretical_loss": 3.354418452838673,
+      "tokens_seen": 2939748352
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011021505376344086,
+      "loss": 1.2009,
+      "theoretical_loss": 3.3544070044789014,
+      "tokens_seen": 2939879424
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011017493179264966,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3543955567724426,
+      "tokens_seen": 2940010496
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011013480982185846,
+      "loss": 1.312,
+      "theoretical_loss": 3.35438410971923,
+      "tokens_seen": 2940141568
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011009468785106725,
+      "loss": 1.2243,
+      "theoretical_loss": 3.354372663319198,
+      "tokens_seen": 2940272640
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011005456588027604,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3543612175722792,
+      "tokens_seen": 2940403712
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011001444390948483,
+      "loss": 1.2572,
+      "theoretical_loss": 3.3543497724784084,
+      "tokens_seen": 2940534784
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010997432193869363,
+      "loss": 1.3532,
+      "theoretical_loss": 3.354338328037518,
+      "tokens_seen": 2940665856
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010993419996790242,
+      "loss": 1.2852,
+      "theoretical_loss": 3.3543268842495424,
+      "tokens_seen": 2940796928
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010989407799711123,
+      "loss": 1.2672,
+      "theoretical_loss": 3.354315441114415,
+      "tokens_seen": 2940928000
+    },
+    {
+      "epoch": 0.78,
+      "objective/train/docs_used": 985430,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1423430442810059,
+      "objective/train/theoretical_loss": 3.35430399863207,
+      "objective/train/tokens_used": 1311518176,
+      "theoretical_loss": 3.35430399863207,
+      "tokens_seen": 2941059072
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010985395602632002,
+      "loss": 1.1697,
+      "theoretical_loss": 3.35430399863207,
+      "tokens_seen": 2941059072
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001098138340555288,
+      "loss": 1.3587,
+      "theoretical_loss": 3.35429255680244,
+      "tokens_seen": 2941190144
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001097737120847376,
+      "loss": 1.2788,
+      "theoretical_loss": 3.3542811156254597,
+      "tokens_seen": 2941321216
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001097335901139464,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3542696751010626,
+      "tokens_seen": 2941452288
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001096934681431552,
+      "loss": 1.2989,
+      "theoretical_loss": 3.354258235229182,
+      "tokens_seen": 2941583360
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010965334617236398,
+      "loss": 1.2743,
+      "theoretical_loss": 3.3542467960097517,
+      "tokens_seen": 2941714432
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010961322420157279,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3542353574427057,
+      "tokens_seen": 2941845504
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010957310223078158,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3542239195279775,
+      "tokens_seen": 2941976576
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010953298025999036,
+      "loss": 1.2503,
+      "theoretical_loss": 3.354212482265501,
+      "tokens_seen": 2942107648
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010949285828919917,
+      "loss": 1.1939,
+      "theoretical_loss": 3.3542010456552096,
+      "tokens_seen": 2942238720
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010945273631840796,
+      "loss": 1.2254,
+      "theoretical_loss": 3.3541896096970376,
+      "tokens_seen": 2942369792
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010941261434761677,
+      "loss": 1.2577,
+      "theoretical_loss": 3.3541781743909183,
+      "tokens_seen": 2942500864
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010937249237682555,
+      "loss": 1.2528,
+      "theoretical_loss": 3.354166739736786,
+      "tokens_seen": 2942631936
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010933237040603435,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3541553057345737,
+      "tokens_seen": 2942763008
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010929224843524314,
+      "loss": 1.1535,
+      "theoretical_loss": 3.3541438723842156,
+      "tokens_seen": 2942894080
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010925212646445194,
+      "loss": 1.1337,
+      "theoretical_loss": 3.354132439685646,
+      "tokens_seen": 2943025152
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010921200449366073,
+      "loss": 1.2544,
+      "theoretical_loss": 3.354121007638798,
+      "tokens_seen": 2943156224
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010917188252286952,
+      "loss": 1.3169,
+      "theoretical_loss": 3.354109576243606,
+      "tokens_seen": 2943287296
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010913176055207833,
+      "loss": 1.2329,
+      "theoretical_loss": 3.354098145500003,
+      "tokens_seen": 2943418368
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010909163858128711,
+      "loss": 1.2431,
+      "theoretical_loss": 3.3540867154079237,
+      "tokens_seen": 2943549440
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001090515166104959,
+      "loss": 1.275,
+      "theoretical_loss": 3.3540752859673013,
+      "tokens_seen": 2943680512
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010901139463970471,
+      "loss": 1.2686,
+      "theoretical_loss": 3.3540638571780708,
+      "tokens_seen": 2943811584
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001089712726689135,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3540524290401645,
+      "tokens_seen": 2943942656
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001089311506981223,
+      "loss": 1.2801,
+      "theoretical_loss": 3.3540410015535174,
+      "tokens_seen": 2944073728
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010889102872733109,
+      "loss": 1.2507,
+      "theoretical_loss": 3.354029574718063,
+      "tokens_seen": 2944204800
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010885090675653989,
+      "loss": 1.2045,
+      "theoretical_loss": 3.3540181485337355,
+      "tokens_seen": 2944335872
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010881078478574869,
+      "loss": 1.1669,
+      "theoretical_loss": 3.354006723000468,
+      "tokens_seen": 2944466944
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010877066281495746,
+      "loss": 1.2036,
+      "theoretical_loss": 3.3539952981181953,
+      "tokens_seen": 2944598016
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010873054084416627,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3539838738868513,
+      "tokens_seen": 2944729088
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010869041887337506,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3539724503063697,
+      "tokens_seen": 2944860160
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010865029690258386,
+      "loss": 1.1852,
+      "theoretical_loss": 3.353961027376684,
+      "tokens_seen": 2944991232
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010861017493179265,
+      "loss": 1.2704,
+      "theoretical_loss": 3.3539496050977284,
+      "tokens_seen": 2945122304
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010857005296100144,
+      "loss": 1.2395,
+      "theoretical_loss": 3.3539381834694377,
+      "tokens_seen": 2945253376
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010852993099021025,
+      "loss": 1.1693,
+      "theoretical_loss": 3.353926762491745,
+      "tokens_seen": 2945384448
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010848980901941903,
+      "loss": 1.2789,
+      "theoretical_loss": 3.3539153421645844,
+      "tokens_seen": 2945515520
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010844968704862783,
+      "loss": 1.1263,
+      "theoretical_loss": 3.3539039224878904,
+      "tokens_seen": 2945646592
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010840956507783663,
+      "loss": 1.3014,
+      "theoretical_loss": 3.353892503461596,
+      "tokens_seen": 2945777664
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010836944310704543,
+      "loss": 1.2077,
+      "theoretical_loss": 3.3538810850856366,
+      "tokens_seen": 2945908736
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010832932113625421,
+      "loss": 1.1703,
+      "theoretical_loss": 3.353869667359945,
+      "tokens_seen": 2946039808
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.000108289199165463,
+      "loss": 1.1684,
+      "theoretical_loss": 3.353858250284456,
+      "tokens_seen": 2946170880
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010824907719467181,
+      "loss": 1.2957,
+      "theoretical_loss": 3.3538468338591034,
+      "tokens_seen": 2946301952
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010820895522388059,
+      "loss": 1.1697,
+      "theoretical_loss": 3.353835418083821,
+      "tokens_seen": 2946433024
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001081688332530894,
+      "loss": 1.3279,
+      "theoretical_loss": 3.353824002958543,
+      "tokens_seen": 2946564096
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010812871128229819,
+      "loss": 1.2933,
+      "theoretical_loss": 3.353812588483204,
+      "tokens_seen": 2946695168
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010808858931150698,
+      "loss": 1.2178,
+      "theoretical_loss": 3.3538011746577374,
+      "tokens_seen": 2946826240
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010804846734071577,
+      "loss": 1.2036,
+      "theoretical_loss": 3.3537897614820777,
+      "tokens_seen": 2946957312
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010800834536992457,
+      "loss": 1.3248,
+      "theoretical_loss": 3.353778348956159,
+      "tokens_seen": 2947088384
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010796822339913337,
+      "loss": 1.2375,
+      "theoretical_loss": 3.3537669370799152,
+      "tokens_seen": 2947219456
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010792810142834217,
+      "loss": 1.2247,
+      "theoretical_loss": 3.3537555258532805,
+      "tokens_seen": 2947350528
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010788797945755096,
+      "loss": 1.2379,
+      "theoretical_loss": 3.353744115276189,
+      "tokens_seen": 2947481600
+    },
+    {
+      "epoch": 0.79,
+      "objective/train/docs_used": 987667,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2303879261016846,
+      "objective/train/theoretical_loss": 3.3537327053485746,
+      "objective/train/tokens_used": 1318071776,
+      "theoretical_loss": 3.3537327053485746,
+      "tokens_seen": 2947612672
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010784785748675975,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3537327053485746,
+      "tokens_seen": 2947612672
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010780773551596854,
+      "loss": 1.2215,
+      "theoretical_loss": 3.353721296070372,
+      "tokens_seen": 2947743744
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010776761354517735,
+      "loss": 1.346,
+      "theoretical_loss": 3.353709887441515,
+      "tokens_seen": 2947874816
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010772749157438613,
+      "loss": 1.3275,
+      "theoretical_loss": 3.353698479461938,
+      "tokens_seen": 2948005888
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010768736960359494,
+      "loss": 1.2856,
+      "theoretical_loss": 3.3536870721315757,
+      "tokens_seen": 2948136960
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010764724763280373,
+      "loss": 1.2888,
+      "theoretical_loss": 3.353675665450361,
+      "tokens_seen": 2948268032
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010760712566201251,
+      "loss": 1.2363,
+      "theoretical_loss": 3.353664259418229,
+      "tokens_seen": 2948399104
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010756700369122131,
+      "loss": 1.2177,
+      "theoretical_loss": 3.353652854035113,
+      "tokens_seen": 2948530176
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010752688172043011,
+      "loss": 1.2316,
+      "theoretical_loss": 3.353641449300949,
+      "tokens_seen": 2948661248
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010748675974963891,
+      "loss": 1.3101,
+      "theoretical_loss": 3.3536300452156693,
+      "tokens_seen": 2948792320
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010744663777884769,
+      "loss": 1.223,
+      "theoretical_loss": 3.3536186417792093,
+      "tokens_seen": 2948923392
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001074065158080565,
+      "loss": 1.2557,
+      "theoretical_loss": 3.353607238991503,
+      "tokens_seen": 2949054464
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010736639383726529,
+      "loss": 1.2451,
+      "theoretical_loss": 3.3535958368524845,
+      "tokens_seen": 2949185536
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010732627186647408,
+      "loss": 1.2542,
+      "theoretical_loss": 3.3535844353620883,
+      "tokens_seen": 2949316608
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010728614989568288,
+      "loss": 1.2419,
+      "theoretical_loss": 3.353573034520248,
+      "tokens_seen": 2949447680
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010724602792489167,
+      "loss": 1.1325,
+      "theoretical_loss": 3.353561634326899,
+      "tokens_seen": 2949578752
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010720590595410048,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3535502347819746,
+      "tokens_seen": 2949709824
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010716578398330926,
+      "loss": 1.2501,
+      "theoretical_loss": 3.35353883588541,
+      "tokens_seen": 2949840896
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010712566201251805,
+      "loss": 1.1931,
+      "theoretical_loss": 3.3535274376371387,
+      "tokens_seen": 2949971968
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010708554004172685,
+      "loss": 1.262,
+      "theoretical_loss": 3.3535160400370954,
+      "tokens_seen": 2950103040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010704541807093565,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3535046430852145,
+      "tokens_seen": 2950234112
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010700529610014444,
+      "loss": 1.2114,
+      "theoretical_loss": 3.3534932467814302,
+      "tokens_seen": 2950365184
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010696517412935323,
+      "loss": 1.2916,
+      "theoretical_loss": 3.353481851125677,
+      "tokens_seen": 2950496256
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010692505215856204,
+      "loss": 1.2423,
+      "theoretical_loss": 3.3534704561178894,
+      "tokens_seen": 2950627328
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010688493018777083,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3534590617580013,
+      "tokens_seen": 2950758400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010684480821697961,
+      "loss": 1.242,
+      "theoretical_loss": 3.3534476680459475,
+      "tokens_seen": 2950889472
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010680468624618842,
+      "loss": 1.3152,
+      "theoretical_loss": 3.3534362749816617,
+      "tokens_seen": 2951020544
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010676456427539721,
+      "loss": 1.2209,
+      "theoretical_loss": 3.353424882565079,
+      "tokens_seen": 2951151616
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.000106724442304606,
+      "loss": 1.1836,
+      "theoretical_loss": 3.353413490796134,
+      "tokens_seen": 2951282688
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001066843203338148,
+      "loss": 1.2348,
+      "theoretical_loss": 3.353402099674761,
+      "tokens_seen": 2951413760
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010664419836302359,
+      "loss": 1.2488,
+      "theoretical_loss": 3.3533907092008937,
+      "tokens_seen": 2951544832
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001066040763922324,
+      "loss": 1.214,
+      "theoretical_loss": 3.353379319374467,
+      "tokens_seen": 2951675904
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010656395442144117,
+      "loss": 1.1485,
+      "theoretical_loss": 3.353367930195416,
+      "tokens_seen": 2951806976
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010652383245064998,
+      "loss": 1.2482,
+      "theoretical_loss": 3.3533565416636737,
+      "tokens_seen": 2951938048
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010648371047985877,
+      "loss": 1.2281,
+      "theoretical_loss": 3.353345153779176,
+      "tokens_seen": 2952069120
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010644358850906758,
+      "loss": 1.1673,
+      "theoretical_loss": 3.3533337665418568,
+      "tokens_seen": 2952200192
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010640346653827636,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3533223799516505,
+      "tokens_seen": 2952331264
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010636334456748515,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3533109940084915,
+      "tokens_seen": 2952462336
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010632322259669396,
+      "loss": 1.3023,
+      "theoretical_loss": 3.3532996087123146,
+      "tokens_seen": 2952593408
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010628310062590274,
+      "loss": 1.1834,
+      "theoretical_loss": 3.3532882240630544,
+      "tokens_seen": 2952724480
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010624297865511154,
+      "loss": 1.218,
+      "theoretical_loss": 3.353276840060645,
+      "tokens_seen": 2952855552
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010620285668432034,
+      "loss": 1.2252,
+      "theoretical_loss": 3.3532654567050217,
+      "tokens_seen": 2952986624
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010616273471352913,
+      "loss": 1.2793,
+      "theoretical_loss": 3.353254073996118,
+      "tokens_seen": 2953117696
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010612261274273792,
+      "loss": 1.1979,
+      "theoretical_loss": 3.353242691933869,
+      "tokens_seen": 2953248768
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010608249077194671,
+      "loss": 1.245,
+      "theoretical_loss": 3.3532313105182094,
+      "tokens_seen": 2953379840
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010604236880115552,
+      "loss": 1.1674,
+      "theoretical_loss": 3.353219929749074,
+      "tokens_seen": 2953510912
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010600224683036431,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3532085496263964,
+      "tokens_seen": 2953641984
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001059621248595731,
+      "loss": 1.1843,
+      "theoretical_loss": 3.353197170150112,
+      "tokens_seen": 2953773056
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001059220028887819,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3531857913201555,
+      "tokens_seen": 2953904128
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010588188091799069,
+      "loss": 1.2393,
+      "theoretical_loss": 3.353174413136461,
+      "tokens_seen": 2954035200
+    },
+    {
+      "epoch": 0.79,
+      "objective/train/docs_used": 989845,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2652764320373535,
+      "objective/train/theoretical_loss": 3.3531630355989632,
+      "objective/train/tokens_used": 1324625376,
+      "theoretical_loss": 3.3531630355989632,
+      "tokens_seen": 2954166272
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010584175894719948,
+      "loss": 1.2846,
+      "theoretical_loss": 3.3531630355989632,
+      "tokens_seen": 2954166272
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010580163697640828,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3531516587075973,
+      "tokens_seen": 2954297344
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010576151500561708,
+      "loss": 1.247,
+      "theoretical_loss": 3.3531402824622973,
+      "tokens_seen": 2954428416
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010572139303482588,
+      "loss": 1.2795,
+      "theoretical_loss": 3.353128906862998,
+      "tokens_seen": 2954559488
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010568127106403466,
+      "loss": 1.2201,
+      "theoretical_loss": 3.3531175319096342,
+      "tokens_seen": 2954690560
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010564114909324346,
+      "loss": 1.2959,
+      "theoretical_loss": 3.3531061576021406,
+      "tokens_seen": 2954821632
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010560102712245225,
+      "loss": 1.2569,
+      "theoretical_loss": 3.3530947839404517,
+      "tokens_seen": 2954952704
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010556090515166106,
+      "loss": 1.2818,
+      "theoretical_loss": 3.3530834109245022,
+      "tokens_seen": 2955083776
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010552078318086984,
+      "loss": 1.1903,
+      "theoretical_loss": 3.3530720385542274,
+      "tokens_seen": 2955214848
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010548066121007865,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3530606668295606,
+      "tokens_seen": 2955345920
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010544053923928744,
+      "loss": 1.277,
+      "theoretical_loss": 3.353049295750438,
+      "tokens_seen": 2955476992
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010540041726849623,
+      "loss": 1.188,
+      "theoretical_loss": 3.3530379253167935,
+      "tokens_seen": 2955608064
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010536029529770502,
+      "loss": 1.1277,
+      "theoretical_loss": 3.3530265555285625,
+      "tokens_seen": 2955739136
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010532017332691382,
+      "loss": 1.2749,
+      "theoretical_loss": 3.3530151863856785,
+      "tokens_seen": 2955870208
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010528005135612262,
+      "loss": 1.2382,
+      "theoretical_loss": 3.353003817888078,
+      "tokens_seen": 2956001280
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001052399293853314,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3529924500356945,
+      "tokens_seen": 2956132352
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001051998074145402,
+      "loss": 1.1733,
+      "theoretical_loss": 3.352981082828463,
+      "tokens_seen": 2956263424
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.000105159685443749,
+      "loss": 1.2082,
+      "theoretical_loss": 3.352969716266318,
+      "tokens_seen": 2956394496
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001051195634729578,
+      "loss": 1.2069,
+      "theoretical_loss": 3.352958350349195,
+      "tokens_seen": 2956525568
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010507944150216659,
+      "loss": 1.2237,
+      "theoretical_loss": 3.352946985077029,
+      "tokens_seen": 2956656640
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010503931953137538,
+      "loss": 1.2265,
+      "theoretical_loss": 3.352935620449754,
+      "tokens_seen": 2956787712
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010499919756058419,
+      "loss": 1.1751,
+      "theoretical_loss": 3.3529242564673045,
+      "tokens_seen": 2956918784
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010495907558979298,
+      "loss": 1.3132,
+      "theoretical_loss": 3.352912893129617,
+      "tokens_seen": 2957049856
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010491895361900176,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3529015304366245,
+      "tokens_seen": 2957180928
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010487883164821056,
+      "loss": 1.2134,
+      "theoretical_loss": 3.352890168388263,
+      "tokens_seen": 2957312000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010483870967741936,
+      "loss": 1.1694,
+      "theoretical_loss": 3.352878806984467,
+      "tokens_seen": 2957443072
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010479858770662815,
+      "loss": 1.2519,
+      "theoretical_loss": 3.3528674462251713,
+      "tokens_seen": 2957574144
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010475846573583694,
+      "loss": 1.2632,
+      "theoretical_loss": 3.352856086110311,
+      "tokens_seen": 2957705216
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010471834376504575,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3528447266398205,
+      "tokens_seen": 2957836288
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010467822179425454,
+      "loss": 1.2218,
+      "theoretical_loss": 3.352833367813635,
+      "tokens_seen": 2957967360
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010463809982346332,
+      "loss": 1.266,
+      "theoretical_loss": 3.35282200963169,
+      "tokens_seen": 2958098432
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010459797785267213,
+      "loss": 1.2276,
+      "theoretical_loss": 3.3528106520939196,
+      "tokens_seen": 2958229504
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010455785588188092,
+      "loss": 1.2463,
+      "theoretical_loss": 3.352799295200259,
+      "tokens_seen": 2958360576
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010451773391108973,
+      "loss": 1.265,
+      "theoretical_loss": 3.352787938950643,
+      "tokens_seen": 2958491648
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001044776119402985,
+      "loss": 1.2186,
+      "theoretical_loss": 3.352776583345007,
+      "tokens_seen": 2958622720
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001044374899695073,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3527652283832854,
+      "tokens_seen": 2958753792
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001043973679987161,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3527538740654137,
+      "tokens_seen": 2958884864
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010435724602792488,
+      "loss": 1.2044,
+      "theoretical_loss": 3.3527425203913266,
+      "tokens_seen": 2959015936
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010431712405713369,
+      "loss": 1.2528,
+      "theoretical_loss": 3.3527311673609583,
+      "tokens_seen": 2959147008
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010427700208634248,
+      "loss": 1.2464,
+      "theoretical_loss": 3.3527198149742454,
+      "tokens_seen": 2959278080
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010423688011555129,
+      "loss": 1.2276,
+      "theoretical_loss": 3.3527084632311217,
+      "tokens_seen": 2959409152
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010419675814476007,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3526971121315223,
+      "tokens_seen": 2959540224
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010415663617396886,
+      "loss": 1.2389,
+      "theoretical_loss": 3.352685761675383,
+      "tokens_seen": 2959671296
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010411651420317767,
+      "loss": 1.2526,
+      "theoretical_loss": 3.352674411862638,
+      "tokens_seen": 2959802368
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010407639223238646,
+      "loss": 1.1421,
+      "theoretical_loss": 3.352663062693223,
+      "tokens_seen": 2959933440
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010403627026159525,
+      "loss": 1.321,
+      "theoretical_loss": 3.352651714167072,
+      "tokens_seen": 2960064512
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010399614829080405,
+      "loss": 1.2458,
+      "theoretical_loss": 3.3526403662841213,
+      "tokens_seen": 2960195584
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010395602632001284,
+      "loss": 1.1677,
+      "theoretical_loss": 3.352629019044305,
+      "tokens_seen": 2960326656
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010391590434922163,
+      "loss": 1.2012,
+      "theoretical_loss": 3.3526176724475594,
+      "tokens_seen": 2960457728
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010387578237843042,
+      "loss": 1.2314,
+      "theoretical_loss": 3.3526063264938184,
+      "tokens_seen": 2960588800
+    },
+    {
+      "debugging/Compilability": 0.08333333333333333,
+      "debugging/distinct-1-grams": 0.7149083490682447,
+      "debugging/entropy-1-grams": 5.232849655845552,
+      "debugging/length": 544.75,
+      "debugging/num_segments": 12,
+      "debugging/score": 0.05948124222167416,
+      "debugging/score_std": 0.02847918363119119,
+      "epoch": 0.79,
+      "objective/train/docs_used": 991929,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3987950086593628,
+      "objective/train/theoretical_loss": 3.3525949811830174,
+      "objective/train/tokens_used": 1331178976,
+      "theoretical_loss": 3.3525949811830174,
+      "tokens_seen": 2960719872
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010383566040763923,
+      "loss": 1.3272,
+      "theoretical_loss": 3.3525949811830174,
+      "tokens_seen": 2960719872
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010379553843684802,
+      "loss": 1.2899,
+      "theoretical_loss": 3.3525836365150914,
+      "tokens_seen": 2960850944
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010375541646605682,
+      "loss": 1.1829,
+      "theoretical_loss": 3.3525722924899757,
+      "tokens_seen": 2960982016
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010371529449526561,
+      "loss": 1.2573,
+      "theoretical_loss": 3.352560949107606,
+      "tokens_seen": 2961113088
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001036751725244744,
+      "loss": 1.1763,
+      "theoretical_loss": 3.3525496063679165,
+      "tokens_seen": 2961244160
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010363505055368321,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3525382642708426,
+      "tokens_seen": 2961375232
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010359492858289199,
+      "loss": 1.1488,
+      "theoretical_loss": 3.35252692281632,
+      "tokens_seen": 2961506304
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001035548066121008,
+      "loss": 1.2641,
+      "theoretical_loss": 3.352515582004283,
+      "tokens_seen": 2961637376
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010351468464130959,
+      "loss": 1.2219,
+      "theoretical_loss": 3.352504241834667,
+      "tokens_seen": 2961768448
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010347456267051837,
+      "loss": 1.1408,
+      "theoretical_loss": 3.352492902307408,
+      "tokens_seen": 2961899520
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010343444069972717,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3524815634224403,
+      "tokens_seen": 2962030592
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010339431872893596,
+      "loss": 1.1907,
+      "theoretical_loss": 3.352470225179699,
+      "tokens_seen": 2962161664
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010335419675814477,
+      "loss": 1.2459,
+      "theoretical_loss": 3.3524588875791204,
+      "tokens_seen": 2962292736
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010331407478735355,
+      "loss": 1.216,
+      "theoretical_loss": 3.3524475506206386,
+      "tokens_seen": 2962423808
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010327395281656236,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3524362143041895,
+      "tokens_seen": 2962554880
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010323383084577115,
+      "loss": 1.2542,
+      "theoretical_loss": 3.3524248786297077,
+      "tokens_seen": 2962685952
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010319370887497994,
+      "loss": 1.2602,
+      "theoretical_loss": 3.352413543597129,
+      "tokens_seen": 2962817024
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010315358690418873,
+      "loss": 1.1858,
+      "theoretical_loss": 3.3524022092063883,
+      "tokens_seen": 2962948096
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010311346493339753,
+      "loss": 1.2293,
+      "theoretical_loss": 3.352390875457421,
+      "tokens_seen": 2963079168
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010307334296260633,
+      "loss": 1.1635,
+      "theoretical_loss": 3.352379542350163,
+      "tokens_seen": 2963210240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010303322099181513,
+      "loss": 1.1762,
+      "theoretical_loss": 3.352368209884548,
+      "tokens_seen": 2963341312
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001029930990210239,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3523568780605135,
+      "tokens_seen": 2963472384
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010295297705023271,
+      "loss": 1.2326,
+      "theoretical_loss": 3.352345546877993,
+      "tokens_seen": 2963603456
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001029128550794415,
+      "loss": 1.249,
+      "theoretical_loss": 3.352334216336922,
+      "tokens_seen": 2963734528
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001028727331086503,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3523228864372365,
+      "tokens_seen": 2963865600
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010283261113785909,
+      "loss": 1.1932,
+      "theoretical_loss": 3.3523115571788717,
+      "tokens_seen": 2963996672
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001027924891670679,
+      "loss": 1.2023,
+      "theoretical_loss": 3.352300228561763,
+      "tokens_seen": 2964127744
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010275236719627669,
+      "loss": 1.2004,
+      "theoretical_loss": 3.352288900585845,
+      "tokens_seen": 2964258816
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010271224522548547,
+      "loss": 1.151,
+      "theoretical_loss": 3.352277573251054,
+      "tokens_seen": 2964389888
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010267212325469427,
+      "loss": 1.2346,
+      "theoretical_loss": 3.352266246557325,
+      "tokens_seen": 2964520960
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010263200128390307,
+      "loss": 1.2754,
+      "theoretical_loss": 3.3522549205045933,
+      "tokens_seen": 2964652032
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010259187931311187,
+      "loss": 1.2736,
+      "theoretical_loss": 3.3522435950927942,
+      "tokens_seen": 2964783104
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010255175734232065,
+      "loss": 1.2205,
+      "theoretical_loss": 3.3522322703218634,
+      "tokens_seen": 2964914176
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010251163537152945,
+      "loss": 1.2034,
+      "theoretical_loss": 3.352220946191736,
+      "tokens_seen": 2965045248
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010247151340073825,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3522096227023477,
+      "tokens_seen": 2965176320
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010243139142994703,
+      "loss": 1.1779,
+      "theoretical_loss": 3.352198299853634,
+      "tokens_seen": 2965307392
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010239126945915584,
+      "loss": 1.1885,
+      "theoretical_loss": 3.3521869776455295,
+      "tokens_seen": 2965438464
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010235114748836463,
+      "loss": 1.217,
+      "theoretical_loss": 3.3521756560779705,
+      "tokens_seen": 2965569536
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010231102551757344,
+      "loss": 1.1798,
+      "theoretical_loss": 3.3521643351508925,
+      "tokens_seen": 2965700608
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010227090354678222,
+      "loss": 1.2006,
+      "theoretical_loss": 3.3521530148642307,
+      "tokens_seen": 2965831680
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010223078157599101,
+      "loss": 1.289,
+      "theoretical_loss": 3.3521416952179206,
+      "tokens_seen": 2965962752
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010219065960519981,
+      "loss": 1.1786,
+      "theoretical_loss": 3.3521303762118975,
+      "tokens_seen": 2966093824
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010215053763440861,
+      "loss": 1.1311,
+      "theoretical_loss": 3.352119057846097,
+      "tokens_seen": 2966224896
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001021104156636174,
+      "loss": 1.1971,
+      "theoretical_loss": 3.3521077401204544,
+      "tokens_seen": 2966355968
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010207029369282619,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3520964230349057,
+      "tokens_seen": 2966487040
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010203017172203499,
+      "loss": 1.1982,
+      "theoretical_loss": 3.352085106589386,
+      "tokens_seen": 2966618112
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010199004975124378,
+      "loss": 1.2254,
+      "theoretical_loss": 3.352073790783831,
+      "tokens_seen": 2966749184
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010194992778045257,
+      "loss": 1.1503,
+      "theoretical_loss": 3.3520624756181765,
+      "tokens_seen": 2966880256
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010190980580966138,
+      "loss": 1.2113,
+      "theoretical_loss": 3.3520511610923576,
+      "tokens_seen": 2967011328
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010186968383887017,
+      "loss": 1.1162,
+      "theoretical_loss": 3.35203984720631,
+      "tokens_seen": 2967142400
+    },
+    {
+      "epoch": 0.8,
+      "objective/train/docs_used": 994007,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1917473077774048,
+      "objective/train/theoretical_loss": 3.3520285339599694,
+      "objective/train/tokens_used": 1337732576,
+      "theoretical_loss": 3.3520285339599694,
+      "tokens_seen": 2967273472
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010182956186807896,
+      "loss": 1.146,
+      "theoretical_loss": 3.3520285339599694,
+      "tokens_seen": 2967273472
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010178943989728776,
+      "loss": 1.1874,
+      "theoretical_loss": 3.352017221353271,
+      "tokens_seen": 2967404544
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010174931792649655,
+      "loss": 1.2484,
+      "theoretical_loss": 3.3520059093861505,
+      "tokens_seen": 2967535616
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010170919595570535,
+      "loss": 1.1898,
+      "theoretical_loss": 3.351994598058544,
+      "tokens_seen": 2967666688
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010166907398491413,
+      "loss": 1.189,
+      "theoretical_loss": 3.3519832873703868,
+      "tokens_seen": 2967797760
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010162895201412294,
+      "loss": 1.256,
+      "theoretical_loss": 3.3519719773216146,
+      "tokens_seen": 2967928832
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010158883004333173,
+      "loss": 1.2265,
+      "theoretical_loss": 3.351960667912162,
+      "tokens_seen": 2968059904
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010154870807254051,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3519493591419667,
+      "tokens_seen": 2968190976
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010150858610174932,
+      "loss": 1.2557,
+      "theoretical_loss": 3.351938051010962,
+      "tokens_seen": 2968322048
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010146846413095811,
+      "loss": 1.1808,
+      "theoretical_loss": 3.3519267435190856,
+      "tokens_seen": 2968453120
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010142834216016692,
+      "loss": 1.2165,
+      "theoretical_loss": 3.351915436666272,
+      "tokens_seen": 2968584192
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001013882201893757,
+      "loss": 1.1804,
+      "theoretical_loss": 3.351904130452457,
+      "tokens_seen": 2968715264
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001013480982185845,
+      "loss": 1.1758,
+      "theoretical_loss": 3.3518928248775763,
+      "tokens_seen": 2968846336
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001013079762477933,
+      "loss": 1.2601,
+      "theoretical_loss": 3.351881519941566,
+      "tokens_seen": 2968977408
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010126785427700209,
+      "loss": 1.1612,
+      "theoretical_loss": 3.3518702156443614,
+      "tokens_seen": 2969108480
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010122773230621088,
+      "loss": 1.1694,
+      "theoretical_loss": 3.3518589119858984,
+      "tokens_seen": 2969239552
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010118761033541967,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3518476089661124,
+      "tokens_seen": 2969370624
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010114748836462848,
+      "loss": 1.1527,
+      "theoretical_loss": 3.3518363065849393,
+      "tokens_seen": 2969501696
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010110736639383726,
+      "loss": 1.1817,
+      "theoretical_loss": 3.351825004842315,
+      "tokens_seen": 2969632768
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010106724442304605,
+      "loss": 1.2428,
+      "theoretical_loss": 3.351813703738175,
+      "tokens_seen": 2969763840
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010102712245225486,
+      "loss": 1.1894,
+      "theoretical_loss": 3.351802403272455,
+      "tokens_seen": 2969894912
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010098700048146365,
+      "loss": 1.1861,
+      "theoretical_loss": 3.351791103445091,
+      "tokens_seen": 2970025984
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010094687851067244,
+      "loss": 1.2024,
+      "theoretical_loss": 3.351779804256019,
+      "tokens_seen": 2970157056
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010090675653988124,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3517685057051745,
+      "tokens_seen": 2970288128
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010086663456909004,
+      "loss": 1.1361,
+      "theoretical_loss": 3.351757207792493,
+      "tokens_seen": 2970419200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010082651259829884,
+      "loss": 1.1479,
+      "theoretical_loss": 3.3517459105179106,
+      "tokens_seen": 2970550272
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010078639062750762,
+      "loss": 1.1534,
+      "theoretical_loss": 3.351734613881363,
+      "tokens_seen": 2970681344
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010074626865671642,
+      "loss": 1.1475,
+      "theoretical_loss": 3.351723317882786,
+      "tokens_seen": 2970812416
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010070614668592521,
+      "loss": 1.1848,
+      "theoretical_loss": 3.3517120225221158,
+      "tokens_seen": 2970943488
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010066602471513402,
+      "loss": 1.2982,
+      "theoretical_loss": 3.3517007277992876,
+      "tokens_seen": 2971074560
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001006259027443428,
+      "loss": 1.1877,
+      "theoretical_loss": 3.3516894337142378,
+      "tokens_seen": 2971205632
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001005857807735516,
+      "loss": 1.2371,
+      "theoretical_loss": 3.3516781402669023,
+      "tokens_seen": 2971336704
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001005456588027604,
+      "loss": 1.2307,
+      "theoretical_loss": 3.351666847457216,
+      "tokens_seen": 2971467776
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010050553683196918,
+      "loss": 1.1976,
+      "theoretical_loss": 3.351655555285116,
+      "tokens_seen": 2971598848
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010046541486117798,
+      "loss": 1.1908,
+      "theoretical_loss": 3.3516442637505377,
+      "tokens_seen": 2971729920
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010042529289038678,
+      "loss": 1.2419,
+      "theoretical_loss": 3.351632972853417,
+      "tokens_seen": 2971860992
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010038517091959558,
+      "loss": 1.2663,
+      "theoretical_loss": 3.3516216825936893,
+      "tokens_seen": 2971992064
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010034504894880436,
+      "loss": 1.1669,
+      "theoretical_loss": 3.351610392971291,
+      "tokens_seen": 2972123136
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010030492697801316,
+      "loss": 1.1572,
+      "theoretical_loss": 3.3515991039861586,
+      "tokens_seen": 2972254208
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010026480500722196,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3515878156382266,
+      "tokens_seen": 2972385280
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010022468303643075,
+      "loss": 1.1266,
+      "theoretical_loss": 3.351576527927432,
+      "tokens_seen": 2972516352
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010018456106563955,
+      "loss": 1.2121,
+      "theoretical_loss": 3.351565240853711,
+      "tokens_seen": 2972647424
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010014443909484834,
+      "loss": 1.1828,
+      "theoretical_loss": 3.3515539544169988,
+      "tokens_seen": 2972778496
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010010431712405715,
+      "loss": 1.2308,
+      "theoretical_loss": 3.351542668617231,
+      "tokens_seen": 2972909568
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010006419515326593,
+      "loss": 1.2101,
+      "theoretical_loss": 3.351531383454345,
+      "tokens_seen": 2973040640
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010002407318247472,
+      "loss": 1.1616,
+      "theoretical_loss": 3.3515200989282756,
+      "tokens_seen": 2973171712
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.998395121168352e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.3515088150389594,
+      "tokens_seen": 2973302784
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.994382924089232e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3514975317863316,
+      "tokens_seen": 2973433856
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.990370727010111e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3514862491703292,
+      "tokens_seen": 2973564928
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.98635852993099e-05,
+      "loss": 1.1539,
+      "theoretical_loss": 3.351474967190888,
+      "tokens_seen": 2973696000
+    },
+    {
+      "epoch": 0.8,
+      "objective/train/docs_used": 996512,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.166604995727539,
+      "objective/train/theoretical_loss": 3.3514636858479436,
+      "objective/train/tokens_used": 1344286176,
+      "theoretical_loss": 3.3514636858479436,
+      "tokens_seen": 2973827072
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.98234633285187e-05,
+      "loss": 1.1642,
+      "theoretical_loss": 3.3514636858479436,
+      "tokens_seen": 2973827072
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.97833413577275e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.351452405141432,
+      "tokens_seen": 2973958144
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.974321938693628e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.3514411250712897,
+      "tokens_seen": 2974089216
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.970309741614509e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.351429845637453,
+      "tokens_seen": 2974220288
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.966297544535388e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.351418566839857,
+      "tokens_seen": 2974351360
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.962285347456267e-05,
+      "loss": 1.1948,
+      "theoretical_loss": 3.3514072886784385,
+      "tokens_seen": 2974482432
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.958273150377147e-05,
+      "loss": 1.1196,
+      "theoretical_loss": 3.3513960111531333,
+      "tokens_seen": 2974613504
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.954260953298026e-05,
+      "loss": 1.2105,
+      "theoretical_loss": 3.3513847342638776,
+      "tokens_seen": 2974744576
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.950248756218906e-05,
+      "loss": 1.1576,
+      "theoretical_loss": 3.351373458010608,
+      "tokens_seen": 2974875648
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.946236559139784e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.3513621823932596,
+      "tokens_seen": 2975006720
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.942224362060665e-05,
+      "loss": 1.1721,
+      "theoretical_loss": 3.351350907411769,
+      "tokens_seen": 2975137792
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.938212164981544e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3513396330660727,
+      "tokens_seen": 2975268864
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.934199967902424e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.3513283593561063,
+      "tokens_seen": 2975399936
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.930187770823303e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.351317086281806,
+      "tokens_seen": 2975531008
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.926175573744182e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.3513058138431084,
+      "tokens_seen": 2975662080
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.922163376665063e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.351294542039949,
+      "tokens_seen": 2975793152
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.918151179585941e-05,
+      "loss": 1.1594,
+      "theoretical_loss": 3.351283270872265,
+      "tokens_seen": 2975924224
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.914138982506821e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3512720003399914,
+      "tokens_seen": 2976055296
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.9101267854277e-05,
+      "loss": 1.1792,
+      "theoretical_loss": 3.3512607304430655,
+      "tokens_seen": 2976186368
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.90611458834858e-05,
+      "loss": 1.1565,
+      "theoretical_loss": 3.3512494611814225,
+      "tokens_seen": 2976317440
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.902102391269459e-05,
+      "loss": 1.1672,
+      "theoretical_loss": 3.351238192554999,
+      "tokens_seen": 2976448512
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.898090194190338e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.3512269245637314,
+      "tokens_seen": 2976579584
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.894077997111219e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3512156572075558,
+      "tokens_seen": 2976710656
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.890065800032098e-05,
+      "loss": 1.1236,
+      "theoretical_loss": 3.3512043904864086,
+      "tokens_seen": 2976841728
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.886053602952976e-05,
+      "loss": 1.181,
+      "theoretical_loss": 3.3511931244002264,
+      "tokens_seen": 2976972800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.882041405873857e-05,
+      "loss": 1.1121,
+      "theoretical_loss": 3.3511818589489444,
+      "tokens_seen": 2977103872
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.878029208794736e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.3511705941324994,
+      "tokens_seen": 2977234944
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.874017011715615e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3511593299508275,
+      "tokens_seen": 2977366016
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.870004814636495e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.3511480664038658,
+      "tokens_seen": 2977497088
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.865992617557375e-05,
+      "loss": 1.1711,
+      "theoretical_loss": 3.3511368034915496,
+      "tokens_seen": 2977628160
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.861980420478255e-05,
+      "loss": 1.204,
+      "theoretical_loss": 3.351125541213816,
+      "tokens_seen": 2977759232
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.857968223399133e-05,
+      "loss": 1.1832,
+      "theoretical_loss": 3.351114279570601,
+      "tokens_seen": 2977890304
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.853956026320013e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.3511030185618402,
+      "tokens_seen": 2978021376
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.849943829240892e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.3510917581874713,
+      "tokens_seen": 2978152448
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.845931632161773e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3510804984474296,
+      "tokens_seen": 2978283520
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.841919435082651e-05,
+      "loss": 1.1839,
+      "theoretical_loss": 3.3510692393416517,
+      "tokens_seen": 2978414592
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.83790723800353e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3510579808700744,
+      "tokens_seen": 2978545664
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.833895040924411e-05,
+      "loss": 1.1531,
+      "theoretical_loss": 3.3510467230326335,
+      "tokens_seen": 2978676736
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.82988284384529e-05,
+      "loss": 1.177,
+      "theoretical_loss": 3.3510354658292654,
+      "tokens_seen": 2978807808
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.82587064676617e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.351024209259907,
+      "tokens_seen": 2978938880
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.821858449687049e-05,
+      "loss": 1.1522,
+      "theoretical_loss": 3.3510129533244943,
+      "tokens_seen": 2979069952
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.81784625260793e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.351001698022964,
+      "tokens_seen": 2979201024
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.813834055528807e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.350990443355252,
+      "tokens_seen": 2979332096
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.809821858449687e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3509791893212952,
+      "tokens_seen": 2979463168
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.805809661370567e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.35096793592103,
+      "tokens_seen": 2979594240
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.801797464291446e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.3509566831543927,
+      "tokens_seen": 2979725312
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.797785267212326e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3509454310213194,
+      "tokens_seen": 2979856384
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.793773070133205e-05,
+      "loss": 1.1884,
+      "theoretical_loss": 3.3509341795217473,
+      "tokens_seen": 2979987456
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.789760873054084e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.3509229286556126,
+      "tokens_seen": 2980118528
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.785748675974965e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.350911678422851,
+      "tokens_seen": 2980249600
+    },
+    {
+      "epoch": 0.81,
+      "objective/train/docs_used": 998650,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2795372009277344,
+      "objective/train/theoretical_loss": 3.3509004288234006,
+      "objective/train/tokens_used": 1350839776,
+      "theoretical_loss": 3.3509004288234006,
+      "tokens_seen": 2980380672
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.781736478895843e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3509004288234006,
+      "tokens_seen": 2980380672
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.777724281816723e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3508891798571967,
+      "tokens_seen": 2980511744
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.773712084737603e-05,
+      "loss": 1.111,
+      "theoretical_loss": 3.350877931524176,
+      "tokens_seen": 2980642816
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.769699887658482e-05,
+      "loss": 1.243,
+      "theoretical_loss": 3.3508666838242753,
+      "tokens_seen": 2980773888
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.765687690579361e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.3508554367574304,
+      "tokens_seen": 2980904960
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.76167549350024e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.3508441903235786,
+      "tokens_seen": 2981036032
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.757663296421121e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3508329445226566,
+      "tokens_seen": 2981167104
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.753651099341999e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.3508216993546003,
+      "tokens_seen": 2981298176
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.74963890226288e-05,
+      "loss": 1.3371,
+      "theoretical_loss": 3.3508104548193467,
+      "tokens_seen": 2981429248
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.745626705183759e-05,
+      "loss": 1.148,
+      "theoretical_loss": 3.350799210916832,
+      "tokens_seen": 2981560320
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.741614508104638e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.350787967646993,
+      "tokens_seen": 2981691392
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.737602311025518e-05,
+      "loss": 1.1766,
+      "theoretical_loss": 3.350776725009766,
+      "tokens_seen": 2981822464
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.733590113946397e-05,
+      "loss": 1.1589,
+      "theoretical_loss": 3.3507654830050884,
+      "tokens_seen": 2981953536
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.729577916867277e-05,
+      "loss": 1.2049,
+      "theoretical_loss": 3.350754241632896,
+      "tokens_seen": 2982084608
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.725565719788155e-05,
+      "loss": 1.1465,
+      "theoretical_loss": 3.350743000893126,
+      "tokens_seen": 2982215680
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.721553522709036e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.3507317607857146,
+      "tokens_seen": 2982346752
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.717541325629915e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3507205213105986,
+      "tokens_seen": 2982477824
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.713529128550795e-05,
+      "loss": 1.1717,
+      "theoretical_loss": 3.3507092824677143,
+      "tokens_seen": 2982608896
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.709516931471674e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.3506980442569994,
+      "tokens_seen": 2982739968
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.705504734392553e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.350686806678389,
+      "tokens_seen": 2982871040
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.701492537313434e-05,
+      "loss": 1.1499,
+      "theoretical_loss": 3.350675569731821,
+      "tokens_seen": 2983002112
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.697480340234313e-05,
+      "loss": 1.151,
+      "theoretical_loss": 3.3506643334172317,
+      "tokens_seen": 2983133184
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.693468143155191e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.3506530977345577,
+      "tokens_seen": 2983264256
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.689455946076072e-05,
+      "loss": 1.1286,
+      "theoretical_loss": 3.350641862683736,
+      "tokens_seen": 2983395328
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.685443748996951e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.350630628264703,
+      "tokens_seen": 2983526400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.68143155191783e-05,
+      "loss": 1.1408,
+      "theoretical_loss": 3.3506193944773957,
+      "tokens_seen": 2983657472
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.67741935483871e-05,
+      "loss": 1.1422,
+      "theoretical_loss": 3.35060816132175,
+      "tokens_seen": 2983788544
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.67340715775959e-05,
+      "loss": 1.2781,
+      "theoretical_loss": 3.350596928797704,
+      "tokens_seen": 2983919616
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.669394960680469e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3505856969051933,
+      "tokens_seen": 2984050688
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.665382763601347e-05,
+      "loss": 1.145,
+      "theoretical_loss": 3.350574465644155,
+      "tokens_seen": 2984181760
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.661370566522228e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.3505632350145262,
+      "tokens_seen": 2984312832
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.657358369443107e-05,
+      "loss": 1.1673,
+      "theoretical_loss": 3.3505520050162434,
+      "tokens_seen": 2984443904
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.653346172363988e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.3505407756492436,
+      "tokens_seen": 2984574976
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.649333975284866e-05,
+      "loss": 1.2124,
+      "theoretical_loss": 3.350529546913463,
+      "tokens_seen": 2984706048
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.645321778205746e-05,
+      "loss": 1.1961,
+      "theoretical_loss": 3.350518318808839,
+      "tokens_seen": 2984837120
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.641309581126626e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.350507091335308,
+      "tokens_seen": 2984968192
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.637297384047504e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3504958644928076,
+      "tokens_seen": 2985099264
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.633285186968384e-05,
+      "loss": 1.2672,
+      "theoretical_loss": 3.3504846382812734,
+      "tokens_seen": 2985230336
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.629272989889263e-05,
+      "loss": 1.166,
+      "theoretical_loss": 3.350473412700643,
+      "tokens_seen": 2985361408
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.625260792810144e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.3504621877508534,
+      "tokens_seen": 2985492480
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.621248595731022e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3504509634318413,
+      "tokens_seen": 2985623552
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.617236398651901e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.350439739743543,
+      "tokens_seen": 2985754624
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.613224201572782e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.3504285166858963,
+      "tokens_seen": 2985885696
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.609212004493661e-05,
+      "loss": 1.1691,
+      "theoretical_loss": 3.350417294258837,
+      "tokens_seen": 2986016768
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.60519980741454e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3504060724623033,
+      "tokens_seen": 2986147840
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.60118761033542e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.350394851296231,
+      "tokens_seen": 2986278912
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.5971754132563e-05,
+      "loss": 1.1671,
+      "theoretical_loss": 3.3503836307605575,
+      "tokens_seen": 2986409984
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.593163216177178e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.3503724108552193,
+      "tokens_seen": 2986541056
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.589151019098058e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.3503611915801543,
+      "tokens_seen": 2986672128
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.585138822018938e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.3503499729352986,
+      "tokens_seen": 2986803200
+    },
+    {
+      "epoch": 0.81,
+      "objective/train/docs_used": 1000654,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2829012870788574,
+      "objective/train/theoretical_loss": 3.350338754920589,
+      "objective/train/tokens_used": 1357393376,
+      "theoretical_loss": 3.350338754920589,
+      "tokens_seen": 2986934272
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.581126624939817e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.350338754920589,
+      "tokens_seen": 2986934272
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.577114427860697e-05,
+      "loss": 1.1332,
+      "theoretical_loss": 3.3503275375359634,
+      "tokens_seen": 2987065344
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.573102230781576e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3503163207813573,
+      "tokens_seen": 2987196416
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.569090033702455e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.350305104656709,
+      "tokens_seen": 2987327488
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.565077836623336e-05,
+      "loss": 1.1401,
+      "theoretical_loss": 3.3502938891619554,
+      "tokens_seen": 2987458560
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.561065639544214e-05,
+      "loss": 1.118,
+      "theoretical_loss": 3.3502826742970324,
+      "tokens_seen": 2987589632
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.557053442465094e-05,
+      "loss": 1.2113,
+      "theoretical_loss": 3.3502714600618786,
+      "tokens_seen": 2987720704
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.553041245385974e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3502602464564295,
+      "tokens_seen": 2987851776
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.549029048306854e-05,
+      "loss": 1.1647,
+      "theoretical_loss": 3.3502490334806225,
+      "tokens_seen": 2987982848
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.545016851227732e-05,
+      "loss": 1.1904,
+      "theoretical_loss": 3.3502378211343955,
+      "tokens_seen": 2988113920
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.541004654148612e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.3502266094176845,
+      "tokens_seen": 2988244992
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.536992457069492e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.3502153983304273,
+      "tokens_seen": 2988376064
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.53298025999037e-05,
+      "loss": 1.1395,
+      "theoretical_loss": 3.3502041878725604,
+      "tokens_seen": 2988507136
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.528968062911251e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.3501929780440207,
+      "tokens_seen": 2988638208
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.52495586583213e-05,
+      "loss": 1.1936,
+      "theoretical_loss": 3.350181768844746,
+      "tokens_seen": 2988769280
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.520943668753009e-05,
+      "loss": 1.1124,
+      "theoretical_loss": 3.350170560274673,
+      "tokens_seen": 2988900352
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.516931471673889e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.350159352333739,
+      "tokens_seen": 2989031424
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.512919274594768e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.3501481450218806,
+      "tokens_seen": 2989162496
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.508907077515648e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.350136938339036,
+      "tokens_seen": 2989293568
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.504894880436528e-05,
+      "loss": 1.1836,
+      "theoretical_loss": 3.3501257322851403,
+      "tokens_seen": 2989424640
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.500882683357407e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.350114526860133,
+      "tokens_seen": 2989555712
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.496870486278286e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.3501033220639496,
+      "tokens_seen": 2989686784
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.492858289199166e-05,
+      "loss": 1.1285,
+      "theoretical_loss": 3.3500921178965277,
+      "tokens_seen": 2989817856
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.488846092120045e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.3500809143578048,
+      "tokens_seen": 2989948928
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.484833895040924e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.350069711447717,
+      "tokens_seen": 2990080000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.480821697961805e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.350058509166203,
+      "tokens_seen": 2990211072
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.476809500882684e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.350047307513199,
+      "tokens_seen": 2990342144
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.472797303803562e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.3500361064886426,
+      "tokens_seen": 2990473216
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.468785106724443e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.3500249060924707,
+      "tokens_seen": 2990604288
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.464772909645322e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.3500137063246207,
+      "tokens_seen": 2990735360
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.460760712566202e-05,
+      "loss": 1.2097,
+      "theoretical_loss": 3.3500025071850295,
+      "tokens_seen": 2990866432
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.45674851548708e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.349991308673635,
+      "tokens_seen": 2990997504
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.452736318407961e-05,
+      "loss": 1.184,
+      "theoretical_loss": 3.3499801107903737,
+      "tokens_seen": 2991128576
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.44872412132884e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.349968913535183,
+      "tokens_seen": 2991259648
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.444711924249718e-05,
+      "loss": 1.1573,
+      "theoretical_loss": 3.3499577169080004,
+      "tokens_seen": 2991390720
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.440699727170599e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.3499465209087633,
+      "tokens_seen": 2991521792
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.436687530091478e-05,
+      "loss": 1.1907,
+      "theoretical_loss": 3.3499353255374085,
+      "tokens_seen": 2991652864
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.432675333012359e-05,
+      "loss": 1.1684,
+      "theoretical_loss": 3.3499241307938736,
+      "tokens_seen": 2991783936
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.428663135933237e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3499129366780958,
+      "tokens_seen": 2991915008
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.424650938854116e-05,
+      "loss": 1.1736,
+      "theoretical_loss": 3.3499017431900127,
+      "tokens_seen": 2992046080
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.420638741774997e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.349890550329561,
+      "tokens_seen": 2992177152
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.416626544695876e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.349879358096678,
+      "tokens_seen": 2992308224
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.412614347616755e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3498681664913024,
+      "tokens_seen": 2992439296
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.408602150537634e-05,
+      "loss": 1.1575,
+      "theoretical_loss": 3.3498569755133696,
+      "tokens_seen": 2992570368
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.404589953458515e-05,
+      "loss": 1.2136,
+      "theoretical_loss": 3.349845785162818,
+      "tokens_seen": 2992701440
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.400577756379393e-05,
+      "loss": 1.1891,
+      "theoretical_loss": 3.349834595439585,
+      "tokens_seen": 2992832512
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.396565559300272e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3498234063436074,
+      "tokens_seen": 2992963584
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.392553362221153e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.3498122178748235,
+      "tokens_seen": 2993094656
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.388541165142032e-05,
+      "loss": 1.1667,
+      "theoretical_loss": 3.3498010300331695,
+      "tokens_seen": 2993225728
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.384528968062911e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.349789842818584,
+      "tokens_seen": 2993356800
+    },
+    {
+      "epoch": 0.81,
+      "objective/train/docs_used": 1002769,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.0934433937072754,
+      "objective/train/theoretical_loss": 3.3497786562310035,
+      "objective/train/tokens_used": 1363946976,
+      "theoretical_loss": 3.3497786562310035,
+      "tokens_seen": 2993487872
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.380516770983791e-05,
+      "loss": 1.1865,
+      "theoretical_loss": 3.3497786562310035,
+      "tokens_seen": 2993487872
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.37650457390467e-05,
+      "loss": 1.198,
+      "theoretical_loss": 3.3497674702703657,
+      "tokens_seen": 2993618944
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.37249237682555e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.349756284936608,
+      "tokens_seen": 2993750016
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.368480179746429e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.3497451002296676,
+      "tokens_seen": 2993881088
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.364467982667309e-05,
+      "loss": 1.2128,
+      "theoretical_loss": 3.3497339161494826,
+      "tokens_seen": 2994012160
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.360455785588188e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.3497227326959895,
+      "tokens_seen": 2994143232
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.356443588509068e-05,
+      "loss": 1.2117,
+      "theoretical_loss": 3.349711549869127,
+      "tokens_seen": 2994274304
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.352431391429947e-05,
+      "loss": 1.1594,
+      "theoretical_loss": 3.3497003676688313,
+      "tokens_seen": 2994405376
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.348419194350826e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.3496891860950404,
+      "tokens_seen": 2994536448
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.344406997271707e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.349678005147692,
+      "tokens_seen": 2994667520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.340394800192585e-05,
+      "loss": 1.1645,
+      "theoretical_loss": 3.3496668248267234,
+      "tokens_seen": 2994798592
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.336382603113465e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.3496556451320716,
+      "tokens_seen": 2994929664
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.332370406034345e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3496444660636753,
+      "tokens_seen": 2995060736
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.328358208955224e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.3496332876214705,
+      "tokens_seen": 2995191808
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.324346011876103e-05,
+      "loss": 1.2905,
+      "theoretical_loss": 3.349622109805396,
+      "tokens_seen": 2995322880
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.320333814796983e-05,
+      "loss": 1.1809,
+      "theoretical_loss": 3.3496109326153887,
+      "tokens_seen": 2995453952
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.316321617717863e-05,
+      "loss": 1.1891,
+      "theoretical_loss": 3.3495997560513864,
+      "tokens_seen": 2995585024
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.312309420638742e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.3495885801133265,
+      "tokens_seen": 2995716096
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.308297223559622e-05,
+      "loss": 1.1759,
+      "theoretical_loss": 3.3495774048011464,
+      "tokens_seen": 2995847168
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.304285026480501e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3495662301147844,
+      "tokens_seen": 2995978240
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.30027282940138e-05,
+      "loss": 1.2362,
+      "theoretical_loss": 3.349555056054177,
+      "tokens_seen": 2996109312
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.29626063232226e-05,
+      "loss": 1.1797,
+      "theoretical_loss": 3.3495438826192623,
+      "tokens_seen": 2996240384
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.292248435243139e-05,
+      "loss": 1.1765,
+      "theoretical_loss": 3.3495327098099783,
+      "tokens_seen": 2996371456
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.28823623816402e-05,
+      "loss": 1.1958,
+      "theoretical_loss": 3.349521537626262,
+      "tokens_seen": 2996502528
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.284224041084899e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.349510366068051,
+      "tokens_seen": 2996633600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.280211844005777e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.3494991951352837,
+      "tokens_seen": 2996764672
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.276199646926657e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.3494880248278966,
+      "tokens_seen": 2996895744
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.272187449847537e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.3494768551458285,
+      "tokens_seen": 2997026816
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.268175252768417e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.349465686089016,
+      "tokens_seen": 2997157888
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.264163055689295e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.3494545176573975,
+      "tokens_seen": 2997288960
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.260150858610176e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3494433498509104,
+      "tokens_seen": 2997420032
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.256138661531055e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.349432182669492,
+      "tokens_seen": 2997551104
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.252126464451933e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.349421016113081,
+      "tokens_seen": 2997682176
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.248114267372814e-05,
+      "loss": 1.2101,
+      "theoretical_loss": 3.349409850181614,
+      "tokens_seen": 2997813248
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.244102070293693e-05,
+      "loss": 1.1842,
+      "theoretical_loss": 3.3493986848750295,
+      "tokens_seen": 2997944320
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.240089873214573e-05,
+      "loss": 1.1601,
+      "theoretical_loss": 3.3493875201932646,
+      "tokens_seen": 2998075392
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.236077676135451e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.3493763561362573,
+      "tokens_seen": 2998206464
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.23206547905633e-05,
+      "loss": 1.2362,
+      "theoretical_loss": 3.3493651927039454,
+      "tokens_seen": 2998337536
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.228053281977211e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3493540298962667,
+      "tokens_seen": 2998468608
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.22404108489809e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.3493428677131587,
+      "tokens_seen": 2998599680
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.22002888781897e-05,
+      "loss": 1.1505,
+      "theoretical_loss": 3.349331706154559,
+      "tokens_seen": 2998730752
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.216016690739849e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3493205452204062,
+      "tokens_seen": 2998861824
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.21200449366073e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.349309384910637,
+      "tokens_seen": 2998992896
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.207992296581608e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.3492982252251897,
+      "tokens_seen": 2999123968
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.203980099502487e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.3492870661640026,
+      "tokens_seen": 2999255040
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.199967902423368e-05,
+      "loss": 1.1788,
+      "theoretical_loss": 3.3492759077270127,
+      "tokens_seen": 2999386112
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.195955705344247e-05,
+      "loss": 1.1395,
+      "theoretical_loss": 3.3492647499141577,
+      "tokens_seen": 2999517184
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.191943508265126e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.3492535927253764,
+      "tokens_seen": 2999648256
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.187931311186005e-05,
+      "loss": 1.1652,
+      "theoretical_loss": 3.3492424361606057,
+      "tokens_seen": 2999779328
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.183919114106886e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.349231280219784,
+      "tokens_seen": 2999910400
+    },
+    {
+      "epoch": 0.82,
+      "objective/train/docs_used": 1004751,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2147324085235596,
+      "objective/train/theoretical_loss": 3.3492201249028484,
+      "objective/train/tokens_used": 1370500576,
+      "theoretical_loss": 3.3492201249028484,
+      "tokens_seen": 3000041472
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.179906917027765e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.3492201249028484,
+      "tokens_seen": 3000041472
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.175894719948643e-05,
+      "loss": 1.1351,
+      "theoretical_loss": 3.3492089702097374,
+      "tokens_seen": 3000172544
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.171882522869524e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.3491978161403892,
+      "tokens_seen": 3000303616
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.167870325790403e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.3491866626947413,
+      "tokens_seen": 3000434688
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.163858128711282e-05,
+      "loss": 1.1836,
+      "theoretical_loss": 3.349175509872731,
+      "tokens_seen": 3000565760
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.159845931632162e-05,
+      "loss": 1.1848,
+      "theoretical_loss": 3.3491643576742964,
+      "tokens_seen": 3000696832
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.155833734553041e-05,
+      "loss": 1.1401,
+      "theoretical_loss": 3.3491532060993765,
+      "tokens_seen": 3000827904
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.151821537473922e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.349142055147908,
+      "tokens_seen": 3000958976
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.1478093403948e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.349130904819829,
+      "tokens_seen": 3001090048
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.14379714331568e-05,
+      "loss": 1.1922,
+      "theoretical_loss": 3.349119755115078,
+      "tokens_seen": 3001221120
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.13978494623656e-05,
+      "loss": 1.1697,
+      "theoretical_loss": 3.3491086060335924,
+      "tokens_seen": 3001352192
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.13577274915744e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.34909745757531,
+      "tokens_seen": 3001483264
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.131760552078318e-05,
+      "loss": 1.1763,
+      "theoretical_loss": 3.3490863097401693,
+      "tokens_seen": 3001614336
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.127748354999197e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.3490751625281083,
+      "tokens_seen": 3001745408
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.123736157920078e-05,
+      "loss": 1.1344,
+      "theoretical_loss": 3.3490640159390646,
+      "tokens_seen": 3001876480
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.119723960840956e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.349052869972976,
+      "tokens_seen": 3002007552
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.115711763761836e-05,
+      "loss": 1.15,
+      "theoretical_loss": 3.349041724629781,
+      "tokens_seen": 3002138624
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.111699566682716e-05,
+      "loss": 1.1674,
+      "theoretical_loss": 3.3490305799094173,
+      "tokens_seen": 3002269696
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.107687369603595e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.3490194358118224,
+      "tokens_seen": 3002400768
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.103675172524474e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3490082923369355,
+      "tokens_seen": 3002531840
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.099662975445354e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.3489971494846937,
+      "tokens_seen": 3002662912
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.095650778366234e-05,
+      "loss": 1.1756,
+      "theoretical_loss": 3.3489860072550357,
+      "tokens_seen": 3002793984
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.091638581287113e-05,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3489748656478993,
+      "tokens_seen": 3002925056
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.087626384207993e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.348963724663222,
+      "tokens_seen": 3003056128
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.083614187128872e-05,
+      "loss": 1.1905,
+      "theoretical_loss": 3.348952584300942,
+      "tokens_seen": 3003187200
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.079601990049751e-05,
+      "loss": 1.1614,
+      "theoretical_loss": 3.3489414445609977,
+      "tokens_seen": 3003318272
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.075589792970632e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.3489303054433277,
+      "tokens_seen": 3003449344
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.07157759589151e-05,
+      "loss": 1.1769,
+      "theoretical_loss": 3.3489191669478693,
+      "tokens_seen": 3003580416
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.06756539881239e-05,
+      "loss": 1.2464,
+      "theoretical_loss": 3.3489080290745608,
+      "tokens_seen": 3003711488
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.06355320173327e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.34889689182334,
+      "tokens_seen": 3003842560
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.059541004654148e-05,
+      "loss": 1.1728,
+      "theoretical_loss": 3.3488857551941456,
+      "tokens_seen": 3003973632
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.055528807575028e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.3488746191869154,
+      "tokens_seen": 3004104704
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.051516610495908e-05,
+      "loss": 1.2362,
+      "theoretical_loss": 3.3488634838015874,
+      "tokens_seen": 3004235776
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.047504413416788e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3488523490381,
+      "tokens_seen": 3004366848
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.043492216337666e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3488412148963915,
+      "tokens_seen": 3004497920
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.039480019258547e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3488300813763994,
+      "tokens_seen": 3004628992
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.035467822179426e-05,
+      "loss": 1.2134,
+      "theoretical_loss": 3.3488189484780624,
+      "tokens_seen": 3004760064
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.031455625100305e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.348807816201319,
+      "tokens_seen": 3004891136
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.027443428021185e-05,
+      "loss": 1.1948,
+      "theoretical_loss": 3.348796684546106,
+      "tokens_seen": 3005022208
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.023431230942064e-05,
+      "loss": 1.1796,
+      "theoretical_loss": 3.3487855535123634,
+      "tokens_seen": 3005153280
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.019419033862944e-05,
+      "loss": 1.1525,
+      "theoretical_loss": 3.348774423100028,
+      "tokens_seen": 3005284352
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.015406836783822e-05,
+      "loss": 1.3029,
+      "theoretical_loss": 3.3487632933090383,
+      "tokens_seen": 3005415424
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.011394639704702e-05,
+      "loss": 1.1554,
+      "theoretical_loss": 3.3487521641393334,
+      "tokens_seen": 3005546496
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.007382442625582e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3487410355908507,
+      "tokens_seen": 3005677568
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.003370245546462e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3487299076635284,
+      "tokens_seen": 3005808640
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.999358048467341e-05,
+      "loss": 1.1676,
+      "theoretical_loss": 3.348718780357305,
+      "tokens_seen": 3005939712
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.99534585138822e-05,
+      "loss": 1.1687,
+      "theoretical_loss": 3.3487076536721188,
+      "tokens_seen": 3006070784
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.991333654309101e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.348696527607908,
+      "tokens_seen": 3006201856
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.98732145722998e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.3486854021646106,
+      "tokens_seen": 3006332928
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.983309260150858e-05,
+      "loss": 1.2955,
+      "theoretical_loss": 3.3486742773421656,
+      "tokens_seen": 3006464000
+    },
+    {
+      "epoch": 0.82,
+      "objective/train/docs_used": 1007026,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1748117208480835,
+      "objective/train/theoretical_loss": 3.34866315314051,
+      "objective/train/tokens_used": 1377054176,
+      "theoretical_loss": 3.34866315314051,
+      "tokens_seen": 3006595072
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.979297063071739e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.34866315314051,
+      "tokens_seen": 3006595072
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.975284865992618e-05,
+      "loss": 1.1804,
+      "theoretical_loss": 3.3486520295595836,
+      "tokens_seen": 3006726144
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.971272668913497e-05,
+      "loss": 1.1863,
+      "theoretical_loss": 3.348640906599324,
+      "tokens_seen": 3006857216
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.967260471834376e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.3486297842596695,
+      "tokens_seen": 3006988288
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.963248274755256e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.3486186625405585,
+      "tokens_seen": 3007119360
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.959236077676136e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.348607541441929,
+      "tokens_seen": 3007250432
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.955223880597014e-05,
+      "loss": 1.1293,
+      "theoretical_loss": 3.3485964209637196,
+      "tokens_seen": 3007381504
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.951211683517895e-05,
+      "loss": 1.2314,
+      "theoretical_loss": 3.348585301105869,
+      "tokens_seen": 3007512576
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.947199486438774e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.348574181868315,
+      "tokens_seen": 3007643648
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.943187289359655e-05,
+      "loss": 1.2252,
+      "theoretical_loss": 3.348563063250996,
+      "tokens_seen": 3007774720
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.939175092280533e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.348551945253851,
+      "tokens_seen": 3007905792
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.935162895201412e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3485408278768176,
+      "tokens_seen": 3008036864
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.931150698122293e-05,
+      "loss": 1.1781,
+      "theoretical_loss": 3.348529711119835,
+      "tokens_seen": 3008167936
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.92713850104317e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.348518594982841,
+      "tokens_seen": 3008299008
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.923126303964051e-05,
+      "loss": 1.148,
+      "theoretical_loss": 3.348507479465774,
+      "tokens_seen": 3008430080
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.91911410688493e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.3484963645685726,
+      "tokens_seen": 3008561152
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.91510190980581e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.3484852502911755,
+      "tokens_seen": 3008692224
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.911089712726689e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.3484741366335204,
+      "tokens_seen": 3008823296
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.907077515647568e-05,
+      "loss": 1.1639,
+      "theoretical_loss": 3.3484630235955466,
+      "tokens_seen": 3008954368
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.903065318568449e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.348451911177192,
+      "tokens_seen": 3009085440
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.899053121489328e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.3484407993783956,
+      "tokens_seen": 3009216512
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.895040924410207e-05,
+      "loss": 1.161,
+      "theoretical_loss": 3.3484296881990954,
+      "tokens_seen": 3009347584
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.891028727331087e-05,
+      "loss": 1.146,
+      "theoretical_loss": 3.3484185776392295,
+      "tokens_seen": 3009478656
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.887016530251966e-05,
+      "loss": 1.1788,
+      "theoretical_loss": 3.3484074676987374,
+      "tokens_seen": 3009609728
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.883004333172845e-05,
+      "loss": 1.1782,
+      "theoretical_loss": 3.3483963583775567,
+      "tokens_seen": 3009740800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.878992136093724e-05,
+      "loss": 1.1707,
+      "theoretical_loss": 3.3483852496756263,
+      "tokens_seen": 3009871872
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.874979939014605e-05,
+      "loss": 1.1891,
+      "theoretical_loss": 3.3483741415928847,
+      "tokens_seen": 3010002944
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.870967741935484e-05,
+      "loss": 1.1938,
+      "theoretical_loss": 3.348363034129271,
+      "tokens_seen": 3010134016
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.866955544856362e-05,
+      "loss": 1.2939,
+      "theoretical_loss": 3.348351927284723,
+      "tokens_seen": 3010265088
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.862943347777243e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3483408210591787,
+      "tokens_seen": 3010396160
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.858931150698122e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.348329715452578,
+      "tokens_seen": 3010527232
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.854918953619003e-05,
+      "loss": 1.2775,
+      "theoretical_loss": 3.3483186104648586,
+      "tokens_seen": 3010658304
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.850906756539881e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.3483075060959595,
+      "tokens_seen": 3010789376
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.846894559460761e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.348296402345819,
+      "tokens_seen": 3010920448
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.842882362381641e-05,
+      "loss": 1.165,
+      "theoretical_loss": 3.3482852992143757,
+      "tokens_seen": 3011051520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.83887016530252e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3482741967015683,
+      "tokens_seen": 3011182592
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.834857968223399e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.3482630948073355,
+      "tokens_seen": 3011313664
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.830845771144279e-05,
+      "loss": 1.1743,
+      "theoretical_loss": 3.3482519935316155,
+      "tokens_seen": 3011444736
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.826833574065159e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.3482408928743475,
+      "tokens_seen": 3011575808
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.822821376986037e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.348229792835469,
+      "tokens_seen": 3011706880
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.818809179906916e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.3482186934149207,
+      "tokens_seen": 3011837952
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.814796982827797e-05,
+      "loss": 1.2747,
+      "theoretical_loss": 3.3482075946126395,
+      "tokens_seen": 3011969024
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.810784785748676e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.348196496428564,
+      "tokens_seen": 3012100096
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.806772588669556e-05,
+      "loss": 1.1503,
+      "theoretical_loss": 3.3481853988626344,
+      "tokens_seen": 3012231168
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.802760391590435e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.348174301914788,
+      "tokens_seen": 3012362240
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.798748194511315e-05,
+      "loss": 1.1493,
+      "theoretical_loss": 3.3481632055849637,
+      "tokens_seen": 3012493312
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.794735997432195e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.3481521098731006,
+      "tokens_seen": 3012624384
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.790723800353073e-05,
+      "loss": 1.1631,
+      "theoretical_loss": 3.3481410147791375,
+      "tokens_seen": 3012755456
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.786711603273953e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.3481299203030126,
+      "tokens_seen": 3012886528
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.782699406194833e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.3481188264446646,
+      "tokens_seen": 3013017600
+    },
+    {
+      "epoch": 0.83,
+      "objective/train/docs_used": 1009292,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1631354093551636,
+      "objective/train/theoretical_loss": 3.3481077332040328,
+      "objective/train/tokens_used": 1383607776,
+      "theoretical_loss": 3.3481077332040328,
+      "tokens_seen": 3013148672
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.778687209115712e-05,
+      "loss": 1.1461,
+      "theoretical_loss": 3.3481077332040328,
+      "tokens_seen": 3013148672
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.774675012036591e-05,
+      "loss": 1.1859,
+      "theoretical_loss": 3.3480966405810557,
+      "tokens_seen": 3013279744
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.770662814957472e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.3480855485756713,
+      "tokens_seen": 3013410816
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.766650617878351e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.3480744571878196,
+      "tokens_seen": 3013541888
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.762638420799229e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3480633664174384,
+      "tokens_seen": 3013672960
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.75862622372011e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.348052276264467,
+      "tokens_seen": 3013804032
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.754614026640989e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.348041186728844,
+      "tokens_seen": 3013935104
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.75060182956187e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.348030097810508,
+      "tokens_seen": 3014066176
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.746589632482747e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.3480190095093985,
+      "tokens_seen": 3014197248
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.742577435403627e-05,
+      "loss": 1.1832,
+      "theoretical_loss": 3.3480079218254533,
+      "tokens_seen": 3014328320
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.738565238324507e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.3479968347586118,
+      "tokens_seen": 3014459392
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.734553041245385e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.347985748308813,
+      "tokens_seen": 3014590464
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.730540844166266e-05,
+      "loss": 1.1936,
+      "theoretical_loss": 3.3479746624759956,
+      "tokens_seen": 3014721536
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.726528647087145e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.347963577260098,
+      "tokens_seen": 3014852608
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.722516450008026e-05,
+      "loss": 1.1826,
+      "theoretical_loss": 3.3479524926610593,
+      "tokens_seen": 3014983680
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.718504252928904e-05,
+      "loss": 1.173,
+      "theoretical_loss": 3.3479414086788184,
+      "tokens_seen": 3015114752
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.714492055849783e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.3479303253133144,
+      "tokens_seen": 3015245824
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.710479858770664e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.347919242564486,
+      "tokens_seen": 3015376896
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.706467661691543e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.347908160432272,
+      "tokens_seen": 3015507968
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.702455464612422e-05,
+      "loss": 1.172,
+      "theoretical_loss": 3.347897078916611,
+      "tokens_seen": 3015639040
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.698443267533301e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3478859980174427,
+      "tokens_seen": 3015770112
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.69443107045418e-05,
+      "loss": 1.3157,
+      "theoretical_loss": 3.3478749177347056,
+      "tokens_seen": 3015901184
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.69041887337506e-05,
+      "loss": 1.1689,
+      "theoretical_loss": 3.3478638380683385,
+      "tokens_seen": 3016032256
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.686406676295939e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.3478527590182803,
+      "tokens_seen": 3016163328
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.68239447921682e-05,
+      "loss": 1.1811,
+      "theoretical_loss": 3.34784168058447,
+      "tokens_seen": 3016294400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.678382282137699e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.3478306027668467,
+      "tokens_seen": 3016425472
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.674370085058578e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.3478195255653493,
+      "tokens_seen": 3016556544
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.670357887979458e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3478084489799165,
+      "tokens_seen": 3016687616
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.666345690900337e-05,
+      "loss": 1.0968,
+      "theoretical_loss": 3.3477973730104877,
+      "tokens_seen": 3016818688
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.662333493821218e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3477862976570014,
+      "tokens_seen": 3016949760
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.658321296742095e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.347775222919397,
+      "tokens_seen": 3017080832
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.654309099662976e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.3477641487976135,
+      "tokens_seen": 3017211904
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.650296902583855e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.34775307529159,
+      "tokens_seen": 3017342976
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.646284705504733e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.3477420024012647,
+      "tokens_seen": 3017474048
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.642272508425614e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3477309301265774,
+      "tokens_seen": 3017605120
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.638260311346493e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.347719858467467,
+      "tokens_seen": 3017736192
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.634248114267374e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.3477087874238727,
+      "tokens_seen": 3017867264
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.630235917188252e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3476977169957327,
+      "tokens_seen": 3017998336
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.626223720109132e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.347686647182987,
+      "tokens_seen": 3018129408
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.622211523030012e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.3476755779855747,
+      "tokens_seen": 3018260480
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.618199325950891e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.347664509403434,
+      "tokens_seen": 3018391552
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.61418712887177e-05,
+      "loss": 1.3052,
+      "theoretical_loss": 3.347653441436505,
+      "tokens_seen": 3018522624
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.61017493179265e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.347642374084726,
+      "tokens_seen": 3018653696
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.60616273471353e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.347631307348036,
+      "tokens_seen": 3018784768
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.60215053763441e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.3476202412263754,
+      "tokens_seen": 3018915840
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.598138340555287e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.3476091757196818,
+      "tokens_seen": 3019046912
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.594126143476168e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3475981108278954,
+      "tokens_seen": 3019177984
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.590113946397047e-05,
+      "loss": 1.1637,
+      "theoretical_loss": 3.3475870465509545,
+      "tokens_seen": 3019309056
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.586101749317927e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.3475759828887988,
+      "tokens_seen": 3019440128
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.582089552238806e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.3475649198413677,
+      "tokens_seen": 3019571200
+    },
+    {
+      "epoch": 0.83,
+      "objective/train/docs_used": 1011539,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1610980033874512,
+      "objective/train/theoretical_loss": 3.3475538574085992,
+      "objective/train/tokens_used": 1390161376,
+      "theoretical_loss": 3.3475538574085992,
+      "tokens_seen": 3019702272
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.578077355159686e-05,
+      "loss": 1.1833,
+      "theoretical_loss": 3.3475538574085992,
+      "tokens_seen": 3019702272
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.574065158080566e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.3475427955904338,
+      "tokens_seen": 3019833344
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.570052961001444e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.34753173438681,
+      "tokens_seen": 3019964416
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.566040763922324e-05,
+      "loss": 1.2127,
+      "theoretical_loss": 3.347520673797667,
+      "tokens_seen": 3020095488
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.562028566843204e-05,
+      "loss": 1.1677,
+      "theoretical_loss": 3.347509613822944,
+      "tokens_seen": 3020226560
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.558016369764084e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.3474985544625806,
+      "tokens_seen": 3020357632
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.554004172684962e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.3474874957165155,
+      "tokens_seen": 3020488704
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.549991975605841e-05,
+      "loss": 1.197,
+      "theoretical_loss": 3.3474764375846884,
+      "tokens_seen": 3020619776
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.545979778526722e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.3474653800670384,
+      "tokens_seen": 3020750848
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.5419675814476e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3474543231635043,
+      "tokens_seen": 3020881920
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.53795538436848e-05,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3474432668740257,
+      "tokens_seen": 3021012992
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.53394318728936e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.3474322111985417,
+      "tokens_seen": 3021144064
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.52993099021024e-05,
+      "loss": 1.1338,
+      "theoretical_loss": 3.347421156136992,
+      "tokens_seen": 3021275136
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.525918793131118e-05,
+      "loss": 1.1522,
+      "theoretical_loss": 3.3474101016893156,
+      "tokens_seen": 3021406208
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.521906596051998e-05,
+      "loss": 1.1699,
+      "theoretical_loss": 3.3473990478554514,
+      "tokens_seen": 3021537280
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.517894398972878e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3473879946353393,
+      "tokens_seen": 3021668352
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.513882201893758e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3473769420289186,
+      "tokens_seen": 3021799424
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.509870004814637e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.347365890036128,
+      "tokens_seen": 3021930496
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.505857807735516e-05,
+      "loss": 1.1882,
+      "theoretical_loss": 3.347354838656907,
+      "tokens_seen": 3022061568
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.501845610656395e-05,
+      "loss": 1.2923,
+      "theoretical_loss": 3.3473437878911954,
+      "tokens_seen": 3022192640
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.497833413577275e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3473327377389324,
+      "tokens_seen": 3022323712
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.493821216498154e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.347321688200057,
+      "tokens_seen": 3022454784
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.489809019419035e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.3473106392745087,
+      "tokens_seen": 3022585856
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.485796822339914e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.347299590962227,
+      "tokens_seen": 3022716928
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.481784625260793e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.347288543263151,
+      "tokens_seen": 3022848000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.477772428181672e-05,
+      "loss": 1.1462,
+      "theoretical_loss": 3.3472774961772203,
+      "tokens_seen": 3022979072
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.473760231102552e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.3472664497043745,
+      "tokens_seen": 3023110144
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.469748034023432e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3472554038445526,
+      "tokens_seen": 3023241216
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.46573583694431e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.347244358597694,
+      "tokens_seen": 3023372288
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.461723639865191e-05,
+      "loss": 1.1651,
+      "theoretical_loss": 3.3472333139637382,
+      "tokens_seen": 3023503360
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.45771144278607e-05,
+      "loss": 1.2655,
+      "theoretical_loss": 3.347222269942625,
+      "tokens_seen": 3023634432
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.453699245706948e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3472112265342933,
+      "tokens_seen": 3023765504
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.449687048627829e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.347200183738683,
+      "tokens_seen": 3023896576
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.445674851548708e-05,
+      "loss": 1.1686,
+      "theoretical_loss": 3.347189141555733,
+      "tokens_seen": 3024027648
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.441662654469589e-05,
+      "loss": 1.3171,
+      "theoretical_loss": 3.347178099985383,
+      "tokens_seen": 3024158720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.437650457390466e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.347167059027573,
+      "tokens_seen": 3024289792
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.433638260311347e-05,
+      "loss": 1.1696,
+      "theoretical_loss": 3.3471560186822416,
+      "tokens_seen": 3024420864
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.429626063232226e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.3471449789493284,
+      "tokens_seen": 3024551936
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.425613866153106e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3471339398287734,
+      "tokens_seen": 3024683008
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.421601669073985e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.347122901320516,
+      "tokens_seen": 3024814080
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.417589471994864e-05,
+      "loss": 1.2193,
+      "theoretical_loss": 3.347111863424496,
+      "tokens_seen": 3024945152
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.413577274915745e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.3471008261406516,
+      "tokens_seen": 3025076224
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.409565077836623e-05,
+      "loss": 1.1733,
+      "theoretical_loss": 3.3470897894689235,
+      "tokens_seen": 3025207296
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.405552880757502e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.347078753409251,
+      "tokens_seen": 3025338368
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.401540683678383e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.347067717961574,
+      "tokens_seen": 3025469440
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.397528486599262e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3470566831258313,
+      "tokens_seen": 3025600512
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.393516289520141e-05,
+      "loss": 1.2655,
+      "theoretical_loss": 3.347045648901963,
+      "tokens_seen": 3025731584
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.38950409244102e-05,
+      "loss": 1.1513,
+      "theoretical_loss": 3.347034615289908,
+      "tokens_seen": 3025862656
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.385491895361901e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.347023582289607,
+      "tokens_seen": 3025993728
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.38147969828278e-05,
+      "loss": 1.1677,
+      "theoretical_loss": 3.3470125499009984,
+      "tokens_seen": 3026124800
+    },
+    {
+      "epoch": 0.83,
+      "objective/train/docs_used": 1013697,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2726037502288818,
+      "objective/train/theoretical_loss": 3.3470015181240225,
+      "objective/train/tokens_used": 1396714976,
+      "theoretical_loss": 3.3470015181240225,
+      "tokens_seen": 3026255872
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.377467501203658e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.3470015181240225,
+      "tokens_seen": 3026255872
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.373455304124539e-05,
+      "loss": 1.1965,
+      "theoretical_loss": 3.3469904869586187,
+      "tokens_seen": 3026386944
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.369443107045418e-05,
+      "loss": 1.1635,
+      "theoretical_loss": 3.3469794564047266,
+      "tokens_seen": 3026518016
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.365430909966299e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.346968426462286,
+      "tokens_seen": 3026649088
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.361418712887177e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.346957397131236,
+      "tokens_seen": 3026780160
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.357406515808056e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3469463684115173,
+      "tokens_seen": 3026911232
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.353394318728937e-05,
+      "loss": 1.2036,
+      "theoretical_loss": 3.3469353403030686,
+      "tokens_seen": 3027042304
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.349382121649815e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.3469243128058297,
+      "tokens_seen": 3027173376
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.345369924570695e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.34691328591974,
+      "tokens_seen": 3027304448
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.341357727491575e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.3469022596447404,
+      "tokens_seen": 3027435520
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.337345530412455e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3468912339807693,
+      "tokens_seen": 3027566592
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.333333333333333e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.3468802089277667,
+      "tokens_seen": 3027697664
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.329321136254212e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.346869184485673,
+      "tokens_seen": 3027828736
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.325308939175093e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.3468581606544268,
+      "tokens_seen": 3027959808
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.321296742095972e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.3468471374339686,
+      "tokens_seen": 3028090880
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.317284545016852e-05,
+      "loss": 1.1815,
+      "theoretical_loss": 3.3468361148242374,
+      "tokens_seen": 3028221952
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.313272347937731e-05,
+      "loss": 1.1612,
+      "theoretical_loss": 3.3468250928251737,
+      "tokens_seen": 3028353024
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.309260150858611e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.346814071436717,
+      "tokens_seen": 3028484096
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.30524795377949e-05,
+      "loss": 1.1866,
+      "theoretical_loss": 3.346803050658807,
+      "tokens_seen": 3028615168
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.301235756700369e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3467920304913834,
+      "tokens_seen": 3028746240
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.297223559621249e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.346781010934386,
+      "tokens_seen": 3028877312
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.293211362542129e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3467699919877547,
+      "tokens_seen": 3029008384
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.289199165463008e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.346758973651429,
+      "tokens_seen": 3029139456
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.285186968383887e-05,
+      "loss": 1.185,
+      "theoretical_loss": 3.346747955925349,
+      "tokens_seen": 3029270528
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.281174771304766e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.346736938809454,
+      "tokens_seen": 3029401600
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.277162574225647e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.3467259223036847,
+      "tokens_seen": 3029532672
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.273150377146525e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.34671490640798,
+      "tokens_seen": 3029663744
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.269138180067406e-05,
+      "loss": 1.155,
+      "theoretical_loss": 3.34670389112228,
+      "tokens_seen": 3029794816
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.265125982988285e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.346692876446525,
+      "tokens_seen": 3029925888
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.261113785909164e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3466818623806542,
+      "tokens_seen": 3030056960
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.257101588830043e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.3466708489246084,
+      "tokens_seen": 3030188032
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.253089391750923e-05,
+      "loss": 1.1472,
+      "theoretical_loss": 3.346659836078326,
+      "tokens_seen": 3030319104
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.249077194671803e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.3466488238417478,
+      "tokens_seen": 3030450176
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.245064997592681e-05,
+      "loss": 1.1618,
+      "theoretical_loss": 3.346637812214814,
+      "tokens_seen": 3030581248
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.241052800513562e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.3466268011974636,
+      "tokens_seen": 3030712320
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.237040603434441e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.346615790789637,
+      "tokens_seen": 3030843392
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.23302840635532e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.346604780991274,
+      "tokens_seen": 3030974464
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.2290162092762e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3465937718023144,
+      "tokens_seen": 3031105536
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.225004012197079e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.3465827632226985,
+      "tokens_seen": 3031236608
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.22099181511796e-05,
+      "loss": 1.1842,
+      "theoretical_loss": 3.346571755252366,
+      "tokens_seen": 3031367680
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.216979618038837e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.3465607478912567,
+      "tokens_seen": 3031498752
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.212967420959718e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.3465497411393104,
+      "tokens_seen": 3031629824
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.208955223880597e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.346538734996468,
+      "tokens_seen": 3031760896
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.204943026801477e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.346527729462668,
+      "tokens_seen": 3031891968
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.200930829722356e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3465167245378513,
+      "tokens_seen": 3032023040
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.196918632643235e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.346505720221958,
+      "tokens_seen": 3032154112
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.192906435564116e-05,
+      "loss": 1.1557,
+      "theoretical_loss": 3.3464947165149277,
+      "tokens_seen": 3032285184
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.188894238484995e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.3464837134167005,
+      "tokens_seen": 3032416256
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.184882041405873e-05,
+      "loss": 1.2815,
+      "theoretical_loss": 3.3464727109272165,
+      "tokens_seen": 3032547328
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.180869844326754e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.3464617090464155,
+      "tokens_seen": 3032678400
+    },
+    {
+      "epoch": 0.84,
+      "objective/train/docs_used": 1015863,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2398011684417725,
+      "objective/train/theoretical_loss": 3.3464507077742374,
+      "objective/train/tokens_used": 1403268576,
+      "theoretical_loss": 3.3464507077742374,
+      "tokens_seen": 3032809472
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.176857647247633e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.3464507077742374,
+      "tokens_seen": 3032809472
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.172845450168512e-05,
+      "loss": 1.2846,
+      "theoretical_loss": 3.346439707110623,
+      "tokens_seen": 3032940544
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.168833253089391e-05,
+      "loss": 1.2362,
+      "theoretical_loss": 3.3464287070555114,
+      "tokens_seen": 3033071616
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.164821056010272e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3464177076088433,
+      "tokens_seen": 3033202688
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.160808858931151e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.3464067087705582,
+      "tokens_seen": 3033333760
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.156796661852029e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.346395710540597,
+      "tokens_seen": 3033464832
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.15278446477291e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3463847129188986,
+      "tokens_seen": 3033595904
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.148772267693789e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.346373715905404,
+      "tokens_seen": 3033726976
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.14476007061467e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.3463627195000534,
+      "tokens_seen": 3033858048
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.140747873535548e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.346351723702786,
+      "tokens_seen": 3033989120
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.136735676456427e-05,
+      "loss": 1.1222,
+      "theoretical_loss": 3.3463407285135425,
+      "tokens_seen": 3034120192
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.132723479377308e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.3463297339322633,
+      "tokens_seen": 3034251264
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.128711282298187e-05,
+      "loss": 1.1903,
+      "theoretical_loss": 3.3463187399588876,
+      "tokens_seen": 3034382336
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.124699085219066e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.3463077465933564,
+      "tokens_seen": 3034513408
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.120686888139945e-05,
+      "loss": 1.234,
+      "theoretical_loss": 3.3462967538356096,
+      "tokens_seen": 3034644480
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.116674691060826e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.3462857616855874,
+      "tokens_seen": 3034775552
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.112662493981704e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3462747701432294,
+      "tokens_seen": 3034906624
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.108650296902583e-05,
+      "loss": 1.1857,
+      "theoretical_loss": 3.3462637792084764,
+      "tokens_seen": 3035037696
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.104638099823464e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3462527888812685,
+      "tokens_seen": 3035168768
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.100625902744343e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.346241799161546,
+      "tokens_seen": 3035299840
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.096613705665222e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.3462308100492484,
+      "tokens_seen": 3035430912
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.092601508586102e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.346219821544316,
+      "tokens_seen": 3035561984
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.088589311506981e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.34620883364669,
+      "tokens_seen": 3035693056
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.084577114427862e-05,
+      "loss": 1.2488,
+      "theoretical_loss": 3.34619784635631,
+      "tokens_seen": 3035824128
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.08056491734874e-05,
+      "loss": 1.1646,
+      "theoretical_loss": 3.346186859673116,
+      "tokens_seen": 3035955200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.07655272026962e-05,
+      "loss": 1.2088,
+      "theoretical_loss": 3.346175873597048,
+      "tokens_seen": 3036086272
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.0725405231905e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3461648881280475,
+      "tokens_seen": 3036217344
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.068528326111379e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.346153903266053,
+      "tokens_seen": 3036348416
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.064516129032258e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3461429190110064,
+      "tokens_seen": 3036479488
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.060503931953137e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.3461319353628474,
+      "tokens_seen": 3036610560
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.056491734874018e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3461209523215154,
+      "tokens_seen": 3036741632
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.052479537794896e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.346109969886952,
+      "tokens_seen": 3036872704
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.048467340715777e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3460989880590963,
+      "tokens_seen": 3037003776
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.044455143636656e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3460880068378898,
+      "tokens_seen": 3037134848
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.040442946557535e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3460770262232717,
+      "tokens_seen": 3037265920
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.036430749478414e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.346066046215183,
+      "tokens_seen": 3037396992
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.032418552399294e-05,
+      "loss": 1.151,
+      "theoretical_loss": 3.3460550668135642,
+      "tokens_seen": 3037528064
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.028406355320174e-05,
+      "loss": 1.2004,
+      "theoretical_loss": 3.3460440880183553,
+      "tokens_seen": 3037659136
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.024394158241052e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.346033109829496,
+      "tokens_seen": 3037790208
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.020381961161933e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3460221322469277,
+      "tokens_seen": 3037921280
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.016369764082812e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.3460111552705905,
+      "tokens_seen": 3038052352
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.012357567003691e-05,
+      "loss": 1.2087,
+      "theoretical_loss": 3.3460001789004243,
+      "tokens_seen": 3038183424
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.00834536992457e-05,
+      "loss": 1.2904,
+      "theoretical_loss": 3.34598920313637,
+      "tokens_seen": 3038314496
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.00433317284545e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.3459782279783674,
+      "tokens_seen": 3038445568
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.00032097576633e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3459672534263576,
+      "tokens_seen": 3038576640
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.99630877868721e-05,
+      "loss": 1.1687,
+      "theoretical_loss": 3.345956279480281,
+      "tokens_seen": 3038707712
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.992296581608088e-05,
+      "loss": 1.2828,
+      "theoretical_loss": 3.3459453061400772,
+      "tokens_seen": 3038838784
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.988284384528968e-05,
+      "loss": 1.1633,
+      "theoretical_loss": 3.3459343334056872,
+      "tokens_seen": 3038969856
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.984272187449848e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.345923361277051,
+      "tokens_seen": 3039100928
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.980259990370727e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.34591238975411,
+      "tokens_seen": 3039232000
+    },
+    {
+      "epoch": 0.84,
+      "objective/train/docs_used": 1018122,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2630326747894287,
+      "objective/train/theoretical_loss": 3.3459014188368035,
+      "objective/train/tokens_used": 1409822176,
+      "theoretical_loss": 3.3459014188368035,
+      "tokens_seen": 3039363072
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.976247793291606e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.3459014188368035,
+      "tokens_seen": 3039363072
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.972235596212487e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.345890448525073,
+      "tokens_seen": 3039494144
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.968223399133366e-05,
+      "loss": 1.1658,
+      "theoretical_loss": 3.345879478818858,
+      "tokens_seen": 3039625216
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.964211202054244e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.345868509718099,
+      "tokens_seen": 3039756288
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.960199004975125e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.345857541222738,
+      "tokens_seen": 3039887360
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.956186807896004e-05,
+      "loss": 1.1846,
+      "theoretical_loss": 3.345846573332713,
+      "tokens_seen": 3040018432
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.952174610816885e-05,
+      "loss": 1.3049,
+      "theoretical_loss": 3.345835606047967,
+      "tokens_seen": 3040149504
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.948162413737762e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.345824639368439,
+      "tokens_seen": 3040280576
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.944150216658642e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.3458136732940695,
+      "tokens_seen": 3040411648
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.940138019579522e-05,
+      "loss": 1.2106,
+      "theoretical_loss": 3.3458027078247996,
+      "tokens_seen": 3040542720
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.9361258225004e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.3457917429605697,
+      "tokens_seen": 3040673792
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.932113625421281e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.3457807787013203,
+      "tokens_seen": 3040804864
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.92810142834216e-05,
+      "loss": 1.2024,
+      "theoretical_loss": 3.345769815046992,
+      "tokens_seen": 3040935936
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.924089231263041e-05,
+      "loss": 1.1775,
+      "theoretical_loss": 3.345758851997525,
+      "tokens_seen": 3041067008
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.920077034183919e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.34574788955286,
+      "tokens_seen": 3041198080
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.916064837104798e-05,
+      "loss": 1.211,
+      "theoretical_loss": 3.345736927712938,
+      "tokens_seen": 3041329152
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.912052640025679e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.345725966477699,
+      "tokens_seen": 3041460224
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.908040442946558e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.345715005847084,
+      "tokens_seen": 3041591296
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.904028245867437e-05,
+      "loss": 1.1736,
+      "theoretical_loss": 3.3457040458210336,
+      "tokens_seen": 3041722368
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.900016048788316e-05,
+      "loss": 1.3135,
+      "theoretical_loss": 3.3456930863994883,
+      "tokens_seen": 3041853440
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.896003851709197e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.3456821275823883,
+      "tokens_seen": 3041984512
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.891991654630075e-05,
+      "loss": 1.1631,
+      "theoretical_loss": 3.345671169369675,
+      "tokens_seen": 3042115584
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.887979457550954e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.345660211761288,
+      "tokens_seen": 3042246656
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.883967260471835e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.345649254757169,
+      "tokens_seen": 3042377728
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.879955063392714e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.345638298357258,
+      "tokens_seen": 3042508800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.875942866313593e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.345627342561496,
+      "tokens_seen": 3042639872
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.871930669234473e-05,
+      "loss": 1.1791,
+      "theoretical_loss": 3.3456163873698235,
+      "tokens_seen": 3042770944
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.867918472155352e-05,
+      "loss": 1.1915,
+      "theoretical_loss": 3.345605432782181,
+      "tokens_seen": 3042902016
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.863906275076233e-05,
+      "loss": 1.1659,
+      "theoretical_loss": 3.34559447879851,
+      "tokens_seen": 3043033088
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.85989407799711e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.34558352541875,
+      "tokens_seen": 3043164160
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.855881880917991e-05,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3455725726428422,
+      "tokens_seen": 3043295232
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.85186968383887e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3455616204707272,
+      "tokens_seen": 3043426304
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.847857486759751e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.345550668902346,
+      "tokens_seen": 3043557376
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.843845289680629e-05,
+      "loss": 1.1567,
+      "theoretical_loss": 3.3455397179376396,
+      "tokens_seen": 3043688448
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.839833092601508e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.345528767576548,
+      "tokens_seen": 3043819520
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.835820895522389e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.3455178178190126,
+      "tokens_seen": 3043950592
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.831808698443267e-05,
+      "loss": 1.1778,
+      "theoretical_loss": 3.345506868664973,
+      "tokens_seen": 3044081664
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.827796501364148e-05,
+      "loss": 1.2026,
+      "theoretical_loss": 3.345495920114372,
+      "tokens_seen": 3044212736
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.823784304285027e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.345484972167148,
+      "tokens_seen": 3044343808
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.819772107205906e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3454740248232433,
+      "tokens_seen": 3044474880
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.815759910126785e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.3454630780825987,
+      "tokens_seen": 3044605952
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.811747713047665e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.345452131945154,
+      "tokens_seen": 3044737024
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.807735515968545e-05,
+      "loss": 1.1722,
+      "theoretical_loss": 3.3454411864108504,
+      "tokens_seen": 3044868096
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.803723318889425e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3454302414796295,
+      "tokens_seen": 3044999168
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.799711121810304e-05,
+      "loss": 1.1582,
+      "theoretical_loss": 3.345419297151431,
+      "tokens_seen": 3045130240
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.795698924731183e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.3454083534261962,
+      "tokens_seen": 3045261312
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.791686727652062e-05,
+      "loss": 1.1719,
+      "theoretical_loss": 3.345397410303866,
+      "tokens_seen": 3045392384
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.787674530572942e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.3453864677843814,
+      "tokens_seen": 3045523456
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.783662333493821e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.345375525867683,
+      "tokens_seen": 3045654528
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.779650136414702e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.345364584553711,
+      "tokens_seen": 3045785600
+    },
+    {
+      "epoch": 0.85,
+      "objective/train/docs_used": 1020383,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.095306634902954,
+      "objective/train/theoretical_loss": 3.345353643842408,
+      "objective/train/tokens_used": 1416375776,
+      "theoretical_loss": 3.345353643842408,
+      "tokens_seen": 3045916672
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.775637939335581e-05,
+      "loss": 1.1176,
+      "theoretical_loss": 3.345353643842408,
+      "tokens_seen": 3045916672
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.771625742256459e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3453427037337136,
+      "tokens_seen": 3046047744
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.76761354517734e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.345331764227568,
+      "tokens_seen": 3046178816
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.763601348098219e-05,
+      "loss": 1.1551,
+      "theoretical_loss": 3.345320825323914,
+      "tokens_seen": 3046309888
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.759589151019099e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.345309887022691,
+      "tokens_seen": 3046440960
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.755576953939977e-05,
+      "loss": 1.1673,
+      "theoretical_loss": 3.3452989493238405,
+      "tokens_seen": 3046572032
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.751564756860858e-05,
+      "loss": 1.2252,
+      "theoretical_loss": 3.3452880122273028,
+      "tokens_seen": 3046703104
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.747552559781737e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.34527707573302,
+      "tokens_seen": 3046834176
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.743540362702615e-05,
+      "loss": 1.1416,
+      "theoretical_loss": 3.345266139840932,
+      "tokens_seen": 3046965248
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.739528165623496e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.34525520455098,
+      "tokens_seen": 3047096320
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.735515968544375e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3452442698631053,
+      "tokens_seen": 3047227392
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.731503771465256e-05,
+      "loss": 1.1235,
+      "theoretical_loss": 3.3452333357772486,
+      "tokens_seen": 3047358464
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.727491574386133e-05,
+      "loss": 1.1881,
+      "theoretical_loss": 3.3452224022933508,
+      "tokens_seen": 3047489536
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.723479377307013e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.345211469411353,
+      "tokens_seen": 3047620608
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.719467180227893e-05,
+      "loss": 1.3047,
+      "theoretical_loss": 3.3452005371311957,
+      "tokens_seen": 3047751680
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.715454983148773e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.3451896054528207,
+      "tokens_seen": 3047882752
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.711442786069652e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.345178674376168,
+      "tokens_seen": 3048013824
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.707430588990531e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3451677439011798,
+      "tokens_seen": 3048144896
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.703418391911412e-05,
+      "loss": 1.1906,
+      "theoretical_loss": 3.3451568140277965,
+      "tokens_seen": 3048275968
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.69940619483229e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.3451458847559588,
+      "tokens_seen": 3048407040
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.695393997753169e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.3451349560856083,
+      "tokens_seen": 3048538112
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.69138180067405e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.345124028016686,
+      "tokens_seen": 3048669184
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.687369603594929e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.345113100549132,
+      "tokens_seen": 3048800256
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.683357406515808e-05,
+      "loss": 1.2102,
+      "theoretical_loss": 3.3451021736828888,
+      "tokens_seen": 3048931328
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.679345209436687e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.345091247417897,
+      "tokens_seen": 3049062400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.675333012357567e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.3450803217540965,
+      "tokens_seen": 3049193472
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.671320815278447e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.34506939669143,
+      "tokens_seen": 3049324544
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.667308618199325e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.3450584722298378,
+      "tokens_seen": 3049455616
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.663296421120206e-05,
+      "loss": 1.1696,
+      "theoretical_loss": 3.3450475483692608,
+      "tokens_seen": 3049586688
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.659284224041085e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.345036625109641,
+      "tokens_seen": 3049717760
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.655272026961964e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.3450257024509185,
+      "tokens_seen": 3049848832
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.651259829882844e-05,
+      "loss": 1.2268,
+      "theoretical_loss": 3.3450147803930346,
+      "tokens_seen": 3049979904
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.647247632803723e-05,
+      "loss": 1.1942,
+      "theoretical_loss": 3.345003858935931,
+      "tokens_seen": 3050110976
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.643235435724604e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.3449929380795487,
+      "tokens_seen": 3050242048
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.639223238645482e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3449820178238285,
+      "tokens_seen": 3050373120
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.635211041566362e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.3449710981687115,
+      "tokens_seen": 3050504192
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.631198844487241e-05,
+      "loss": 1.2885,
+      "theoretical_loss": 3.344960179114139,
+      "tokens_seen": 3050635264
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.627186647408121e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.3449492606600524,
+      "tokens_seen": 3050766336
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.623174450329e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.344938342806393,
+      "tokens_seen": 3050897408
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.619162253249879e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3449274255531014,
+      "tokens_seen": 3051028480
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.61515005617076e-05,
+      "loss": 1.1536,
+      "theoretical_loss": 3.3449165089001194,
+      "tokens_seen": 3051159552
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.611137859091639e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.344905592847388,
+      "tokens_seen": 3051290624
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.607125662012518e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.344894677394848,
+      "tokens_seen": 3051421696
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.603113464933398e-05,
+      "loss": 1.1994,
+      "theoretical_loss": 3.344883762542441,
+      "tokens_seen": 3051552768
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.599101267854277e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.344872848290108,
+      "tokens_seen": 3051683840
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.595089070775156e-05,
+      "loss": 1.308,
+      "theoretical_loss": 3.3448619346377906,
+      "tokens_seen": 3051814912
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.591076873696036e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.3448510215854297,
+      "tokens_seen": 3051945984
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.587064676616916e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.344840109132967,
+      "tokens_seen": 3052077056
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.583052479537795e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.3448291972803434,
+      "tokens_seen": 3052208128
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.579040282458673e-05,
+      "loss": 1.1862,
+      "theoretical_loss": 3.3448182860275004,
+      "tokens_seen": 3052339200
+    },
+    {
+      "epoch": 0.85,
+      "objective/train/docs_used": 1022554,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2686384916305542,
+      "objective/train/theoretical_loss": 3.344807375374379,
+      "objective/train/tokens_used": 1422929376,
+      "theoretical_loss": 3.344807375374379,
+      "tokens_seen": 3052470272
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.575028085379554e-05,
+      "loss": 1.3216,
+      "theoretical_loss": 3.344807375374379,
+      "tokens_seen": 3052470272
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.571015888300433e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3447964653209206,
+      "tokens_seen": 3052601344
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.567003691221314e-05,
+      "loss": 1.181,
+      "theoretical_loss": 3.3447855558670665,
+      "tokens_seen": 3052732416
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.562991494142192e-05,
+      "loss": 1.2064,
+      "theoretical_loss": 3.344774647012758,
+      "tokens_seen": 3052863488
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.558979297063073e-05,
+      "loss": 1.1997,
+      "theoretical_loss": 3.3447637387579365,
+      "tokens_seen": 3052994560
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.554967099983952e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.344752831102543,
+      "tokens_seen": 3053125632
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.55095490290483e-05,
+      "loss": 1.3095,
+      "theoretical_loss": 3.3447419240465193,
+      "tokens_seen": 3053256704
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.54694270582571e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.3447310175898064,
+      "tokens_seen": 3053387776
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.54293050874659e-05,
+      "loss": 1.141,
+      "theoretical_loss": 3.344720111732346,
+      "tokens_seen": 3053518848
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.53891831166747e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3447092064740795,
+      "tokens_seen": 3053649920
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.534906114588348e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.3446983018149474,
+      "tokens_seen": 3053780992
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.530893917509227e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.3446873977548917,
+      "tokens_seen": 3053912064
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.526881720430108e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.344676494293854,
+      "tokens_seen": 3054043136
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.522869523350987e-05,
+      "loss": 1.1265,
+      "theoretical_loss": 3.344665591431775,
+      "tokens_seen": 3054174208
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.518857326271867e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.344654689168597,
+      "tokens_seen": 3054305280
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.514845129192746e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.3446437875042605,
+      "tokens_seen": 3054436352
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.510832932113627e-05,
+      "loss": 1.1623,
+      "theoretical_loss": 3.3446328864387076,
+      "tokens_seen": 3054567424
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.506820735034504e-05,
+      "loss": 1.1849,
+      "theoretical_loss": 3.3446219859718793,
+      "tokens_seen": 3054698496
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.502808537955384e-05,
+      "loss": 1.1702,
+      "theoretical_loss": 3.344611086103717,
+      "tokens_seen": 3054829568
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.498796340876264e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3446001868341626,
+      "tokens_seen": 3054960640
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.494784143797144e-05,
+      "loss": 1.1627,
+      "theoretical_loss": 3.3445892881631574,
+      "tokens_seen": 3055091712
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.490771946718023e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3445783900906423,
+      "tokens_seen": 3055222784
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.486759749638902e-05,
+      "loss": 1.1933,
+      "theoretical_loss": 3.344567492616559,
+      "tokens_seen": 3055353856
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.482747552559781e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.34455659574085,
+      "tokens_seen": 3055484928
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.478735355480662e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3445456994634553,
+      "tokens_seen": 3055616000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.47472315840154e-05,
+      "loss": 1.1672,
+      "theoretical_loss": 3.344534803784317,
+      "tokens_seen": 3055747072
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.47071096132242e-05,
+      "loss": 1.26,
+      "theoretical_loss": 3.3445239087033762,
+      "tokens_seen": 3055878144
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.4666987642433e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.3445130142205755,
+      "tokens_seen": 3056009216
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.462686567164179e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.3445021203358554,
+      "tokens_seen": 3056140288
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.458674370085058e-05,
+      "loss": 1.158,
+      "theoretical_loss": 3.344491227049158,
+      "tokens_seen": 3056271360
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.454662173005938e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.344480334360424,
+      "tokens_seen": 3056402432
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.450649975926818e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.3444694422695957,
+      "tokens_seen": 3056533504
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.446637778847696e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.3444585507766145,
+      "tokens_seen": 3056664576
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.442625581768577e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.344447659881422,
+      "tokens_seen": 3056795648
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.438613384689456e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.3444367695839596,
+      "tokens_seen": 3056926720
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.434601187610337e-05,
+      "loss": 1.1604,
+      "theoretical_loss": 3.3444258798841684,
+      "tokens_seen": 3057057792
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.430588990531215e-05,
+      "loss": 1.1689,
+      "theoretical_loss": 3.344414990781991,
+      "tokens_seen": 3057188864
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.426576793452094e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.3444041022773687,
+      "tokens_seen": 3057319936
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.422564596372975e-05,
+      "loss": 1.1515,
+      "theoretical_loss": 3.344393214370242,
+      "tokens_seen": 3057451008
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.418552399293853e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.344382327060554,
+      "tokens_seen": 3057582080
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.414540202214733e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3443714403482456,
+      "tokens_seen": 3057713152
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.410528005135612e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.3443605542332584,
+      "tokens_seen": 3057844224
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.406515808056492e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.344349668715534,
+      "tokens_seen": 3057975296
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.402503610977371e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.3443387837950143,
+      "tokens_seen": 3058106368
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.39849141389825e-05,
+      "loss": 1.1669,
+      "theoretical_loss": 3.344327899471641,
+      "tokens_seen": 3058237440
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.394479216819131e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.344317015745355,
+      "tokens_seen": 3058368512
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.39046701974001e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.344306132616099,
+      "tokens_seen": 3058499584
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.38645482266089e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3442952500838135,
+      "tokens_seen": 3058630656
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.382442625581769e-05,
+      "loss": 1.1964,
+      "theoretical_loss": 3.3442843681484415,
+      "tokens_seen": 3058761728
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.378430428502648e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.3442734868099233,
+      "tokens_seen": 3058892800
+    },
+    {
+      "epoch": 0.85,
+      "objective/train/docs_used": 1025165,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2076010704040527,
+      "objective/train/theoretical_loss": 3.344262606068202,
+      "objective/train/tokens_used": 1429482976,
+      "theoretical_loss": 3.344262606068202,
+      "tokens_seen": 3059023872
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.374418231423529e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.344262606068202,
+      "tokens_seen": 3059023872
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.370406034344407e-05,
+      "loss": 1.2004,
+      "theoretical_loss": 3.3442517259232183,
+      "tokens_seen": 3059154944
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.366393837265287e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.3442408463749143,
+      "tokens_seen": 3059286016
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.362381640186166e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3442299674232316,
+      "tokens_seen": 3059417088
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.358369443107044e-05,
+      "loss": 1.2005,
+      "theoretical_loss": 3.344219089068112,
+      "tokens_seen": 3059548160
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.354357246027925e-05,
+      "loss": 1.204,
+      "theoretical_loss": 3.344208211309497,
+      "tokens_seen": 3059679232
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.350345048948804e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3441973341473283,
+      "tokens_seen": 3059810304
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.346332851869685e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.3441864575815483,
+      "tokens_seen": 3059941376
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.342320654790563e-05,
+      "loss": 1.302,
+      "theoretical_loss": 3.344175581612098,
+      "tokens_seen": 3060072448
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.338308457711443e-05,
+      "loss": 1.184,
+      "theoretical_loss": 3.34416470623892,
+      "tokens_seen": 3060203520
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.334296260632323e-05,
+      "loss": 1.2077,
+      "theoretical_loss": 3.344153831461955,
+      "tokens_seen": 3060334592
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.330284063553202e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3441429572811456,
+      "tokens_seen": 3060465664
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.326271866474081e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3441320836964334,
+      "tokens_seen": 3060596736
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.32225966939496e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.34412121070776,
+      "tokens_seen": 3060727808
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.318247472315841e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.344110338315067,
+      "tokens_seen": 3060858880
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.314235275236719e-05,
+      "loss": 1.1706,
+      "theoretical_loss": 3.3440994665182973,
+      "tokens_seen": 3060989952
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.310223078157598e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.3440885953173916,
+      "tokens_seen": 3061121024
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.306210881078479e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.344077724712292,
+      "tokens_seen": 3061252096
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.302198683999358e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.3440668547029406,
+      "tokens_seen": 3061383168
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.298186486920238e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.344055985289279,
+      "tokens_seen": 3061514240
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.294174289841117e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.344045116471249,
+      "tokens_seen": 3061645312
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.290162092761998e-05,
+      "loss": 1.1719,
+      "theoretical_loss": 3.344034248248793,
+      "tokens_seen": 3061776384
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.286149895682877e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.3440233806218522,
+      "tokens_seen": 3061907456
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.282137698603755e-05,
+      "loss": 1.2064,
+      "theoretical_loss": 3.3440125135903687,
+      "tokens_seen": 3062038528
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.278125501524635e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.3440016471542844,
+      "tokens_seen": 3062169600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.274113304445515e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.343990781313541,
+      "tokens_seen": 3062300672
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.270101107366394e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.343979916068081,
+      "tokens_seen": 3062431744
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.266088910287273e-05,
+      "loss": 1.1475,
+      "theoretical_loss": 3.343969051417846,
+      "tokens_seen": 3062562816
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.262076713208152e-05,
+      "loss": 1.1807,
+      "theoretical_loss": 3.343958187362778,
+      "tokens_seen": 3062693888
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.258064516129033e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.3439473239028183,
+      "tokens_seen": 3062824960
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.254052319049911e-05,
+      "loss": 1.2744,
+      "theoretical_loss": 3.34393646103791,
+      "tokens_seen": 3062956032
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.250040121970792e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.343925598767994,
+      "tokens_seen": 3063087104
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.246027924891671e-05,
+      "loss": 1.2161,
+      "theoretical_loss": 3.3439147370930122,
+      "tokens_seen": 3063218176
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.242015727812552e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.3439038760129076,
+      "tokens_seen": 3063349248
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.23800353073343e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3438930155276214,
+      "tokens_seen": 3063480320
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.233991333654309e-05,
+      "loss": 1.1162,
+      "theoretical_loss": 3.3438821556370955,
+      "tokens_seen": 3063611392
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.22997913657519e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.3438712963412724,
+      "tokens_seen": 3063742464
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.225966939496067e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.3438604376400933,
+      "tokens_seen": 3063873536
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.221954742416948e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.3438495795335013,
+      "tokens_seen": 3064004608
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.217942545337827e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.3438387220214376,
+      "tokens_seen": 3064135680
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.213930348258706e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3438278651038447,
+      "tokens_seen": 3064266752
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.209918151179586e-05,
+      "loss": 1.33,
+      "theoretical_loss": 3.343817008780664,
+      "tokens_seen": 3064397824
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.205905954100465e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.3438061530518377,
+      "tokens_seen": 3064528896
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.201893757021346e-05,
+      "loss": 1.1962,
+      "theoretical_loss": 3.3437952979173087,
+      "tokens_seen": 3064659968
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.197881559942225e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.343784443377018,
+      "tokens_seen": 3064791040
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.193869362863104e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.3437735894309077,
+      "tokens_seen": 3064922112
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.189857165783983e-05,
+      "loss": 1.219,
+      "theoretical_loss": 3.343762736078921,
+      "tokens_seen": 3065053184
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.185844968704863e-05,
+      "loss": 1.1506,
+      "theoretical_loss": 3.3437518833209987,
+      "tokens_seen": 3065184256
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.181832771625742e-05,
+      "loss": 1.2083,
+      "theoretical_loss": 3.343741031157083,
+      "tokens_seen": 3065315328
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.177820574546621e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.343730179587117,
+      "tokens_seen": 3065446400
+    },
+    {
+      "epoch": 0.86,
+      "objective/train/docs_used": 1027393,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1676499843597412,
+      "objective/train/theoretical_loss": 3.343719328611042,
+      "objective/train/tokens_used": 1436036576,
+      "theoretical_loss": 3.343719328611042,
+      "tokens_seen": 3065577472
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.173808377467502e-05,
+      "loss": 1.1579,
+      "theoretical_loss": 3.343719328611042,
+      "tokens_seen": 3065577472
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.169796180388381e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3437084782288005,
+      "tokens_seen": 3065708544
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.165783983309259e-05,
+      "loss": 1.2004,
+      "theoretical_loss": 3.343697628440334,
+      "tokens_seen": 3065839616
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.16177178623014e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.3436867792455853,
+      "tokens_seen": 3065970688
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.157759589151019e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.343675930644496,
+      "tokens_seen": 3066101760
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.1537473920719e-05,
+      "loss": 1.3095,
+      "theoretical_loss": 3.3436650826370085,
+      "tokens_seen": 3066232832
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.149735194992778e-05,
+      "loss": 1.172,
+      "theoretical_loss": 3.343654235223065,
+      "tokens_seen": 3066363904
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.145722997913658e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.3436433884026076,
+      "tokens_seen": 3066494976
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.141710800834537e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.343632542175578,
+      "tokens_seen": 3066626048
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.137698603755417e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.3436216965419194,
+      "tokens_seen": 3066757120
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.133686406676296e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.3436108515015732,
+      "tokens_seen": 3066888192
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.129674209597175e-05,
+      "loss": 1.1756,
+      "theoretical_loss": 3.343600007054482,
+      "tokens_seen": 3067019264
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.125662012518056e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.343589163200587,
+      "tokens_seen": 3067150336
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.121649815438934e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.343578319939832,
+      "tokens_seen": 3067281408
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.117637618359813e-05,
+      "loss": 1.1796,
+      "theoretical_loss": 3.343567477272158,
+      "tokens_seen": 3067412480
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.113625421280694e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3435566351975075,
+      "tokens_seen": 3067543552
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.109613224201573e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3435457937158235,
+      "tokens_seen": 3067674624
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.105601027122452e-05,
+      "loss": 1.1556,
+      "theoretical_loss": 3.3435349528270466,
+      "tokens_seen": 3067805696
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.101588830043332e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.343524112531121,
+      "tokens_seen": 3067936768
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.097576632964212e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3435132728279875,
+      "tokens_seen": 3068067840
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.093564435885091e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.343502433717589,
+      "tokens_seen": 3068198912
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.08955223880597e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.343491595199867,
+      "tokens_seen": 3068329984
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.08554004172685e-05,
+      "loss": 1.1733,
+      "theoretical_loss": 3.343480757274765,
+      "tokens_seen": 3068461056
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.081527844647729e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.3434699199422244,
+      "tokens_seen": 3068592128
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.077515647568609e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.343459083202188,
+      "tokens_seen": 3068723200
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.073503450489488e-05,
+      "loss": 1.173,
+      "theoretical_loss": 3.3434482470545976,
+      "tokens_seen": 3068854272
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.069491253410367e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.343437411499396,
+      "tokens_seen": 3068985344
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.065479056331248e-05,
+      "loss": 1.2438,
+      "theoretical_loss": 3.343426576536525,
+      "tokens_seen": 3069116416
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.061466859252126e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.3434157421659276,
+      "tokens_seen": 3069247488
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.057454662173006e-05,
+      "loss": 1.1587,
+      "theoretical_loss": 3.3434049083875457,
+      "tokens_seen": 3069378560
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.053442465093886e-05,
+      "loss": 1.1583,
+      "theoretical_loss": 3.3433940752013216,
+      "tokens_seen": 3069509632
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.049430268014766e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.3433832426071977,
+      "tokens_seen": 3069640704
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.045418070935644e-05,
+      "loss": 1.1993,
+      "theoretical_loss": 3.343372410605116,
+      "tokens_seen": 3069771776
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.041405873856523e-05,
+      "loss": 1.1791,
+      "theoretical_loss": 3.3433615791950198,
+      "tokens_seen": 3069902848
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.037393676777404e-05,
+      "loss": 1.154,
+      "theoretical_loss": 3.3433507483768508,
+      "tokens_seen": 3070033920
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.033381479698282e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.3433399181505514,
+      "tokens_seen": 3070164992
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.029369282619163e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.343329088516064,
+      "tokens_seen": 3070296064
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.025357085540042e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3433182594733317,
+      "tokens_seen": 3070427136
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.021344888460923e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.3433074310222954,
+      "tokens_seen": 3070558208
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.0173326913818e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.3432966031628992,
+      "tokens_seen": 3070689280
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.01332049430268e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3432857758950845,
+      "tokens_seen": 3070820352
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.00930829722356e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.343274949218794,
+      "tokens_seen": 3070951424
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.00529610014444e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.34326412313397,
+      "tokens_seen": 3071082496
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.001283903065319e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.343253297640555,
+      "tokens_seen": 3071213568
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.997271705986198e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.343242472738492,
+      "tokens_seen": 3071344640
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.993259508907077e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.3432316484277225,
+      "tokens_seen": 3071475712
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.989247311827957e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.34322082470819,
+      "tokens_seen": 3071606784
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.985235114748836e-05,
+      "loss": 1.186,
+      "theoretical_loss": 3.343210001579836,
+      "tokens_seen": 3071737856
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.981222917669717e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.343199179042603,
+      "tokens_seen": 3071868928
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.977210720590596e-05,
+      "loss": 1.2702,
+      "theoretical_loss": 3.3431883570964347,
+      "tokens_seen": 3072000000
+    },
+    {
+      "epoch": 0.86,
+      "objective/train/docs_used": 1029784,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.166704773902893,
+      "objective/train/theoretical_loss": 3.3431775357412725,
+      "objective/train/tokens_used": 1442590176,
+      "theoretical_loss": 3.3431775357412725,
+      "tokens_seen": 3072131072
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.973198523511474e-05,
+      "loss": 1.2139,
+      "theoretical_loss": 3.3431775357412725,
+      "tokens_seen": 3072131072
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.969186326432354e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.343166714977059,
+      "tokens_seen": 3072262144
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.965174129353234e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.343155894803737,
+      "tokens_seen": 3072393216
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.961161932274114e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3431450752212495,
+      "tokens_seen": 3072524288
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.957149735194992e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.343134256229538,
+      "tokens_seen": 3072655360
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.953137538115873e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.3431234378285457,
+      "tokens_seen": 3072786432
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.949125341036752e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3431126200182146,
+      "tokens_seen": 3072917504
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.94511314395763e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.3431018027984885,
+      "tokens_seen": 3073048576
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.941100946878511e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.3430909861693086,
+      "tokens_seen": 3073179648
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.93708874979939e-05,
+      "loss": 1.1719,
+      "theoretical_loss": 3.343080170130618,
+      "tokens_seen": 3073310720
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.93307655272027e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3430693546823593,
+      "tokens_seen": 3073441792
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.929064355641149e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.343058539824475,
+      "tokens_seen": 3073572864
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.925052158562029e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.343047725556908,
+      "tokens_seen": 3073703936
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.921039961482908e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.343036911879601,
+      "tokens_seen": 3073835008
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.917027764403788e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.3430260987924956,
+      "tokens_seen": 3073966080
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.913015567324667e-05,
+      "loss": 1.1474,
+      "theoretical_loss": 3.343015286295535,
+      "tokens_seen": 3074097152
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.909003370245546e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.3430044743886627,
+      "tokens_seen": 3074228224
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.904991173166427e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3429936630718204,
+      "tokens_seen": 3074359296
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.900978976087306e-05,
+      "loss": 1.1819,
+      "theoretical_loss": 3.3429828523449507,
+      "tokens_seen": 3074490368
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.896966779008184e-05,
+      "loss": 1.2761,
+      "theoretical_loss": 3.3429720422079967,
+      "tokens_seen": 3074621440
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.892954581929065e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3429612326609006,
+      "tokens_seen": 3074752512
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.888942384849944e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.3429504237036056,
+      "tokens_seen": 3074883584
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.884930187770823e-05,
+      "loss": 1.2188,
+      "theoretical_loss": 3.342939615336054,
+      "tokens_seen": 3075014656
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.880917990691703e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.3429288075581884,
+      "tokens_seen": 3075145728
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.876905793612583e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.342918000369952,
+      "tokens_seen": 3075276800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.872893596533462e-05,
+      "loss": 1.294,
+      "theoretical_loss": 3.3429071937712873,
+      "tokens_seen": 3075407872
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.86888139945434e-05,
+      "loss": 1.1572,
+      "theoretical_loss": 3.3428963877621367,
+      "tokens_seen": 3075538944
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.864869202375221e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.342885582342443,
+      "tokens_seen": 3075670016
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.8608570052961e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.342874777512149,
+      "tokens_seen": 3075801088
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.856844808216981e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3428639732711978,
+      "tokens_seen": 3075932160
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.852832611137859e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.3428531696195316,
+      "tokens_seen": 3076063232
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.848820414058738e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.3428423665570937,
+      "tokens_seen": 3076194304
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.844808216979619e-05,
+      "loss": 1.1904,
+      "theoretical_loss": 3.342831564083826,
+      "tokens_seen": 3076325376
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.840796019900497e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3428207621996724,
+      "tokens_seen": 3076456448
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.836783822821377e-05,
+      "loss": 1.2049,
+      "theoretical_loss": 3.342809960904575,
+      "tokens_seen": 3076587520
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.832771625742257e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.3427991601984766,
+      "tokens_seen": 3076718592
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.828759428663137e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.34278836008132,
+      "tokens_seen": 3076849664
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.824747231584015e-05,
+      "loss": 1.1589,
+      "theoretical_loss": 3.342777560553048,
+      "tokens_seen": 3076980736
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.820735034504894e-05,
+      "loss": 1.2656,
+      "theoretical_loss": 3.3427667616136034,
+      "tokens_seen": 3077111808
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.816722837425775e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.3427559632629293,
+      "tokens_seen": 3077242880
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.812710640346654e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3427451655009683,
+      "tokens_seen": 3077373952
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.808698443267534e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.342734368327663,
+      "tokens_seen": 3077505024
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.804686246188413e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.342723571742957,
+      "tokens_seen": 3077636096
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.800674049109292e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.3427127757467923,
+      "tokens_seen": 3077767168
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.796661852030171e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.342701980339112,
+      "tokens_seen": 3077898240
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.792649654951051e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.3426911855198593,
+      "tokens_seen": 3078029312
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.788637457871931e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.3426803912889764,
+      "tokens_seen": 3078160384
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.78462526079281e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.342669597646407,
+      "tokens_seen": 3078291456
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.78061306371369e-05,
+      "loss": 1.2049,
+      "theoretical_loss": 3.3426588045920935,
+      "tokens_seen": 3078422528
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.776600866634569e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.342648012125979,
+      "tokens_seen": 3078553600
+    },
+    {
+      "epoch": 0.87,
+      "objective/train/docs_used": 1031977,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2693638801574707,
+      "objective/train/theoretical_loss": 3.342637220248006,
+      "objective/train/tokens_used": 1449143776,
+      "theoretical_loss": 3.342637220248006,
+      "tokens_seen": 3078684672
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.772588669555448e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.342637220248006,
+      "tokens_seen": 3078684672
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.768576472476329e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3426264289581176,
+      "tokens_seen": 3078815744
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.764564275397207e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.342615638256257,
+      "tokens_seen": 3078946816
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.760552078318088e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.342604848142367,
+      "tokens_seen": 3079077888
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.756539881238967e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.3425940586163905,
+      "tokens_seen": 3079208960
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.752527684159845e-05,
+      "loss": 1.2054,
+      "theoretical_loss": 3.3425832696782702,
+      "tokens_seen": 3079340032
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.748515487080725e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.3425724813279496,
+      "tokens_seen": 3079471104
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.744503290001605e-05,
+      "loss": 1.1664,
+      "theoretical_loss": 3.3425616935653713,
+      "tokens_seen": 3079602176
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.740491092922485e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3425509063904784,
+      "tokens_seen": 3079733248
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.736478895843363e-05,
+      "loss": 1.1596,
+      "theoretical_loss": 3.342540119803213,
+      "tokens_seen": 3079864320
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.732466698764244e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.34252933380352,
+      "tokens_seen": 3079995392
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.728454501685123e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.3425185483913404,
+      "tokens_seen": 3080126464
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.724442304606002e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3425077635666183,
+      "tokens_seen": 3080257536
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.720430107526882e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3424969793292965,
+      "tokens_seen": 3080388608
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.716417910447761e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.342486195679318,
+      "tokens_seen": 3080519680
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.712405713368642e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.3424754126166256,
+      "tokens_seen": 3080650752
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.70839351628952e-05,
+      "loss": 1.1564,
+      "theoretical_loss": 3.342464630141163,
+      "tokens_seen": 3080781824
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.704381319210399e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.3424538482528723,
+      "tokens_seen": 3080912896
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.70036912213128e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.342443066951697,
+      "tokens_seen": 3081043968
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.696356925052159e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3424322862375804,
+      "tokens_seen": 3081175040
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.692344727973038e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.342421506110465,
+      "tokens_seen": 3081306112
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.688332530893917e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.3424107265702947,
+      "tokens_seen": 3081437184
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.684320333814798e-05,
+      "loss": 1.1764,
+      "theoretical_loss": 3.342399947617012,
+      "tokens_seen": 3081568256
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.680308136735677e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3423891692505596,
+      "tokens_seen": 3081699328
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.676295939656555e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.342378391470881,
+      "tokens_seen": 3081830400
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.672283742577436e-05,
+      "loss": 1.3002,
+      "theoretical_loss": 3.34236761427792,
+      "tokens_seen": 3081961472
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.668271545498315e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.342356837671619,
+      "tokens_seen": 3082092544
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.664259348419196e-05,
+      "loss": 1.2232,
+      "theoretical_loss": 3.3423460616519205,
+      "tokens_seen": 3082223616
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.660247151340074e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.342335286218769,
+      "tokens_seen": 3082354688
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.656234954260953e-05,
+      "loss": 1.169,
+      "theoretical_loss": 3.3423245113721065,
+      "tokens_seen": 3082485760
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.652222757181833e-05,
+      "loss": 1.2318,
+      "theoretical_loss": 3.342313737111877,
+      "tokens_seen": 3082616832
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.648210560102711e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.342302963438023,
+      "tokens_seen": 3082747904
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.644198363023592e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.342292190350488,
+      "tokens_seen": 3082878976
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.640186165944471e-05,
+      "loss": 1.1736,
+      "theoretical_loss": 3.3422814178492146,
+      "tokens_seen": 3083010048
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.636173968865352e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.342270645934147,
+      "tokens_seen": 3083141120
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.63216177178623e-05,
+      "loss": 1.1946,
+      "theoretical_loss": 3.3422598746052277,
+      "tokens_seen": 3083272192
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.628149574707109e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.3422491038624,
+      "tokens_seen": 3083403264
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.62413737762799e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.342238333705607,
+      "tokens_seen": 3083534336
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.620125180548869e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.342227564134792,
+      "tokens_seen": 3083665408
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.616112983469748e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.3422167951498984,
+      "tokens_seen": 3083796480
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.612100786390628e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.342206026750869,
+      "tokens_seen": 3083927552
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.608088589311508e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3421952589376476,
+      "tokens_seen": 3084058624
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.604076392232386e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.3421844917101766,
+      "tokens_seen": 3084189696
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.600064195153265e-05,
+      "loss": 1.1171,
+      "theoretical_loss": 3.3421737250684003,
+      "tokens_seen": 3084320768
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.596051998074146e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.3421629590122612,
+      "tokens_seen": 3084451840
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.592039800995025e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.3421521935417027,
+      "tokens_seen": 3084582912
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.588027603915905e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.342141428656668,
+      "tokens_seen": 3084713984
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.584015406836784e-05,
+      "loss": 1.1444,
+      "theoretical_loss": 3.342130664357101,
+      "tokens_seen": 3084845056
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.580003209757663e-05,
+      "loss": 1.158,
+      "theoretical_loss": 3.342119900642944,
+      "tokens_seen": 3084976128
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.575991012678544e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.342109137514141,
+      "tokens_seen": 3085107200
+    },
+    {
+      "epoch": 0.87,
+      "objective/train/docs_used": 1034060,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2746806144714355,
+      "objective/train/theoretical_loss": 3.342098374970635,
+      "objective/train/tokens_used": 1455697376,
+      "theoretical_loss": 3.342098374970635,
+      "tokens_seen": 3085238272
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.571978815599422e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.342098374970635,
+      "tokens_seen": 3085238272
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.567966618520302e-05,
+      "loss": 1.1937,
+      "theoretical_loss": 3.3420876130123696,
+      "tokens_seen": 3085369344
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.563954421441182e-05,
+      "loss": 1.3006,
+      "theoretical_loss": 3.3420768516392876,
+      "tokens_seen": 3085500416
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.55994222436206e-05,
+      "loss": 1.2128,
+      "theoretical_loss": 3.342066090851333,
+      "tokens_seen": 3085631488
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.55593002728294e-05,
+      "loss": 1.1508,
+      "theoretical_loss": 3.3420553306484484,
+      "tokens_seen": 3085762560
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.55191783020382e-05,
+      "loss": 1.1809,
+      "theoretical_loss": 3.342044571030578,
+      "tokens_seen": 3085893632
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.5479056331247e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.3420338119976645,
+      "tokens_seen": 3086024704
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.543893436045578e-05,
+      "loss": 1.2976,
+      "theoretical_loss": 3.342023053549651,
+      "tokens_seen": 3086155776
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.539881238966459e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.3420122956864815,
+      "tokens_seen": 3086286848
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.535869041887338e-05,
+      "loss": 1.1697,
+      "theoretical_loss": 3.342001538408099,
+      "tokens_seen": 3086417920
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.531856844808217e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3419907817144474,
+      "tokens_seen": 3086548992
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.527844647729096e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.3419800256054697,
+      "tokens_seen": 3086680064
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.523832450649976e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3419692700811092,
+      "tokens_seen": 3086811136
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.519820253570856e-05,
+      "loss": 1.1795,
+      "theoretical_loss": 3.341958515141309,
+      "tokens_seen": 3086942208
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.515808056491734e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.341947760786014,
+      "tokens_seen": 3087073280
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.511795859412615e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.3419370070151655,
+      "tokens_seen": 3087204352
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.507783662333494e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.3419262538287082,
+      "tokens_seen": 3087335424
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.503771465254373e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.3419155012265858,
+      "tokens_seen": 3087466496
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.499759268175253e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.341904749208741,
+      "tokens_seen": 3087597568
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.495747071096132e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.341893997775117,
+      "tokens_seen": 3087728640
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.491734874017013e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.341883246925658,
+      "tokens_seen": 3087859712
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.487722676937892e-05,
+      "loss": 1.1857,
+      "theoretical_loss": 3.3418724966603075,
+      "tokens_seen": 3087990784
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.48371047985877e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.341861746979008,
+      "tokens_seen": 3088121856
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.47969828277965e-05,
+      "loss": 1.1662,
+      "theoretical_loss": 3.3418509978817044,
+      "tokens_seen": 3088252928
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.47568608570053e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.3418402493683392,
+      "tokens_seen": 3088384000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.471673888621409e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.3418295014388564,
+      "tokens_seen": 3088515072
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.467661691542288e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.341818754093199,
+      "tokens_seen": 3088646144
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.463649494463169e-05,
+      "loss": 1.1782,
+      "theoretical_loss": 3.341808007331311,
+      "tokens_seen": 3088777216
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.459637297384048e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.3417972611531357,
+      "tokens_seen": 3088908288
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.455625100304926e-05,
+      "loss": 1.1653,
+      "theoretical_loss": 3.341786515558616,
+      "tokens_seen": 3089039360
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.451612903225807e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.341775770547697,
+      "tokens_seen": 3089170432
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.447600706146686e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.3417650261203207,
+      "tokens_seen": 3089301504
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.443588509067567e-05,
+      "loss": 1.1742,
+      "theoretical_loss": 3.3417542822764315,
+      "tokens_seen": 3089432576
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.439576311988445e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.3417435390159724,
+      "tokens_seen": 3089563648
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.435564114909324e-05,
+      "loss": 1.1171,
+      "theoretical_loss": 3.3417327963388876,
+      "tokens_seen": 3089694720
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.431551917830204e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.3417220542451203,
+      "tokens_seen": 3089825792
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.427539720751084e-05,
+      "loss": 1.161,
+      "theoretical_loss": 3.3417113127346143,
+      "tokens_seen": 3089956864
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.423527523671963e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.341700571807313,
+      "tokens_seen": 3090087936
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.419515326592842e-05,
+      "loss": 1.165,
+      "theoretical_loss": 3.3416898314631602,
+      "tokens_seen": 3090219008
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.415503129513723e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.341679091702099,
+      "tokens_seen": 3090350080
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.411490932434601e-05,
+      "loss": 1.1641,
+      "theoretical_loss": 3.3416683525240733,
+      "tokens_seen": 3090481152
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.40747873535548e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.341657613929027,
+      "tokens_seen": 3090612224
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.403466538276361e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3416468759169033,
+      "tokens_seen": 3090743296
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.39945434119724e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3416361384876465,
+      "tokens_seen": 3090874368
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.395442144118119e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3416254016411995,
+      "tokens_seen": 3091005440
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.391429947038999e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.3416146653775063,
+      "tokens_seen": 3091136512
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.387417749959878e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.3416039296965105,
+      "tokens_seen": 3091267584
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.383405552880758e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.341593194598156,
+      "tokens_seen": 3091398656
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.379393355801636e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.341582460082386,
+      "tokens_seen": 3091529728
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.375381158722517e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.3415717261491444,
+      "tokens_seen": 3091660800
+    },
+    {
+      "epoch": 0.87,
+      "objective/train/docs_used": 1036380,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1523369550704956,
+      "objective/train/theoretical_loss": 3.341560992798375,
+      "objective/train/tokens_used": 1462250976,
+      "theoretical_loss": 3.341560992798375,
+      "tokens_seen": 3091791872
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.371368961643396e-05,
+      "loss": 1.1494,
+      "theoretical_loss": 3.341560992798375,
+      "tokens_seen": 3091791872
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.367356764564276e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.3415502600300213,
+      "tokens_seen": 3091922944
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.363344567485155e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3415395278440276,
+      "tokens_seen": 3092054016
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.359332370406034e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.341528796240337,
+      "tokens_seen": 3092185088
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.355320173326915e-05,
+      "loss": 1.1508,
+      "theoretical_loss": 3.341518065218893,
+      "tokens_seen": 3092316160
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.351307976247793e-05,
+      "loss": 1.3099,
+      "theoretical_loss": 3.3415073347796405,
+      "tokens_seen": 3092447232
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.347295779168673e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.341496604922522,
+      "tokens_seen": 3092578304
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.343283582089553e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3414858756474817,
+      "tokens_seen": 3092709376
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.339271385010432e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3414751469544632,
+      "tokens_seen": 3092840448
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.335259187931311e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.341464418843411,
+      "tokens_seen": 3092971520
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.33124699085219e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.3414536913142676,
+      "tokens_seen": 3093102592
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.327234793773071e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.341442964366978,
+      "tokens_seen": 3093233664
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.323222596693949e-05,
+      "loss": 1.1882,
+      "theoretical_loss": 3.3414322380014854,
+      "tokens_seen": 3093364736
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.31921039961483e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.3414215122177335,
+      "tokens_seen": 3093495808
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.315198202535709e-05,
+      "loss": 1.1734,
+      "theoretical_loss": 3.3414107870156666,
+      "tokens_seen": 3093626880
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.311186005456588e-05,
+      "loss": 1.1616,
+      "theoretical_loss": 3.3414000623952282,
+      "tokens_seen": 3093757952
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.307173808377467e-05,
+      "loss": 1.1278,
+      "theoretical_loss": 3.341389338356362,
+      "tokens_seen": 3093889024
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.303161611298347e-05,
+      "loss": 1.1926,
+      "theoretical_loss": 3.3413786148990114,
+      "tokens_seen": 3094020096
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.299149414219227e-05,
+      "loss": 1.1695,
+      "theoretical_loss": 3.3413678920231216,
+      "tokens_seen": 3094151168
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.295137217140107e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.341357169728635,
+      "tokens_seen": 3094282240
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.291125020060985e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.3413464480154964,
+      "tokens_seen": 3094413312
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.287112822981865e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.341335726883649,
+      "tokens_seen": 3094544384
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.283100625902744e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3413250063330375,
+      "tokens_seen": 3094675456
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.279088428823624e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.341314286363605,
+      "tokens_seen": 3094806528
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.275076231744503e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.3413035669752955,
+      "tokens_seen": 3094937600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.271064034665384e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.3412928481680533,
+      "tokens_seen": 3095068672
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.267051837586263e-05,
+      "loss": 1.1625,
+      "theoretical_loss": 3.341282129941822,
+      "tokens_seen": 3095199744
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.263039640507141e-05,
+      "loss": 1.1247,
+      "theoretical_loss": 3.3412714122965452,
+      "tokens_seen": 3095330816
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.259027443428021e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.341260695232167,
+      "tokens_seen": 3095461888
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.255015246348901e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.341249978748632,
+      "tokens_seen": 3095592960
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.251003049269781e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.3412392628458836,
+      "tokens_seen": 3095724032
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.24699085219066e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.341228547523865,
+      "tokens_seen": 3095855104
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.242978655111539e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.3412178327825215,
+      "tokens_seen": 3095986176
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.238966458032419e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.341207118621796,
+      "tokens_seen": 3096117248
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.234954260953298e-05,
+      "loss": 1.1746,
+      "theoretical_loss": 3.341196405041633,
+      "tokens_seen": 3096248320
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.230942063874178e-05,
+      "loss": 1.1611,
+      "theoretical_loss": 3.341185692041977,
+      "tokens_seen": 3096379392
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.226929866795057e-05,
+      "loss": 1.1554,
+      "theoretical_loss": 3.3411749796227705,
+      "tokens_seen": 3096510464
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.222917669715936e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.3411642677839586,
+      "tokens_seen": 3096641536
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.218905472636816e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.341153556525485,
+      "tokens_seen": 3096772608
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.214893275557695e-05,
+      "loss": 1.2088,
+      "theoretical_loss": 3.341142845847294,
+      "tokens_seen": 3096903680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.210881078478575e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.341132135749329,
+      "tokens_seen": 3097034752
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.206868881399455e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.341121426231534,
+      "tokens_seen": 3097165824
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.202856684320334e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.3411107172938537,
+      "tokens_seen": 3097296896
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.198844487241213e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.3411000089362317,
+      "tokens_seen": 3097427968
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.194832290162093e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.341089301158612,
+      "tokens_seen": 3097559040
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.190820093082972e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3410785939609386,
+      "tokens_seen": 3097690112
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.186807896003851e-05,
+      "loss": 1.1845,
+      "theoretical_loss": 3.341067887343156,
+      "tokens_seen": 3097821184
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.182795698924732e-05,
+      "loss": 1.3024,
+      "theoretical_loss": 3.341057181305208,
+      "tokens_seen": 3097952256
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.178783501845611e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3410464758470386,
+      "tokens_seen": 3098083328
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.17477130476649e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.3410357709685914,
+      "tokens_seen": 3098214400
+    },
+    {
+      "epoch": 0.88,
+      "objective/train/docs_used": 1038401,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.168662667274475,
+      "objective/train/theoretical_loss": 3.3410250666698116,
+      "objective/train/tokens_used": 1468804576,
+      "theoretical_loss": 3.3410250666698116,
+      "tokens_seen": 3098345472
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.17075910768737e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.3410250666698116,
+      "tokens_seen": 3098345472
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.166746910608249e-05,
+      "loss": 1.1731,
+      "theoretical_loss": 3.3410143629506424,
+      "tokens_seen": 3098476544
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.162734713529128e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.3410036598110278,
+      "tokens_seen": 3098607616
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.158722516450009e-05,
+      "loss": 1.1968,
+      "theoretical_loss": 3.3409929572509127,
+      "tokens_seen": 3098738688
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.154710319370888e-05,
+      "loss": 1.1868,
+      "theoretical_loss": 3.340982255270241,
+      "tokens_seen": 3098869760
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.150698122291767e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.340971553868956,
+      "tokens_seen": 3099000832
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.146685925212647e-05,
+      "loss": 1.2158,
+      "theoretical_loss": 3.3409608530470027,
+      "tokens_seen": 3099131904
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.142673728133526e-05,
+      "loss": 1.2252,
+      "theoretical_loss": 3.340950152804325,
+      "tokens_seen": 3099262976
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.138661531054405e-05,
+      "loss": 1.2128,
+      "theoretical_loss": 3.340939453140867,
+      "tokens_seen": 3099394048
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.134649333975284e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.340928754056573,
+      "tokens_seen": 3099525120
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.130637136896165e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.340918055551387,
+      "tokens_seen": 3099656192
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.126624939817044e-05,
+      "loss": 1.1588,
+      "theoretical_loss": 3.3409073576252535,
+      "tokens_seen": 3099787264
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.122612742737924e-05,
+      "loss": 1.1526,
+      "theoretical_loss": 3.3408966602781156,
+      "tokens_seen": 3099918336
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.118600545658803e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.340885963509919,
+      "tokens_seen": 3100049408
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.114588348579682e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.340875267320607,
+      "tokens_seen": 3100180480
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.110576151500561e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3408645717101235,
+      "tokens_seen": 3100311552
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.106563954421442e-05,
+      "loss": 1.2075,
+      "theoretical_loss": 3.3408538766784135,
+      "tokens_seen": 3100442624
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.1025517573423206e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.340843182225421,
+      "tokens_seen": 3100573696
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0985395602632006e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.34083248835109,
+      "tokens_seen": 3100704768
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.09452736318408e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.3408217950553647,
+      "tokens_seen": 3100835840
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.090515166104959e-05,
+      "loss": 1.1997,
+      "theoretical_loss": 3.34081110233819,
+      "tokens_seen": 3100966912
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.086502969025839e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.340800410199509,
+      "tokens_seen": 3101097984
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.082490771946718e-05,
+      "loss": 1.1544,
+      "theoretical_loss": 3.3407897186392668,
+      "tokens_seen": 3101229056
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0784785748675976e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3407790276574074,
+      "tokens_seen": 3101360128
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.074466377788477e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.340768337253875,
+      "tokens_seen": 3101491200
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.070454180709357e-05,
+      "loss": 1.2007,
+      "theoretical_loss": 3.340757647428614,
+      "tokens_seen": 3101622272
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.066441983630236e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.3407469581815685,
+      "tokens_seen": 3101753344
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.062429786551116e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3407362695126834,
+      "tokens_seen": 3101884416
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.058417589471995e-05,
+      "loss": 1.1413,
+      "theoretical_loss": 3.3407255814219026,
+      "tokens_seen": 3102015488
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0544053923928746e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.34071489390917,
+      "tokens_seen": 3102146560
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.050393195313754e-05,
+      "loss": 1.1834,
+      "theoretical_loss": 3.34070420697443,
+      "tokens_seen": 3102277632
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.046380998234633e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.3406935206176276,
+      "tokens_seen": 3102408704
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.042368801155513e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.3406828348387068,
+      "tokens_seen": 3102539776
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0383566040763924e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3406721496376117,
+      "tokens_seen": 3102670848
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.034344406997272e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.340661465014287,
+      "tokens_seen": 3102801920
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.030332209918151e-05,
+      "loss": 1.211,
+      "theoretical_loss": 3.3406507809686765,
+      "tokens_seen": 3102932992
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.026320012839031e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.3406400975007253,
+      "tokens_seen": 3103064064
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.02230781575991e-05,
+      "loss": 1.0959,
+      "theoretical_loss": 3.340629414610377,
+      "tokens_seen": 3103195136
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.01829561868079e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.340618732297577,
+      "tokens_seen": 3103326208
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0142834216016694e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.3406080505622686,
+      "tokens_seen": 3103457280
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.010271224522549e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.3405973694043967,
+      "tokens_seen": 3103588352
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.006259027443428e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.3405866888239055,
+      "tokens_seen": 3103719424
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.002246830364308e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.34057600882074,
+      "tokens_seen": 3103850496
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.998234633285187e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.340565329394844,
+      "tokens_seen": 3103981568
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9942224362060665e-05,
+      "loss": 1.3158,
+      "theoretical_loss": 3.340554650546162,
+      "tokens_seen": 3104112640
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9902102391269464e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.340543972274639,
+      "tokens_seen": 3104243712
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.986198042047825e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.340533294580218,
+      "tokens_seen": 3104374784
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.982185844968705e-05,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3405226174628453,
+      "tokens_seen": 3104505856
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.978173647889584e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.340511940922464,
+      "tokens_seen": 3104636928
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.974161450810464e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.3405012649590193,
+      "tokens_seen": 3104768000
+    },
+    {
+      "epoch": 0.88,
+      "objective/train/docs_used": 1040688,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2770309448242188,
+      "objective/train/theoretical_loss": 3.3404905895724553,
+      "objective/train/tokens_used": 1475358176,
+      "theoretical_loss": 3.3404905895724553,
+      "tokens_seen": 3104899072
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9701492537313435e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.3404905895724553,
+      "tokens_seen": 3104899072
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9661370566522234e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.3404799147627164,
+      "tokens_seen": 3105030144
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.962124859573102e-05,
+      "loss": 1.1876,
+      "theoretical_loss": 3.3404692405297474,
+      "tokens_seen": 3105161216
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.958112662493982e-05,
+      "loss": 1.1455,
+      "theoretical_loss": 3.340458566873493,
+      "tokens_seen": 3105292288
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.954100465414861e-05,
+      "loss": 1.154,
+      "theoretical_loss": 3.340447893793897,
+      "tokens_seen": 3105423360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9500882683357405e-05,
+      "loss": 1.1909,
+      "theoretical_loss": 3.3404372212909044,
+      "tokens_seen": 3105554432
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9460760712566205e-05,
+      "loss": 1.1738,
+      "theoretical_loss": 3.340426549364459,
+      "tokens_seen": 3105685504
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9420638741775e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3404158780145066,
+      "tokens_seen": 3105816576
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.938051677098379e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.3404052072409907,
+      "tokens_seen": 3105947648
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.934039480019258e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3403945370438564,
+      "tokens_seen": 3106078720
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.930027282940138e-05,
+      "loss": 1.1705,
+      "theoretical_loss": 3.340383867423048,
+      "tokens_seen": 3106209792
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9260150858610175e-05,
+      "loss": 1.3021,
+      "theoretical_loss": 3.34037319837851,
+      "tokens_seen": 3106340864
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9220028887818975e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.340362529910187,
+      "tokens_seen": 3106471936
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.917990691702777e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.3403518620180237,
+      "tokens_seen": 3106603008
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.913978494623656e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3403411947019643,
+      "tokens_seen": 3106734080
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.909966297544535e-05,
+      "loss": 1.1761,
+      "theoretical_loss": 3.340330527961954,
+      "tokens_seen": 3106865152
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9059541004654146e-05,
+      "loss": 1.182,
+      "theoretical_loss": 3.340319861797937,
+      "tokens_seen": 3106996224
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.9019419033862945e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.3403091962098577,
+      "tokens_seen": 3107127296
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.897929706307174e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.340298531197661,
+      "tokens_seen": 3107258368
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.893917509228054e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.340287866761292,
+      "tokens_seen": 3107389440
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8899053121489324e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3402772029006944,
+      "tokens_seen": 3107520512
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.885893115069812e-05,
+      "loss": 1.1663,
+      "theoretical_loss": 3.340266539615813,
+      "tokens_seen": 3107651584
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8818809179906916e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3402558769065935,
+      "tokens_seen": 3107782656
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8778687209115716e-05,
+      "loss": 1.1591,
+      "theoretical_loss": 3.340245214772979,
+      "tokens_seen": 3107913728
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.873856523832451e-05,
+      "loss": 1.1412,
+      "theoretical_loss": 3.340234553214915,
+      "tokens_seen": 3108044800
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.869844326753331e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.340223892232346,
+      "tokens_seen": 3108175872
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8658321296742094e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.3402132318252167,
+      "tokens_seen": 3108306944
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.861819932595089e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.340202571993472,
+      "tokens_seen": 3108438016
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8578077355159686e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.3401919127370565,
+      "tokens_seen": 3108569088
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.853795538436848e-05,
+      "loss": 1.1041,
+      "theoretical_loss": 3.3401812540559144,
+      "tokens_seen": 3108700160
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.849783341357728e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3401705959499908,
+      "tokens_seen": 3108831232
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.845771144278607e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.34015993841923,
+      "tokens_seen": 3108962304
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8417589471994864e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.340149281463578,
+      "tokens_seen": 3109093376
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.837746750120366e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.340138625082978,
+      "tokens_seen": 3109224448
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8337345530412456e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3401279692773755,
+      "tokens_seen": 3109355520
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.829722355962125e-05,
+      "loss": 1.1926,
+      "theoretical_loss": 3.340117314046715,
+      "tokens_seen": 3109486592
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.825710158883005e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.340106659390941,
+      "tokens_seen": 3109617664
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.821697961803884e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.340096005309999,
+      "tokens_seen": 3109748736
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8176857647247634e-05,
+      "loss": 1.209,
+      "theoretical_loss": 3.3400853518038334,
+      "tokens_seen": 3109879808
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.813673567645643e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.3400746988723884,
+      "tokens_seen": 3110010880
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.809661370566522e-05,
+      "loss": 1.3185,
+      "theoretical_loss": 3.3400640465156095,
+      "tokens_seen": 3110141952
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.805649173487402e-05,
+      "loss": 1.1731,
+      "theoretical_loss": 3.3400533947334416,
+      "tokens_seen": 3110273024
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.801636976408281e-05,
+      "loss": 1.1803,
+      "theoretical_loss": 3.340042743525829,
+      "tokens_seen": 3110404096
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.797624779329161e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.3400320928927165,
+      "tokens_seen": 3110535168
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.79361258225004e-05,
+      "loss": 1.1964,
+      "theoretical_loss": 3.3400214428340487,
+      "tokens_seen": 3110666240
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.78960038517092e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.340010793349771,
+      "tokens_seen": 3110797312
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.785588188091799e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.3400001444398284,
+      "tokens_seen": 3110928384
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.781575991012679e-05,
+      "loss": 1.26,
+      "theoretical_loss": 3.339989496104165,
+      "tokens_seen": 3111059456
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.777563793933558e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.339978848342726,
+      "tokens_seen": 3111190528
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.773551596854438e-05,
+      "loss": 1.1907,
+      "theoretical_loss": 3.339968201155456,
+      "tokens_seen": 3111321600
+    },
+    {
+      "epoch": 0.89,
+      "objective/train/docs_used": 1042979,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.291170597076416,
+      "objective/train/theoretical_loss": 3.3399575545423,
+      "objective/train/tokens_used": 1481911776,
+      "theoretical_loss": 3.3399575545423,
+      "tokens_seen": 3111452672
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.769539399775317e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3399575545423,
+      "tokens_seen": 3111452672
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.765527202696196e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3399469085032036,
+      "tokens_seen": 3111583744
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.761515005617076e-05,
+      "loss": 1.3212,
+      "theoretical_loss": 3.3399362630381106,
+      "tokens_seen": 3111714816
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.757502808537955e-05,
+      "loss": 1.1131,
+      "theoretical_loss": 3.339925618146966,
+      "tokens_seen": 3111845888
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.753490611458835e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.339914973829715,
+      "tokens_seen": 3111976960
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7494784143797145e-05,
+      "loss": 1.3255,
+      "theoretical_loss": 3.339904330086303,
+      "tokens_seen": 3112108032
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.745466217300594e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3398936869166738,
+      "tokens_seen": 3112239104
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.741454020221473e-05,
+      "loss": 1.1694,
+      "theoretical_loss": 3.3398830443207723,
+      "tokens_seen": 3112370176
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.737441823142353e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.339872402298545,
+      "tokens_seen": 3112501248
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.733429626063232e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.339861760849935,
+      "tokens_seen": 3112632320
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.729417428984112e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.339851119974888,
+      "tokens_seen": 3112763392
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7254052319049915e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3398404796733496,
+      "tokens_seen": 3112894464
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7213930348258714e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.3398298399452635,
+      "tokens_seen": 3113025536
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.71738083774675e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.3398192007905756,
+      "tokens_seen": 3113156608
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.713368640667629e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.3398085622092304,
+      "tokens_seen": 3113287680
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.709356443588509e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.339797924201173,
+      "tokens_seen": 3113418752
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7053442465093885e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.339787286766348,
+      "tokens_seen": 3113549824
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7013320494302685e-05,
+      "loss": 1.2132,
+      "theoretical_loss": 3.339776649904701,
+      "tokens_seen": 3113680896
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.697319852351147e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.339766013616176,
+      "tokens_seen": 3113811968
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.693307655272027e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.33975537790072,
+      "tokens_seen": 3113943040
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.689295458192906e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.3397447427582754,
+      "tokens_seen": 3114074112
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.685283261113786e-05,
+      "loss": 1.1545,
+      "theoretical_loss": 3.339734108188789,
+      "tokens_seen": 3114205184
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6812710640346655e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.3397234741922057,
+      "tokens_seen": 3114336256
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6772588669555455e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.3397128407684695,
+      "tokens_seen": 3114467328
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.673246669876425e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.3397022079175267,
+      "tokens_seen": 3114598400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6692344727973034e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.3396915756393213,
+      "tokens_seen": 3114729472
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.665222275718183e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.339680943933799,
+      "tokens_seen": 3114860544
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6612100786390626e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.339670312800904,
+      "tokens_seen": 3114991616
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6571978815599425e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.339659682240583,
+      "tokens_seen": 3115122688
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.653185684480822e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.3396490522527795,
+      "tokens_seen": 3115253760
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.649173487401702e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.3396384228374396,
+      "tokens_seen": 3115384832
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6451612903225804e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3396277939945076,
+      "tokens_seen": 3115515904
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.64114909324346e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.339617165723929,
+      "tokens_seen": 3115646976
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6371368961643396e-05,
+      "loss": 1.1261,
+      "theoretical_loss": 3.3396065380256488,
+      "tokens_seen": 3115778048
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6331246990852196e-05,
+      "loss": 1.1469,
+      "theoretical_loss": 3.339595910899612,
+      "tokens_seen": 3115909120
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.629112502006099e-05,
+      "loss": 1.1892,
+      "theoretical_loss": 3.339585284345764,
+      "tokens_seen": 3116040192
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.625100304926979e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3395746583640498,
+      "tokens_seen": 3116171264
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6210881078478574e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.3395640329544145,
+      "tokens_seen": 3116302336
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6170759107687367e-05,
+      "loss": 1.2022,
+      "theoretical_loss": 3.339553408116803,
+      "tokens_seen": 3116433408
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6130637136896166e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.3395427838511615,
+      "tokens_seen": 3116564480
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.609051516610496e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.3395321601574337,
+      "tokens_seen": 3116695552
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.605039319531376e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3395215370355658,
+      "tokens_seen": 3116826624
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.601027122452255e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.339510914485502,
+      "tokens_seen": 3116957696
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5970149253731344e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.339500292507189,
+      "tokens_seen": 3117088768
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5930027282940137e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.33948967110057,
+      "tokens_seen": 3117219840
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5889905312148936e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.3394790502655924,
+      "tokens_seen": 3117350912
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.584978334135773e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.3394684300021997,
+      "tokens_seen": 3117481984
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.580966137056653e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.3394578103103374,
+      "tokens_seen": 3117613056
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.576953939977532e-05,
+      "loss": 1.1593,
+      "theoretical_loss": 3.3394471911899517,
+      "tokens_seen": 3117744128
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.572941742898411e-05,
+      "loss": 1.1309,
+      "theoretical_loss": 3.3394365726409867,
+      "tokens_seen": 3117875200
+    },
+    {
+      "epoch": 0.89,
+      "objective/train/docs_used": 1045203,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1844286918640137,
+      "objective/train/theoretical_loss": 3.3394259546633878,
+      "objective/train/tokens_used": 1488465376,
+      "theoretical_loss": 3.3394259546633878,
+      "tokens_seen": 3118006272
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.568929545819291e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.3394259546633878,
+      "tokens_seen": 3118006272
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.56491734874017e-05,
+      "loss": 1.2169,
+      "theoretical_loss": 3.3394153372571007,
+      "tokens_seen": 3118137344
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.56090515166105e-05,
+      "loss": 1.3204,
+      "theoretical_loss": 3.3394047204220705,
+      "tokens_seen": 3118268416
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.556892954581929e-05,
+      "loss": 1.1793,
+      "theoretical_loss": 3.3393941041582424,
+      "tokens_seen": 3118399488
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.552880757502809e-05,
+      "loss": 1.2018,
+      "theoretical_loss": 3.339383488465562,
+      "tokens_seen": 3118530560
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.548868560423688e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.3393728733439736,
+      "tokens_seen": 3118661632
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.544856363344568e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.3393622587934235,
+      "tokens_seen": 3118792704
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.540844166265447e-05,
+      "loss": 1.18,
+      "theoretical_loss": 3.3393516448138563,
+      "tokens_seen": 3118923776
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.536831969186327e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.3393410314052177,
+      "tokens_seen": 3119054848
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.532819772107206e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.339330418567453,
+      "tokens_seen": 3119185920
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5288075750280855e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.339319806300507,
+      "tokens_seen": 3119316992
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.524795377948965e-05,
+      "loss": 1.1799,
+      "theoretical_loss": 3.3393091946043256,
+      "tokens_seen": 3119448064
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.520783180869844e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.339298583478854,
+      "tokens_seen": 3119579136
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.516770983790724e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3392879729240375,
+      "tokens_seen": 3119710208
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.512758786711603e-05,
+      "loss": 1.1573,
+      "theoretical_loss": 3.3392773629398214,
+      "tokens_seen": 3119841280
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.508746589632483e-05,
+      "loss": 1.273,
+      "theoretical_loss": 3.3392667535261507,
+      "tokens_seen": 3119972352
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5047343925533625e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.339256144682971,
+      "tokens_seen": 3120103424
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.500722195474242e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.3392455364102283,
+      "tokens_seen": 3120234496
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.496709998395121e-05,
+      "loss": 1.1768,
+      "theoretical_loss": 3.339234928707867,
+      "tokens_seen": 3120365568
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.492697801316001e-05,
+      "loss": 1.2956,
+      "theoretical_loss": 3.3392243215758333,
+      "tokens_seen": 3120496640
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.48868560423688e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.3392137150140715,
+      "tokens_seen": 3120627712
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.48467340715776e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3392031090225283,
+      "tokens_seen": 3120758784
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4806612100786395e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.339192503601148,
+      "tokens_seen": 3120889856
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.476649012999518e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3391818987498767,
+      "tokens_seen": 3121020928
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.472636815920398e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3391712944686596,
+      "tokens_seen": 3121152000
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.468624618841277e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.339160690757442,
+      "tokens_seen": 3121283072
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.464612421762157e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.339150087616169,
+      "tokens_seen": 3121414144
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4606002246830365e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.339139485044787,
+      "tokens_seen": 3121545216
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4565880276039165e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.339128883043241,
+      "tokens_seen": 3121676288
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.452575830524795e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.339118281611476,
+      "tokens_seen": 3121807360
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.448563633445675e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3391076807494375,
+      "tokens_seen": 3121938432
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.444551436366554e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.339097080457072,
+      "tokens_seen": 3122069504
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.440539239287434e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.3390864807343235,
+      "tokens_seen": 3122200576
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4365270422083135e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.339075881581139,
+      "tokens_seen": 3122331648
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.432514845129193e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.3390652829974625,
+      "tokens_seen": 3122462720
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.428502648050072e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.3390546849832408,
+      "tokens_seen": 3122593792
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4244904509709514e-05,
+      "loss": 1.1942,
+      "theoretical_loss": 3.339044087538418,
+      "tokens_seen": 3122724864
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.420478253891831e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.339033490662941,
+      "tokens_seen": 3122855936
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4164660568127106e-05,
+      "loss": 1.1864,
+      "theoretical_loss": 3.339022894356755,
+      "tokens_seen": 3122987008
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4124538597335905e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3390122986198048,
+      "tokens_seen": 3123118080
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.40844166265447e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.3390017034520363,
+      "tokens_seen": 3123249152
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.404429465575349e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.3389911088533952,
+      "tokens_seen": 3123380224
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4004172684962284e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.3389805148238274,
+      "tokens_seen": 3123511296
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.396405071417108e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.3389699213632777,
+      "tokens_seen": 3123642368
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3923928743379876e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.3389593284716916,
+      "tokens_seen": 3123773440
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3883806772588675e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.3389487361490158,
+      "tokens_seen": 3123904512
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.384368480179747e-05,
+      "loss": 1.1631,
+      "theoretical_loss": 3.3389381443951947,
+      "tokens_seen": 3124035584
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3803562831006254e-05,
+      "loss": 1.2703,
+      "theoretical_loss": 3.3389275532101745,
+      "tokens_seen": 3124166656
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3763440860215054e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3389169625939004,
+      "tokens_seen": 3124297728
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3723318889423846e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3389063725463184,
+      "tokens_seen": 3124428800
+    },
+    {
+      "epoch": 0.89,
+      "objective/train/docs_used": 1047326,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.186725378036499,
+      "objective/train/theoretical_loss": 3.338895783067374,
+      "objective/train/tokens_used": 1495018976,
+      "theoretical_loss": 3.338895783067374,
+      "tokens_seen": 3124559872
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3683196918632646e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.338895783067374,
+      "tokens_seen": 3124559872
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.364307494784144e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3388851941570126,
+      "tokens_seen": 3124690944
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.360295297705024e-05,
+      "loss": 1.1689,
+      "theoretical_loss": 3.33887460581518,
+      "tokens_seen": 3124822016
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3562831006259024e-05,
+      "loss": 1.3103,
+      "theoretical_loss": 3.3388640180418214,
+      "tokens_seen": 3124953088
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3522709035467824e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.338853430836884,
+      "tokens_seen": 3125084160
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3482587064676617e-05,
+      "loss": 1.2083,
+      "theoretical_loss": 3.338842844200311,
+      "tokens_seen": 3125215232
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3442465093885416e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.33883225813205,
+      "tokens_seen": 3125346304
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.340234312309421e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.3388216726320454,
+      "tokens_seen": 3125477376
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3362221152303e-05,
+      "loss": 1.1945,
+      "theoretical_loss": 3.338811087700244,
+      "tokens_seen": 3125608448
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3322099181511794e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.338800503336591,
+      "tokens_seen": 3125739520
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.328197721072059e-05,
+      "loss": 1.151,
+      "theoretical_loss": 3.3387899195410315,
+      "tokens_seen": 3125870592
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.324185523992939e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.338779336313512,
+      "tokens_seen": 3126001664
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.320173326913818e-05,
+      "loss": 1.1869,
+      "theoretical_loss": 3.338768753653978,
+      "tokens_seen": 3126132736
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.316161129834698e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.338758171562375,
+      "tokens_seen": 3126263808
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.312148932755577e-05,
+      "loss": 1.1724,
+      "theoretical_loss": 3.3387475900386487,
+      "tokens_seen": 3126394880
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3081367356764564e-05,
+      "loss": 1.2024,
+      "theoretical_loss": 3.338737009082745,
+      "tokens_seen": 3126525952
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.304124538597336e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.3387264286946094,
+      "tokens_seen": 3126657024
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.300112341518216e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.3387158488741884,
+      "tokens_seen": 3126788096
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.296100144439095e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.3387052696214266,
+      "tokens_seen": 3126919168
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.292087947359974e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3386946909362702,
+      "tokens_seen": 3127050240
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.288075750280854e-05,
+      "loss": 1.1939,
+      "theoretical_loss": 3.3386841128186653,
+      "tokens_seen": 3127181312
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.284063553201733e-05,
+      "loss": 1.1488,
+      "theoretical_loss": 3.3386735352685575,
+      "tokens_seen": 3127312384
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.280051356122613e-05,
+      "loss": 1.2194,
+      "theoretical_loss": 3.338662958285892,
+      "tokens_seen": 3127443456
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.276039159043492e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3386523818706157,
+      "tokens_seen": 3127574528
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.272026961964372e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.338641806022673,
+      "tokens_seen": 3127705600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.268014764885251e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.338631230742011,
+      "tokens_seen": 3127836672
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.264002567806131e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.338620656028575,
+      "tokens_seen": 3127967744
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.25999037072701e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3386100818823103,
+      "tokens_seen": 3128098816
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.25597817364789e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.3385995083031634,
+      "tokens_seen": 3128229888
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.251965976568769e-05,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3385889352910803,
+      "tokens_seen": 3128360960
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.247953779489649e-05,
+      "loss": 1.2727,
+      "theoretical_loss": 3.3385783628460057,
+      "tokens_seen": 3128492032
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.243941582410528e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3385677909678866,
+      "tokens_seen": 3128623104
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2399293853314075e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.338557219656668,
+      "tokens_seen": 3128754176
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2359171882522875e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.338546648912297,
+      "tokens_seen": 3128885248
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.231904991173166e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.338536078734718,
+      "tokens_seen": 3129016320
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.227892794094046e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.3385255091238775,
+      "tokens_seen": 3129147392
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.223880597014925e-05,
+      "loss": 1.1522,
+      "theoretical_loss": 3.338514940079721,
+      "tokens_seen": 3129278464
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.219868399935805e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.3385043716021956,
+      "tokens_seen": 3129409536
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2158562028566845e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.3384938036912457,
+      "tokens_seen": 3129540608
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2118440057775645e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.338483236346818,
+      "tokens_seen": 3129671680
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.207831808698443e-05,
+      "loss": 1.2466,
+      "theoretical_loss": 3.3384726695688585,
+      "tokens_seen": 3129802752
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.203819611619323e-05,
+      "loss": 1.2795,
+      "theoretical_loss": 3.3384621033573127,
+      "tokens_seen": 3129933824
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.199807414540202e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.3384515377121264,
+      "tokens_seen": 3130064896
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1957952174610816e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.3384409726332462,
+      "tokens_seen": 3130195968
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1917830203819615e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.3384304081206175,
+      "tokens_seen": 3130327040
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.187770823302841e-05,
+      "loss": 1.1635,
+      "theoretical_loss": 3.3384198441741866,
+      "tokens_seen": 3130458112
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.18375862622372e-05,
+      "loss": 1.169,
+      "theoretical_loss": 3.3384092807938988,
+      "tokens_seen": 3130589184
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1797464291445994e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.338398717979701,
+      "tokens_seen": 3130720256
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.175734232065479e-05,
+      "loss": 1.1472,
+      "theoretical_loss": 3.338388155731538,
+      "tokens_seen": 3130851328
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1717220349863586e-05,
+      "loss": 1.1826,
+      "theoretical_loss": 3.3383775940493567,
+      "tokens_seen": 3130982400
+    },
+    {
+      "epoch": 0.9,
+      "objective/train/docs_used": 1049585,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2419450283050537,
+      "objective/train/theoretical_loss": 3.338367032933103,
+      "objective/train/tokens_used": 1501572576,
+      "theoretical_loss": 3.338367032933103,
+      "tokens_seen": 3131113472
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1677098379072385e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.338367032933103,
+      "tokens_seen": 3131113472
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.163697640828118e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3383564723827224,
+      "tokens_seen": 3131244544
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.159685443748997e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.338345912398162,
+      "tokens_seen": 3131375616
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1556732466698764e-05,
+      "loss": 1.1964,
+      "theoretical_loss": 3.338335352979366,
+      "tokens_seen": 3131506688
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.151661049590756e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.3383247941262817,
+      "tokens_seen": 3131637760
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1476488525116356e-05,
+      "loss": 1.175,
+      "theoretical_loss": 3.3383142358388547,
+      "tokens_seen": 3131768832
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.143636655432515e-05,
+      "loss": 1.1867,
+      "theoretical_loss": 3.3383036781170317,
+      "tokens_seen": 3131899904
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.139624458353395e-05,
+      "loss": 1.1199,
+      "theoretical_loss": 3.3382931209607576,
+      "tokens_seen": 3132030976
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1356122612742734e-05,
+      "loss": 1.222,
+      "theoretical_loss": 3.3382825643699796,
+      "tokens_seen": 3132162048
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1316000641951534e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.3382720083446427,
+      "tokens_seen": 3132293120
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1275878671160326e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3382614528846934,
+      "tokens_seen": 3132424192
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1235756700369126e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.338250897990078,
+      "tokens_seen": 3132555264
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.119563472957792e-05,
+      "loss": 1.1734,
+      "theoretical_loss": 3.338240343660742,
+      "tokens_seen": 3132686336
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.115551275878672e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.338229789896632,
+      "tokens_seen": 3132817408
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1115390787995504e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.338219236697694,
+      "tokens_seen": 3132948480
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1075268817204304e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3382086840638743,
+      "tokens_seen": 3133079552
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1035146846413096e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.338198131995118,
+      "tokens_seen": 3133210624
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.099502487562189e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.3381875804913728,
+      "tokens_seen": 3133341696
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.095490290483069e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.338177029552583,
+      "tokens_seen": 3133472768
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.091478093403948e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.338166479178696,
+      "tokens_seen": 3133603840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.0874658963248274e-05,
+      "loss": 1.1852,
+      "theoretical_loss": 3.3381559293696577,
+      "tokens_seen": 3133734912
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.083453699245707e-05,
+      "loss": 1.1552,
+      "theoretical_loss": 3.338145380125414,
+      "tokens_seen": 3133865984
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.0794415021665867e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.3381348314459114,
+      "tokens_seen": 3133997056
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.075429305087466e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.338124283331095,
+      "tokens_seen": 3134128128
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.071417108008346e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.3381137357809125,
+      "tokens_seen": 3134259200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.067404910929225e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.3381031887953094,
+      "tokens_seen": 3134390272
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.0633927138501044e-05,
+      "loss": 1.1296,
+      "theoretical_loss": 3.338092642374231,
+      "tokens_seen": 3134521344
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.059380516770984e-05,
+      "loss": 1.2375,
+      "theoretical_loss": 3.338082096517625,
+      "tokens_seen": 3134652416
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.055368319691863e-05,
+      "loss": 1.1749,
+      "theoretical_loss": 3.3380715512254366,
+      "tokens_seen": 3134783488
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.051356122612743e-05,
+      "loss": 1.2758,
+      "theoretical_loss": 3.3380610064976124,
+      "tokens_seen": 3134914560
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.047343925533622e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.338050462334098,
+      "tokens_seen": 3135045632
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.043331728454502e-05,
+      "loss": 1.1654,
+      "theoretical_loss": 3.338039918734841,
+      "tokens_seen": 3135176704
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.039319531375381e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3380293756997856,
+      "tokens_seen": 3135307776
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.035307334296261e-05,
+      "loss": 1.1603,
+      "theoretical_loss": 3.3380188332288796,
+      "tokens_seen": 3135438848
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.03129513721714e-05,
+      "loss": 1.1856,
+      "theoretical_loss": 3.338008291322069,
+      "tokens_seen": 3135569920
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.02728294013802e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.337997749979299,
+      "tokens_seen": 3135700992
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.023270743058899e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.3379872092005174,
+      "tokens_seen": 3135832064
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.019258545979779e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3379766689856694,
+      "tokens_seen": 3135963136
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.015246348900658e-05,
+      "loss": 1.1697,
+      "theoretical_loss": 3.3379661293347014,
+      "tokens_seen": 3136094208
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.011234151821538e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.33795559024756,
+      "tokens_seen": 3136225280
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.007221954742417e-05,
+      "loss": 1.1744,
+      "theoretical_loss": 3.337945051724191,
+      "tokens_seen": 3136356352
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.003209757663296e-05,
+      "loss": 1.1812,
+      "theoretical_loss": 3.3379345137645413,
+      "tokens_seen": 3136487424
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.999197560584176e-05,
+      "loss": 1.1644,
+      "theoretical_loss": 3.3379239763685566,
+      "tokens_seen": 3136618496
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9951853635050555e-05,
+      "loss": 1.186,
+      "theoretical_loss": 3.337913439536184,
+      "tokens_seen": 3136749568
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.991173166425935e-05,
+      "loss": 1.178,
+      "theoretical_loss": 3.337902903267368,
+      "tokens_seen": 3136880640
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.987160969346814e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.3378923675620573,
+      "tokens_seen": 3137011712
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.983148772267694e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.3378818324201966,
+      "tokens_seen": 3137142784
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.979136575188573e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.337871297841733,
+      "tokens_seen": 3137273856
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.975124378109453e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.3378607638266122,
+      "tokens_seen": 3137404928
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9711121810303325e-05,
+      "loss": 1.1648,
+      "theoretical_loss": 3.337850230374781,
+      "tokens_seen": 3137536000
+    },
+    {
+      "epoch": 0.9,
+      "objective/train/docs_used": 1051923,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.176695466041565,
+      "objective/train/theoretical_loss": 3.337839697486186,
+      "objective/train/tokens_used": 1508126176,
+      "theoretical_loss": 3.337839697486186,
+      "tokens_seen": 3137667072
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.967099983951212e-05,
+      "loss": 1.18,
+      "theoretical_loss": 3.337839697486186,
+      "tokens_seen": 3137667072
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.963087786872091e-05,
+      "loss": 1.1814,
+      "theoretical_loss": 3.3378291651607728,
+      "tokens_seen": 3137798144
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9590755897929703e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.3378186333984883,
+      "tokens_seen": 3137929216
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.95506339271385e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3378081021992787,
+      "tokens_seen": 3138060288
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9510511956347296e-05,
+      "loss": 1.1857,
+      "theoretical_loss": 3.3377975715630903,
+      "tokens_seen": 3138191360
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9470389985556095e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.33778704148987,
+      "tokens_seen": 3138322432
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.943026801476488e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.3377765119795635,
+      "tokens_seen": 3138453504
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.939014604397368e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.337765983032117,
+      "tokens_seen": 3138584576
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9350024073182473e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.3377554546474784,
+      "tokens_seen": 3138715648
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.930990210239127e-05,
+      "loss": 1.2251,
+      "theoretical_loss": 3.3377449268255925,
+      "tokens_seen": 3138846720
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9269780131600066e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3377343995664064,
+      "tokens_seen": 3138977792
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9229658160808865e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.3377238728698666,
+      "tokens_seen": 3139108864
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.918953619001765e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.337713346735919,
+      "tokens_seen": 3139239936
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.914941421922645e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.337702821164511,
+      "tokens_seen": 3139371008
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9109292248435244e-05,
+      "loss": 1.1223,
+      "theoretical_loss": 3.337692296155588,
+      "tokens_seen": 3139502080
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9069170277644036e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.3376817717090974,
+      "tokens_seen": 3139633152
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9029048306852836e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.337671247824985,
+      "tokens_seen": 3139764224
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.898892633606163e-05,
+      "loss": 1.1809,
+      "theoretical_loss": 3.3376607245031975,
+      "tokens_seen": 3139895296
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.894880436527042e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.3376502017436813,
+      "tokens_seen": 3140026368
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8908682394479214e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.337639679546383,
+      "tokens_seen": 3140157440
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8868560423688014e-05,
+      "loss": 1.1603,
+      "theoretical_loss": 3.337629157911249,
+      "tokens_seen": 3140288512
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8828438452896806e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.337618636838226,
+      "tokens_seen": 3140419584
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8788316482105606e-05,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3376081163272606,
+      "tokens_seen": 3140550656
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.87481945113144e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.337597596378299,
+      "tokens_seen": 3140681728
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.870807254052319e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3375870769912876,
+      "tokens_seen": 3140812800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8667950569731984e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.337576558166173,
+      "tokens_seen": 3140943872
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.862782859894078e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.337566039902902,
+      "tokens_seen": 3141074944
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8587706628149576e-05,
+      "loss": 1.1858,
+      "theoretical_loss": 3.3375555222014217,
+      "tokens_seen": 3141206016
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.854758465735837e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.3375450050616773,
+      "tokens_seen": 3141337088
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.850746268656717e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.337534488483616,
+      "tokens_seen": 3141468160
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8467340715775955e-05,
+      "loss": 1.1845,
+      "theoretical_loss": 3.337523972467185,
+      "tokens_seen": 3141599232
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8427218744984754e-05,
+      "loss": 1.1824,
+      "theoretical_loss": 3.3375134570123297,
+      "tokens_seen": 3141730304
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.838709677419355e-05,
+      "loss": 1.1438,
+      "theoretical_loss": 3.3375029421189977,
+      "tokens_seen": 3141861376
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8346974803402347e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.337492427787135,
+      "tokens_seen": 3141992448
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.830685283261114e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.3374819140166885,
+      "tokens_seen": 3142123520
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.826673086181994e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.3374714008076047,
+      "tokens_seen": 3142254592
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.822660889102873e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.33746088815983,
+      "tokens_seen": 3142385664
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.818648692023752e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.337450376073311,
+      "tokens_seen": 3142516736
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.814636494944632e-05,
+      "loss": 1.1824,
+      "theoretical_loss": 3.3374398645479952,
+      "tokens_seen": 3142647808
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.810624297865511e-05,
+      "loss": 1.1896,
+      "theoretical_loss": 3.337429353583828,
+      "tokens_seen": 3142778880
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.806612100786391e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.337418843180757,
+      "tokens_seen": 3142909952
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.80259990370727e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.337408333338728,
+      "tokens_seen": 3143041024
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.79858770662815e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.337397824057689,
+      "tokens_seen": 3143172096
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.794575509549029e-05,
+      "loss": 1.2817,
+      "theoretical_loss": 3.337387315337585,
+      "tokens_seen": 3143303168
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.790563312469909e-05,
+      "loss": 1.2949,
+      "theoretical_loss": 3.3373768071783636,
+      "tokens_seen": 3143434240
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.786551115390788e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.3373662995799713,
+      "tokens_seen": 3143565312
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.782538918311668e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.3373557925423554,
+      "tokens_seen": 3143696384
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.778526721232547e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.3373452860654615,
+      "tokens_seen": 3143827456
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.774514524153427e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.3373347801492366,
+      "tokens_seen": 3143958528
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.770502327074306e-05,
+      "loss": 1.2877,
+      "theoretical_loss": 3.337324274793628,
+      "tokens_seen": 3144089600
+    },
+    {
+      "epoch": 0.91,
+      "objective/train/docs_used": 1054249,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2518975734710693,
+      "objective/train/theoretical_loss": 3.337313769998582,
+      "objective/train/tokens_used": 1514679776,
+      "theoretical_loss": 3.337313769998582,
+      "tokens_seen": 3144220672
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.766490129995185e-05,
+      "loss": 1.263,
+      "theoretical_loss": 3.337313769998582,
+      "tokens_seen": 3144220672
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.762477932916065e-05,
+      "loss": 1.1927,
+      "theoretical_loss": 3.3373032657640453,
+      "tokens_seen": 3144351744
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.758465735836944e-05,
+      "loss": 1.1965,
+      "theoretical_loss": 3.337292762089965,
+      "tokens_seen": 3144482816
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.754453538757824e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.337282258976287,
+      "tokens_seen": 3144613888
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7504413416787035e-05,
+      "loss": 1.1567,
+      "theoretical_loss": 3.3372717564229593,
+      "tokens_seen": 3144744960
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.746429144599583e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.337261254429927,
+      "tokens_seen": 3144876032
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.742416947520462e-05,
+      "loss": 1.2135,
+      "theoretical_loss": 3.3372507529971385,
+      "tokens_seen": 3145007104
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.738404750441342e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.3372402521245395,
+      "tokens_seen": 3145138176
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.734392553362221e-05,
+      "loss": 1.1715,
+      "theoretical_loss": 3.3372297518120773,
+      "tokens_seen": 3145269248
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.730380356283101e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3372192520596986,
+      "tokens_seen": 3145400320
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7263681592039805e-05,
+      "loss": 1.2873,
+      "theoretical_loss": 3.3372087528673497,
+      "tokens_seen": 3145531392
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.722355962124859e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.3371982542349783,
+      "tokens_seen": 3145662464
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.718343765045739e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.33718775616253,
+      "tokens_seen": 3145793536
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.714331567966618e-05,
+      "loss": 1.1708,
+      "theoretical_loss": 3.337177258649953,
+      "tokens_seen": 3145924608
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.710319370887498e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.337166761697193,
+      "tokens_seen": 3146055680
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7063071738083776e-05,
+      "loss": 1.1389,
+      "theoretical_loss": 3.3371562653041975,
+      "tokens_seen": 3146186752
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7022949767292575e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.3371457694709132,
+      "tokens_seen": 3146317824
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.698282779650136e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3371352741972866,
+      "tokens_seen": 3146448896
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.694270582571016e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.337124779483265,
+      "tokens_seen": 3146579968
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6902583854918953e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.3371142853287945,
+      "tokens_seen": 3146711040
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.686246188412775e-05,
+      "loss": 1.176,
+      "theoretical_loss": 3.3371037917338224,
+      "tokens_seen": 3146842112
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6822339913336546e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.337093298698296,
+      "tokens_seen": 3146973184
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.678221794254534e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.337082806222162,
+      "tokens_seen": 3147104256
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.674209597175413e-05,
+      "loss": 1.1871,
+      "theoretical_loss": 3.337072314305367,
+      "tokens_seen": 3147235328
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6701974000962924e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.3370618229478577,
+      "tokens_seen": 3147366400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6661852030171723e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.337051332149582,
+      "tokens_seen": 3147497472
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6621730059380516e-05,
+      "loss": 1.1972,
+      "theoretical_loss": 3.3370408419104853,
+      "tokens_seen": 3147628544
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6581608088589316e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.3370303522305154,
+      "tokens_seen": 3147759616
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.654148611779811e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.3370198631096195,
+      "tokens_seen": 3147890688
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.65013641470069e-05,
+      "loss": 1.1747,
+      "theoretical_loss": 3.337009374547744,
+      "tokens_seen": 3148021760
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6461242176215694e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.336998886544836,
+      "tokens_seen": 3148152832
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6421120205424494e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.336988399100842,
+      "tokens_seen": 3148283904
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6380998234633286e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.33697791221571,
+      "tokens_seen": 3148414976
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6340876263842086e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.336967425889386,
+      "tokens_seen": 3148546048
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.630075429305088e-05,
+      "loss": 1.1532,
+      "theoretical_loss": 3.336956940121817,
+      "tokens_seen": 3148677120
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6260632322259665e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.336946454912951,
+      "tokens_seen": 3148808192
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6220510351468464e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.3369359702627337,
+      "tokens_seen": 3148939264
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.618038838067726e-05,
+      "loss": 1.294,
+      "theoretical_loss": 3.3369254861711126,
+      "tokens_seen": 3149070336
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6140266409886056e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.336915002638035,
+      "tokens_seen": 3149201408
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.610014443909485e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3369045196634475,
+      "tokens_seen": 3149332480
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.606002246830365e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.3368940372472973,
+      "tokens_seen": 3149463552
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.6019900497512435e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.336883555389531,
+      "tokens_seen": 3149594624
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5979778526721234e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.336873074090096,
+      "tokens_seen": 3149725696
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.593965655593003e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3368625933489393,
+      "tokens_seen": 3149856768
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5899534585138826e-05,
+      "loss": 1.2672,
+      "theoretical_loss": 3.336852113166008,
+      "tokens_seen": 3149987840
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.585941261434762e-05,
+      "loss": 1.176,
+      "theoretical_loss": 3.3368416335412494,
+      "tokens_seen": 3150118912
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.581929064355641e-05,
+      "loss": 1.1883,
+      "theoretical_loss": 3.3368311544746097,
+      "tokens_seen": 3150249984
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5779168672765205e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.3368206759660364,
+      "tokens_seen": 3150381056
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5739046701974e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.336810198015477,
+      "tokens_seen": 3150512128
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.56989247311828e-05,
+      "loss": 1.1885,
+      "theoretical_loss": 3.336799720622878,
+      "tokens_seen": 3150643200
+    },
+    {
+      "epoch": 0.91,
+      "objective/train/docs_used": 1056328,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2356164455413818,
+      "objective/train/theoretical_loss": 3.3367892437881865,
+      "objective/train/tokens_used": 1521233376,
+      "theoretical_loss": 3.3367892437881865,
+      "tokens_seen": 3150774272
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.565880276039159e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3367892437881865,
+      "tokens_seen": 3150774272
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.561868078960039e-05,
+      "loss": 1.1803,
+      "theoretical_loss": 3.3367787675113503,
+      "tokens_seen": 3150905344
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.557855881880918e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.3367682917923154,
+      "tokens_seen": 3151036416
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5538436848017975e-05,
+      "loss": 1.1496,
+      "theoretical_loss": 3.3367578166310294,
+      "tokens_seen": 3151167488
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.549831487722677e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.3367473420274396,
+      "tokens_seen": 3151298560
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.545819290643557e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.336736867981493,
+      "tokens_seen": 3151429632
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.541807093564436e-05,
+      "loss": 1.2139,
+      "theoretical_loss": 3.336726394493137,
+      "tokens_seen": 3151560704
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.537794896485316e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.336715921562318,
+      "tokens_seen": 3151691776
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.533782699406195e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.3367054491889836,
+      "tokens_seen": 3151822848
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.529770502327074e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.3366949773730807,
+      "tokens_seen": 3151953920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.525758305247954e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.336684506114557,
+      "tokens_seen": 3152084992
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.521746108168833e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3366740354133593,
+      "tokens_seen": 3152216064
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.517733911089713e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3366635652694345,
+      "tokens_seen": 3152347136
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.513721714010592e-05,
+      "loss": 1.1968,
+      "theoretical_loss": 3.3366530956827303,
+      "tokens_seen": 3152478208
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.509709516931472e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3366426266531932,
+      "tokens_seen": 3152609280
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.505697319852351e-05,
+      "loss": 1.2922,
+      "theoretical_loss": 3.3366321581807714,
+      "tokens_seen": 3152740352
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.501685122773231e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.336621690265411,
+      "tokens_seen": 3152871424
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.49767292569411e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.33661122290706,
+      "tokens_seen": 3153002496
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.49366072861499e-05,
+      "loss": 1.1759,
+      "theoretical_loss": 3.336600756105665,
+      "tokens_seen": 3153133568
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.489648531535869e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.3365902898611735,
+      "tokens_seen": 3153264640
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4856363344567486e-05,
+      "loss": 1.1705,
+      "theoretical_loss": 3.336579824173533,
+      "tokens_seen": 3153395712
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.481624137377628e-05,
+      "loss": 1.1712,
+      "theoretical_loss": 3.3365693590426906,
+      "tokens_seen": 3153526784
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.477611940298507e-05,
+      "loss": 1.1757,
+      "theoretical_loss": 3.3365588944685927,
+      "tokens_seen": 3153657856
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.473599743219387e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.336548430451188,
+      "tokens_seen": 3153788928
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.469587546140266e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3365379669904227,
+      "tokens_seen": 3153920000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.465575349061146e-05,
+      "loss": 1.2107,
+      "theoretical_loss": 3.336527504086244,
+      "tokens_seen": 3154051072
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4615631519820256e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3365170417386003,
+      "tokens_seen": 3154182144
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.457550954902905e-05,
+      "loss": 1.1712,
+      "theoretical_loss": 3.3365065799474376,
+      "tokens_seen": 3154313216
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.453538757823784e-05,
+      "loss": 1.1551,
+      "theoretical_loss": 3.3364961187127036,
+      "tokens_seen": 3154444288
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.449526560744664e-05,
+      "loss": 1.1808,
+      "theoretical_loss": 3.336485658034346,
+      "tokens_seen": 3154575360
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.445514363665543e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.3364751979123115,
+      "tokens_seen": 3154706432
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4415021665864226e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.3364647383465473,
+      "tokens_seen": 3154837504
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4374899695073026e-05,
+      "loss": 1.1665,
+      "theoretical_loss": 3.336454279337002,
+      "tokens_seen": 3154968576
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.433477772428181e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.336443820883621,
+      "tokens_seen": 3155099648
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.429465575349061e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.3364333629863534,
+      "tokens_seen": 3155230720
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4254533782699404e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.3364229056451453,
+      "tokens_seen": 3155361792
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4214411811908203e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.336412448859944,
+      "tokens_seen": 3155492864
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4174289841116996e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.336401992630698,
+      "tokens_seen": 3155623936
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4134167870325796e-05,
+      "loss": 1.1775,
+      "theoretical_loss": 3.336391536957354,
+      "tokens_seen": 3155755008
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.409404589953458e-05,
+      "loss": 1.2925,
+      "theoretical_loss": 3.336381081839859,
+      "tokens_seen": 3155886080
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.405392392874338e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.336370627278161,
+      "tokens_seen": 3156017152
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4013801957952174e-05,
+      "loss": 1.1955,
+      "theoretical_loss": 3.336360173272207,
+      "tokens_seen": 3156148224
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3973679987160974e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.336349719821944,
+      "tokens_seen": 3156279296
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3933558016369766e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.33633926692732,
+      "tokens_seen": 3156410368
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.389343604557856e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.3363288145882826,
+      "tokens_seen": 3156541440
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.385331407478736e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.3363183628047786,
+      "tokens_seen": 3156672512
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3813192103996145e-05,
+      "loss": 1.1843,
+      "theoretical_loss": 3.3363079115767555,
+      "tokens_seen": 3156803584
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3773070133204944e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.336297460904161,
+      "tokens_seen": 3156934656
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.373294816241374e-05,
+      "loss": 1.198,
+      "theoretical_loss": 3.3362870107869425,
+      "tokens_seen": 3157065728
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3692826191622536e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3362765612250467,
+      "tokens_seen": 3157196800
+    },
+    {
+      "epoch": 0.91,
+      "objective/train/docs_used": 1058478,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2008397579193115,
+      "objective/train/theoretical_loss": 3.3362661122184223,
+      "objective/train/tokens_used": 1527786976,
+      "theoretical_loss": 3.3362661122184223,
+      "tokens_seen": 3157327872
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.365270422083133e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3362661122184223,
+      "tokens_seen": 3157327872
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.361258225004013e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.3362556637670155,
+      "tokens_seen": 3157458944
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3572460279248915e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.336245215870775,
+      "tokens_seen": 3157590016
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3572460279248915e-05,
+      "loss": 1.183,
+      "theoretical_loss": 3.336234768529647,
+      "tokens_seen": 3157721088
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3532338308457714e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.33622432174358,
+      "tokens_seen": 3157852160
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.349221633766651e-05,
+      "loss": 1.2766,
+      "theoretical_loss": 3.3362138755125206,
+      "tokens_seen": 3157983232
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.34520943668753e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.336203429836417,
+      "tokens_seen": 3158114304
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.34119723960841e-05,
+      "loss": 1.1917,
+      "theoretical_loss": 3.3361929847152165,
+      "tokens_seen": 3158245376
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.337185042529289e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.336182540148866,
+      "tokens_seen": 3158376448
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3331728454501685e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.336172096137314,
+      "tokens_seen": 3158507520
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.329160648371048e-05,
+      "loss": 1.3104,
+      "theoretical_loss": 3.3361616526805076,
+      "tokens_seen": 3158638592
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.325148451291928e-05,
+      "loss": 1.1724,
+      "theoretical_loss": 3.336151209778394,
+      "tokens_seen": 3158769664
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.321136254212807e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.3361407674309205,
+      "tokens_seen": 3158900736
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.317124057133687e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.3361303256380355,
+      "tokens_seen": 3159031808
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.313111860054566e-05,
+      "loss": 1.2845,
+      "theoretical_loss": 3.336119884399686,
+      "tokens_seen": 3159162880
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3090996629754455e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.33610944371582,
+      "tokens_seen": 3159293952
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.305087465896325e-05,
+      "loss": 1.1478,
+      "theoretical_loss": 3.3360990035863844,
+      "tokens_seen": 3159425024
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.301075268817205e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3360885640113267,
+      "tokens_seen": 3159556096
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.297063071738084e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.3360781249905953,
+      "tokens_seen": 3159687168
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.293050874658963e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3360676865241374,
+      "tokens_seen": 3159818240
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.289038677579843e-05,
+      "loss": 1.1811,
+      "theoretical_loss": 3.3360572486119002,
+      "tokens_seen": 3159949312
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.285026480500722e-05,
+      "loss": 1.147,
+      "theoretical_loss": 3.3360468112538317,
+      "tokens_seen": 3160080384
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.281014283421602e-05,
+      "loss": 1.2338,
+      "theoretical_loss": 3.3360363744498795,
+      "tokens_seen": 3160211456
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.277002086342481e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.336025938199991,
+      "tokens_seen": 3160342528
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.272989889263361e-05,
+      "loss": 1.1975,
+      "theoretical_loss": 3.3360155025041136,
+      "tokens_seen": 3160473600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.26897769218424e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.3360050673621955,
+      "tokens_seen": 3160604672
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.26496549510512e-05,
+      "loss": 1.1524,
+      "theoretical_loss": 3.3359946327741836,
+      "tokens_seen": 3160735744
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.260953298025999e-05,
+      "loss": 1.1707,
+      "theoretical_loss": 3.3359841987400265,
+      "tokens_seen": 3160866816
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.256941100946879e-05,
+      "loss": 1.2026,
+      "theoretical_loss": 3.335973765259671,
+      "tokens_seen": 3160997888
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.252928903867758e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.335963332333065,
+      "tokens_seen": 3161128960
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.248916706788637e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.3359528999601564,
+      "tokens_seen": 3161260032
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.244904509709517e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.3359424681408925,
+      "tokens_seen": 3161391104
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.2408923126303965e-05,
+      "loss": 1.1631,
+      "theoretical_loss": 3.3359320368752208,
+      "tokens_seen": 3161522176
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.236880115551276e-05,
+      "loss": 1.2158,
+      "theoretical_loss": 3.3359216061630894,
+      "tokens_seen": 3161653248
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.232867918472155e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.335911176004446,
+      "tokens_seen": 3161784320
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.228855721393035e-05,
+      "loss": 1.1727,
+      "theoretical_loss": 3.3359007463992385,
+      "tokens_seen": 3161915392
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.224843524313914e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.3358903173474137,
+      "tokens_seen": 3162046464
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.220831327234794e-05,
+      "loss": 1.1731,
+      "theoretical_loss": 3.33587988884892,
+      "tokens_seen": 3162177536
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.2168191301556736e-05,
+      "loss": 1.2654,
+      "theoretical_loss": 3.335869460903705,
+      "tokens_seen": 3162308608
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.212806933076553e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.335859033511716,
+      "tokens_seen": 3162439680
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.208794735997432e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3358486066729016,
+      "tokens_seen": 3162570752
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.2047825389183114e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.335838180387209,
+      "tokens_seen": 3162701824
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.200770341839191e-05,
+      "loss": 1.1621,
+      "theoretical_loss": 3.335827754654586,
+      "tokens_seen": 3162832896
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1967581447600706e-05,
+      "loss": 1.1766,
+      "theoretical_loss": 3.3358173294749798,
+      "tokens_seen": 3162963968
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1927459476809506e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.3358069048483387,
+      "tokens_seen": 3163095040
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.188733750601829e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.3357964807746106,
+      "tokens_seen": 3163226112
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.184721553522709e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.335786057253743,
+      "tokens_seen": 3163357184
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1807093564435884e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.3357756342856844,
+      "tokens_seen": 3163488256
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1766971593644683e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.335765211870381,
+      "tokens_seen": 3163619328
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1726849622853476e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.335754790007782,
+      "tokens_seen": 3163750400
+    },
+    {
+      "epoch": 0.92,
+      "objective/train/docs_used": 1060550,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.273166537284851,
+      "objective/train/theoretical_loss": 3.3357443686978345,
+      "objective/train/tokens_used": 1534340576,
+      "theoretical_loss": 3.3357443686978345,
+      "tokens_seen": 3163881472
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1686727652062276e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.3357443686978345,
+      "tokens_seen": 3163881472
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.164660568127106e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.3357339479404864,
+      "tokens_seen": 3164012544
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.160648371047986e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.335723527735686,
+      "tokens_seen": 3164143616
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1566361739688654e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.3357131080833806,
+      "tokens_seen": 3164274688
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.152623976889745e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.3357026889835177,
+      "tokens_seen": 3164405760
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1486117798106246e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.335692270436046,
+      "tokens_seen": 3164536832
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.144599582731504e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.335681852440913,
+      "tokens_seen": 3164667904
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.140587385652383e-05,
+      "loss": 1.1906,
+      "theoretical_loss": 3.335671434998066,
+      "tokens_seen": 3164798976
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1365751885732624e-05,
+      "loss": 1.1631,
+      "theoretical_loss": 3.3356610181074537,
+      "tokens_seen": 3164930048
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1325629914941424e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.3356506017690233,
+      "tokens_seen": 3165061120
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.128550794415022e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.335640185982723,
+      "tokens_seen": 3165192192
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1245385973359016e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.335629770748501,
+      "tokens_seen": 3165323264
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.120526400256781e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.3356193560663043,
+      "tokens_seen": 3165454336
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.11651420317766e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3356089419360817,
+      "tokens_seen": 3165585408
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1125020060985395e-05,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3355985283577803,
+      "tokens_seen": 3165716480
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.108489809019419e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.3355881153313485,
+      "tokens_seen": 3165847552
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.104477611940299e-05,
+      "loss": 1.1675,
+      "theoretical_loss": 3.335577702856734,
+      "tokens_seen": 3165978624
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.100465414861178e-05,
+      "loss": 1.1742,
+      "theoretical_loss": 3.3355672909338847,
+      "tokens_seen": 3166109696
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.096453217782058e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.3355568795627484,
+      "tokens_seen": 3166240768
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0924410207029365e-05,
+      "loss": 1.2611,
+      "theoretical_loss": 3.3355464687432734,
+      "tokens_seen": 3166371840
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0884288236238165e-05,
+      "loss": 1.1883,
+      "theoretical_loss": 3.335536058475408,
+      "tokens_seen": 3166502912
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.084416626544696e-05,
+      "loss": 1.2942,
+      "theoretical_loss": 3.335525648759099,
+      "tokens_seen": 3166633984
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.080404429465576e-05,
+      "loss": 1.1934,
+      "theoretical_loss": 3.335515239594295,
+      "tokens_seen": 3166765056
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.076392232386455e-05,
+      "loss": 1.2767,
+      "theoretical_loss": 3.335504830980944,
+      "tokens_seen": 3166896128
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.072380035307335e-05,
+      "loss": 1.1914,
+      "theoretical_loss": 3.335494422918994,
+      "tokens_seen": 3167027200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0683678382282135e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.335484015408393,
+      "tokens_seen": 3167158272
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0643556411490935e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.3354736084490884,
+      "tokens_seen": 3167289344
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.060343444069973e-05,
+      "loss": 1.1946,
+      "theoretical_loss": 3.3354632020410286,
+      "tokens_seen": 3167420416
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.056331246990852e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.335452796184162,
+      "tokens_seen": 3167551488
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.052319049911732e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.335442390878436,
+      "tokens_seen": 3167682560
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.048306852832611e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.3354319861237984,
+      "tokens_seen": 3167813632
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0442946557534905e-05,
+      "loss": 1.1871,
+      "theoretical_loss": 3.335421581920198,
+      "tokens_seen": 3167944704
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.04028245867437e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.3354111782675826,
+      "tokens_seen": 3168075776
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.03627026159525e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.3354007751659,
+      "tokens_seen": 3168206848
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.032258064516129e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.335390372615098,
+      "tokens_seen": 3168337920
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.028245867437009e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.335379970615125,
+      "tokens_seen": 3168468992
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.024233670357888e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.335369569165929,
+      "tokens_seen": 3168600064
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0202214732787675e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.335359168267458,
+      "tokens_seen": 3168731136
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.016209276199647e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.3353487679196605,
+      "tokens_seen": 3168862208
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.012197079120526e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3353383681224837,
+      "tokens_seen": 3168993280
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.008184882041406e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.335327968875877,
+      "tokens_seen": 3169124352
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.004172684962285e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.335317570179787,
+      "tokens_seen": 3169255424
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.000160487883165e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.3353071720341623,
+      "tokens_seen": 3169386496
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.996148290804044e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.335296774438951,
+      "tokens_seen": 3169517568
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.992136093724924e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3352863773941017,
+      "tokens_seen": 3169648640
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.988123896645803e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.335275980899562,
+      "tokens_seen": 3169779712
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.984111699566683e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.33526558495528,
+      "tokens_seen": 3169910784
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.980099502487562e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.335255189561204,
+      "tokens_seen": 3170041856
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.976087305408442e-05,
+      "loss": 1.204,
+      "theoretical_loss": 3.3352447947172825,
+      "tokens_seen": 3170172928
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.972075108329321e-05,
+      "loss": 1.2077,
+      "theoretical_loss": 3.335234400423463,
+      "tokens_seen": 3170304000
+    },
+    {
+      "epoch": 0.92,
+      "objective/train/docs_used": 1062678,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1779983043670654,
+      "objective/train/theoretical_loss": 3.335224006679694,
+      "objective/train/tokens_used": 1540894176,
+      "theoretical_loss": 3.335224006679694,
+      "tokens_seen": 3170435072
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9680629112502e-05,
+      "loss": 1.197,
+      "theoretical_loss": 3.335224006679694,
+      "tokens_seen": 3170435072
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.96405071417108e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.335213613485923,
+      "tokens_seen": 3170566144
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9600385170919594e-05,
+      "loss": 1.1533,
+      "theoretical_loss": 3.335203220842099,
+      "tokens_seen": 3170697216
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.956026320012839e-05,
+      "loss": 1.2815,
+      "theoretical_loss": 3.3351928287481707,
+      "tokens_seen": 3170828288
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9520141229337186e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.3351824372040846,
+      "tokens_seen": 3170959360
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9480019258545986e-05,
+      "loss": 1.1908,
+      "theoretical_loss": 3.33517204620979,
+      "tokens_seen": 3171090432
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.943989728775477e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.335161655765235,
+      "tokens_seen": 3171221504
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.939977531696357e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.3351512658703673,
+      "tokens_seen": 3171352576
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9359653346172364e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.335140876525135,
+      "tokens_seen": 3171483648
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.931953137538116e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3351304877294874,
+      "tokens_seen": 3171614720
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9279409404589956e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.335120099483372,
+      "tokens_seen": 3171745792
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9239287433798756e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.335109711786737,
+      "tokens_seen": 3171876864
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.919916546300754e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.335099324639531,
+      "tokens_seen": 3172007936
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9159043492216334e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.3350889380417015,
+      "tokens_seen": 3172139008
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9118921521425134e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.3350785519931976,
+      "tokens_seen": 3172270080
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.907879955063393e-05,
+      "loss": 1.1616,
+      "theoretical_loss": 3.3350681664939668,
+      "tokens_seen": 3172401152
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9038677579842726e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3350577815439575,
+      "tokens_seen": 3172532224
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.899855560905152e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.3350473971431187,
+      "tokens_seen": 3172663296
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.895843363826031e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.335037013291398,
+      "tokens_seen": 3172794368
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8918311667469104e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.335026629988744,
+      "tokens_seen": 3172925440
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8878189696677904e-05,
+      "loss": 1.2971,
+      "theoretical_loss": 3.3350162472351044,
+      "tokens_seen": 3173056512
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.88380677258867e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.335005865030428,
+      "tokens_seen": 3173187584
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8797945755095496e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.334995483374663,
+      "tokens_seen": 3173318656
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.875782378430429e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.3349851022677575,
+      "tokens_seen": 3173449728
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8717701813513075e-05,
+      "loss": 1.18,
+      "theoretical_loss": 3.334974721709661,
+      "tokens_seen": 3173580800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8677579842721875e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.33496434170032,
+      "tokens_seen": 3173711872
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.863745787193067e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.3349539622396835,
+      "tokens_seen": 3173842944
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.859733590113947e-05,
+      "loss": 1.1569,
+      "theoretical_loss": 3.3349435833277004,
+      "tokens_seen": 3173974016
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.855721393034826e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3349332049643188,
+      "tokens_seen": 3174105088
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.851709195955706e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.3349228271494864,
+      "tokens_seen": 3174236160
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8476969988765845e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.3349124498831526,
+      "tokens_seen": 3174367232
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8436848017974645e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.334902073165265,
+      "tokens_seen": 3174498304
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.839672604718344e-05,
+      "loss": 1.2161,
+      "theoretical_loss": 3.334891696995772,
+      "tokens_seen": 3174629376
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.835660407639224e-05,
+      "loss": 1.1438,
+      "theoretical_loss": 3.3348813213746227,
+      "tokens_seen": 3174760448
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.831648210560103e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.3348709463017645,
+      "tokens_seen": 3174891520
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.827636013480982e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.3348605717771465,
+      "tokens_seen": 3175022592
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8236238164018615e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.3348501978007166,
+      "tokens_seen": 3175153664
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.819611619322741e-05,
+      "loss": 1.2013,
+      "theoretical_loss": 3.3348398243724238,
+      "tokens_seen": 3175284736
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.815599422243621e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.334829451492216,
+      "tokens_seen": 3175415808
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8115872251645e-05,
+      "loss": 1.2586,
+      "theoretical_loss": 3.3348190791600416,
+      "tokens_seen": 3175546880
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.80757502808538e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.3348087073758497,
+      "tokens_seen": 3175677952
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.803562831006259e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3347983361395883,
+      "tokens_seen": 3175809024
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.7995506339271385e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.334787965451205,
+      "tokens_seen": 3175940096
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.795538436848018e-05,
+      "loss": 1.1388,
+      "theoretical_loss": 3.33477759531065,
+      "tokens_seen": 3176071168
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.791526239768898e-05,
+      "loss": 1.1502,
+      "theoretical_loss": 3.3347672257178704,
+      "tokens_seen": 3176202240
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.787514042689777e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.334756856672815,
+      "tokens_seen": 3176333312
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.783501845610657e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.334746488175432,
+      "tokens_seen": 3176464384
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.779489648531536e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.3347361202256707,
+      "tokens_seen": 3176595456
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.775477451452415e-05,
+      "loss": 1.1471,
+      "theoretical_loss": 3.334725752823479,
+      "tokens_seen": 3176726528
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.771465254373295e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3347153859688055,
+      "tokens_seen": 3176857600
+    },
+    {
+      "epoch": 0.93,
+      "objective/train/docs_used": 1064723,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2230794429779053,
+      "objective/train/theoretical_loss": 3.3347050196615986,
+      "objective/train/tokens_used": 1547447776,
+      "theoretical_loss": 3.3347050196615986,
+      "tokens_seen": 3176988672
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.767453057294174e-05,
+      "loss": 1.2088,
+      "theoretical_loss": 3.3347050196615986,
+      "tokens_seen": 3176988672
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.763440860215054e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.3346946539018067,
+      "tokens_seen": 3177119744
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.759428663135933e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.334684288689379,
+      "tokens_seen": 3177250816
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.755416466056813e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.334673924024263,
+      "tokens_seen": 3177381888
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.751404268977692e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.334663559906408,
+      "tokens_seen": 3177512960
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.747392071898572e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.334653196335762,
+      "tokens_seen": 3177644032
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.743379874819451e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.334642833312274,
+      "tokens_seen": 3177775104
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.739367677740331e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3346324708358925,
+      "tokens_seen": 3177906176
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.73535548066121e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.334622108906566,
+      "tokens_seen": 3178037248
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.7313432835820896e-05,
+      "loss": 1.1787,
+      "theoretical_loss": 3.3346117475242423,
+      "tokens_seen": 3178168320
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.727331086502969e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3346013866888713,
+      "tokens_seen": 3178299392
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.723318889423848e-05,
+      "loss": 1.1927,
+      "theoretical_loss": 3.3345910264004006,
+      "tokens_seen": 3178430464
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.719306692344728e-05,
+      "loss": 1.1945,
+      "theoretical_loss": 3.334580666658779,
+      "tokens_seen": 3178561536
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.7152944952656074e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3345703074639554,
+      "tokens_seen": 3178692608
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.711282298186487e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.334559948815878,
+      "tokens_seen": 3178823680
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.7072701011073666e-05,
+      "loss": 1.1561,
+      "theoretical_loss": 3.334549590714496,
+      "tokens_seen": 3178954752
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.703257904028246e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.3345392331597568,
+      "tokens_seen": 3179085824
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.699245706949125e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.3345288761516105,
+      "tokens_seen": 3179216896
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.695233509870005e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.3345185196900045,
+      "tokens_seen": 3179347968
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6912213127908844e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.3345081637748883,
+      "tokens_seen": 3179479040
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.687209115711764e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.33449780840621,
+      "tokens_seen": 3179610112
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6831969186326436e-05,
+      "loss": 1.1637,
+      "theoretical_loss": 3.334487453583918,
+      "tokens_seen": 3179741184
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.679184721553522e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.334477099307962,
+      "tokens_seen": 3179872256
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.675172524474402e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.33446674557829,
+      "tokens_seen": 3180003328
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6711603273952814e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.3344563923948507,
+      "tokens_seen": 3180134400
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6671481303161614e-05,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3344460397575926,
+      "tokens_seen": 3180265472
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6631359332370407e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.3344356876664643,
+      "tokens_seen": 3180396544
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6591237361579206e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.334425336121415,
+      "tokens_seen": 3180527616
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.655111539078799e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.334414985122393,
+      "tokens_seen": 3180658688
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.651099341999679e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.3344046346693474,
+      "tokens_seen": 3180789760
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6470871449205584e-05,
+      "loss": 1.2136,
+      "theoretical_loss": 3.3343942847622263,
+      "tokens_seen": 3180920832
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6430749478414384e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.3343839354009788,
+      "tokens_seen": 3181051904
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.639062750762318e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.3343735865855537,
+      "tokens_seen": 3181182976
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.635050553683197e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.334363238315899,
+      "tokens_seen": 3181314048
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.631038356604076e-05,
+      "loss": 1.2993,
+      "theoretical_loss": 3.3343528905919646,
+      "tokens_seen": 3181445120
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6270261595249555e-05,
+      "loss": 1.1397,
+      "theoretical_loss": 3.3343425434136984,
+      "tokens_seen": 3181576192
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6230139624458354e-05,
+      "loss": 1.1214,
+      "theoretical_loss": 3.33433219678105,
+      "tokens_seen": 3181707264
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.619001765366715e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.334321850693967,
+      "tokens_seen": 3181838336
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.614989568287595e-05,
+      "loss": 1.1744,
+      "theoretical_loss": 3.3343115051523986,
+      "tokens_seen": 3181969408
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.610977371208474e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.3343011601562935,
+      "tokens_seen": 3182100480
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.606965174129353e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.334290815705601,
+      "tokens_seen": 3182231552
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6029529770502325e-05,
+      "loss": 1.1863,
+      "theoretical_loss": 3.3342804718002697,
+      "tokens_seen": 3182362624
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5989407799711125e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.334270128440248,
+      "tokens_seen": 3182493696
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.594928582891992e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.334259785625485,
+      "tokens_seen": 3182624768
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.590916385812871e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.3342494433559295,
+      "tokens_seen": 3182755840
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.586904188733751e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.33423910163153,
+      "tokens_seen": 3182886912
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5828919916546296e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.334228760452236,
+      "tokens_seen": 3183017984
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5788797945755095e-05,
+      "loss": 1.1981,
+      "theoretical_loss": 3.3342184198179954,
+      "tokens_seen": 3183149056
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.574867597496389e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.334208079728757,
+      "tokens_seen": 3183280128
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.570855400417269e-05,
+      "loss": 1.2042,
+      "theoretical_loss": 3.334197740184471,
+      "tokens_seen": 3183411200
+    },
+    {
+      "epoch": 0.93,
+      "objective/train/docs_used": 1067002,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1871349811553955,
+      "objective/train/theoretical_loss": 3.334187401185085,
+      "objective/train/tokens_used": 1554001376,
+      "theoretical_loss": 3.334187401185085,
+      "tokens_seen": 3183542272
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.566843203338148e-05,
+      "loss": 1.1334,
+      "theoretical_loss": 3.334187401185085,
+      "tokens_seen": 3183542272
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.562831006259028e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.3341770627305487,
+      "tokens_seen": 3183673344
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5588188091799066e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3341667248208102,
+      "tokens_seen": 3183804416
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5548066121007865e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3341563874558187,
+      "tokens_seen": 3183935488
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.550794415021666e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.334146050635523,
+      "tokens_seen": 3184066560
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.546782217942546e-05,
+      "loss": 1.3025,
+      "theoretical_loss": 3.3341357143598715,
+      "tokens_seen": 3184197632
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.542770020863425e-05,
+      "loss": 1.2358,
+      "theoretical_loss": 3.3341253786288143,
+      "tokens_seen": 3184328704
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.538757823784304e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3341150434422993,
+      "tokens_seen": 3184459776
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5347456267051836e-05,
+      "loss": 1.1644,
+      "theoretical_loss": 3.3341047088002758,
+      "tokens_seen": 3184590848
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.530733429626063e-05,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3340943747026923,
+      "tokens_seen": 3184721920
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.526721232546943e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.334084041149498,
+      "tokens_seen": 3184852992
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.522709035467822e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.334073708140642,
+      "tokens_seen": 3184984064
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.518696838388702e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.334063375676073,
+      "tokens_seen": 3185115136
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.514684641309581e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.3340530437557403,
+      "tokens_seen": 3185246208
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.510672444230461e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.334042712379592,
+      "tokens_seen": 3185377280
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.50666024715134e-05,
+      "loss": 1.1322,
+      "theoretical_loss": 3.3340323815475776,
+      "tokens_seen": 3185508352
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.50264805007222e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3340220512596463,
+      "tokens_seen": 3185639424
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.498635852993099e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.3340117215157465,
+      "tokens_seen": 3185770496
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4946236559139784e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3340013923158276,
+      "tokens_seen": 3185901568
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.490611458834858e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.3339910636598384,
+      "tokens_seen": 3186032640
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.486599261755737e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.333980735547728,
+      "tokens_seen": 3186163712
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.482587064676617e-05,
+      "loss": 1.1778,
+      "theoretical_loss": 3.333970407979445,
+      "tokens_seen": 3186294784
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.478574867597496e-05,
+      "loss": 1.1269,
+      "theoretical_loss": 3.3339600809549395,
+      "tokens_seen": 3186425856
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.474562670518376e-05,
+      "loss": 1.1664,
+      "theoretical_loss": 3.3339497544741588,
+      "tokens_seen": 3186556928
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4705504734392554e-05,
+      "loss": 1.1912,
+      "theoretical_loss": 3.3339394285370534,
+      "tokens_seen": 3186688000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.466538276360135e-05,
+      "loss": 1.2049,
+      "theoretical_loss": 3.3339291031435714,
+      "tokens_seen": 3186819072
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4625260792810146e-05,
+      "loss": 1.1979,
+      "theoretical_loss": 3.333918778293662,
+      "tokens_seen": 3186950144
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.458513882201894e-05,
+      "loss": 1.1334,
+      "theoretical_loss": 3.3339084539872745,
+      "tokens_seen": 3187081216
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.454501685122773e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.333898130224358,
+      "tokens_seen": 3187212288
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.450489488043653e-05,
+      "loss": 1.1505,
+      "theoretical_loss": 3.333887807004861,
+      "tokens_seen": 3187343360
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4464772909645324e-05,
+      "loss": 1.1757,
+      "theoretical_loss": 3.333877484328733,
+      "tokens_seen": 3187474432
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4424650938854116e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.333867162195923,
+      "tokens_seen": 3187605504
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4384528968062916e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.3338568406063804,
+      "tokens_seen": 3187736576
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.43444069972717e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3338465195600535,
+      "tokens_seen": 3187867648
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.43042850264805e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3338361990568917,
+      "tokens_seen": 3187998720
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4264163055689294e-05,
+      "loss": 1.1644,
+      "theoretical_loss": 3.3338258790968442,
+      "tokens_seen": 3188129792
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4224041084898094e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.33381555967986,
+      "tokens_seen": 3188260864
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4183919114106887e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.3338052408058885,
+      "tokens_seen": 3188391936
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4143797143315686e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3337949224748784,
+      "tokens_seen": 3188523008
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.410367517252447e-05,
+      "loss": 1.1649,
+      "theoretical_loss": 3.333784604686779,
+      "tokens_seen": 3188654080
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.406355320173327e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3337742874415395,
+      "tokens_seen": 3188785152
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4023431230942064e-05,
+      "loss": 1.1473,
+      "theoretical_loss": 3.333763970739109,
+      "tokens_seen": 3188916224
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.398330926015086e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.333753654579436,
+      "tokens_seen": 3189047296
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.394318728935966e-05,
+      "loss": 1.1506,
+      "theoretical_loss": 3.3337433389624707,
+      "tokens_seen": 3189178368
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.390306531856845e-05,
+      "loss": 1.1906,
+      "theoretical_loss": 3.3337330238881613,
+      "tokens_seen": 3189309440
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.386294334777724e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.333722709356458,
+      "tokens_seen": 3189440512
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3822821376986035e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3337123953673085,
+      "tokens_seen": 3189571584
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3782699406194834e-05,
+      "loss": 1.333,
+      "theoretical_loss": 3.3337020819206633,
+      "tokens_seen": 3189702656
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.374257743540363e-05,
+      "loss": 1.1619,
+      "theoretical_loss": 3.333691769016471,
+      "tokens_seen": 3189833728
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.370245546461243e-05,
+      "loss": 1.174,
+      "theoretical_loss": 3.3336814566546806,
+      "tokens_seen": 3189964800
+    },
+    {
+      "epoch": 0.93,
+      "objective/train/docs_used": 1069349,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2405363321304321,
+      "objective/train/theoretical_loss": 3.3336711448352423,
+      "objective/train/tokens_used": 1560554976,
+      "theoretical_loss": 3.3336711448352423,
+      "tokens_seen": 3190095872
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.366233349382122e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3336711448352423,
+      "tokens_seen": 3190095872
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.362221152303001e-05,
+      "loss": 1.1195,
+      "theoretical_loss": 3.333660833558104,
+      "tokens_seen": 3190226944
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3582089552238805e-05,
+      "loss": 1.2742,
+      "theoretical_loss": 3.3336505228232154,
+      "tokens_seen": 3190358016
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.35419675814476e-05,
+      "loss": 1.1674,
+      "theoretical_loss": 3.333640212630526,
+      "tokens_seen": 3190489088
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.35018456106564e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3336299029799847,
+      "tokens_seen": 3190620160
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.346172363986519e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.3336195938715405,
+      "tokens_seen": 3190751232
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.342160166907399e-05,
+      "loss": 1.1692,
+      "theoretical_loss": 3.333609285305143,
+      "tokens_seen": 3190882304
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3381479698282775e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.333598977280742,
+      "tokens_seen": 3191013376
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3341357727491575e-05,
+      "loss": 1.2913,
+      "theoretical_loss": 3.3335886697982855,
+      "tokens_seen": 3191144448
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.330123575670037e-05,
+      "loss": 1.1727,
+      "theoretical_loss": 3.3335783628577236,
+      "tokens_seen": 3191275520
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.326111378590917e-05,
+      "loss": 1.2927,
+      "theoretical_loss": 3.333568056459005,
+      "tokens_seen": 3191406592
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.322099181511796e-05,
+      "loss": 1.1857,
+      "theoretical_loss": 3.3335577506020795,
+      "tokens_seen": 3191537664
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.318086984432676e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.333547445286896,
+      "tokens_seen": 3191668736
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3140747873535546e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.3335371405134038,
+      "tokens_seen": 3191799808
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3100625902744345e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.3335268362815524,
+      "tokens_seen": 3191930880
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.306050393195314e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3335165325912914,
+      "tokens_seen": 3192061952
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.302038196116193e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.3335062294425692,
+      "tokens_seen": 3192193024
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.298025999037073e-05,
+      "loss": 1.1375,
+      "theoretical_loss": 3.3334959268353357,
+      "tokens_seen": 3192324096
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.294013801957952e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.33348562476954,
+      "tokens_seen": 3192455168
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.2900016048788316e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.333475323245132,
+      "tokens_seen": 3192586240
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.285989407799711e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.33346502226206,
+      "tokens_seen": 3192717312
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.281977210720591e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.3334547218202744,
+      "tokens_seen": 3192848384
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.27796501364147e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.3334444219197232,
+      "tokens_seen": 3192979456
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.27395281656235e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.3334341225603574,
+      "tokens_seen": 3193110528
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.269940619483229e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.333423823742125,
+      "tokens_seen": 3193241600
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2659284224041086e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.333413525464976,
+      "tokens_seen": 3193372672
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.261916225324988e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.3334032277288594,
+      "tokens_seen": 3193503744
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.257904028245867e-05,
+      "loss": 1.2375,
+      "theoretical_loss": 3.3333929305337255,
+      "tokens_seen": 3193634816
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.253891831166747e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3333826338795225,
+      "tokens_seen": 3193765888
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2498796340876264e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.3333723377662,
+      "tokens_seen": 3193896960
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.245867437008506e-05,
+      "loss": 1.1785,
+      "theoretical_loss": 3.3333620421937082,
+      "tokens_seen": 3194028032
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.241855239929385e-05,
+      "loss": 1.1592,
+      "theoretical_loss": 3.333351747161996,
+      "tokens_seen": 3194159104
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.237843042850265e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.333341452671012,
+      "tokens_seen": 3194290176
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.233830845771144e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.333331158720707,
+      "tokens_seen": 3194421248
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.229818648692024e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.3333208653110296,
+      "tokens_seen": 3194552320
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2258064516129034e-05,
+      "loss": 1.234,
+      "theoretical_loss": 3.3333105724419294,
+      "tokens_seen": 3194683392
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.221794254533783e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.333300280113356,
+      "tokens_seen": 3194814464
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.217782057454662e-05,
+      "loss": 1.182,
+      "theoretical_loss": 3.3332899883252587,
+      "tokens_seen": 3194945536
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.213769860375542e-05,
+      "loss": 1.1326,
+      "theoretical_loss": 3.333279697077587,
+      "tokens_seen": 3195076608
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.209757663296421e-05,
+      "loss": 1.3254,
+      "theoretical_loss": 3.33326940637029,
+      "tokens_seen": 3195207680
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2057454662173004e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.3332591162033176,
+      "tokens_seen": 3195338752
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2017332691381804e-05,
+      "loss": 1.1796,
+      "theoretical_loss": 3.3332488265766194,
+      "tokens_seen": 3195469824
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1977210720590596e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.333238537490144,
+      "tokens_seen": 3195600896
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.193708874979939e-05,
+      "loss": 1.1886,
+      "theoretical_loss": 3.333228248943842,
+      "tokens_seen": 3195731968
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.189696677900818e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.333217960937662,
+      "tokens_seen": 3195863040
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.185684480821698e-05,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3332076734715543,
+      "tokens_seen": 3195994112
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1816722837425774e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.333197386545468,
+      "tokens_seen": 3196125184
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1776600866634574e-05,
+      "loss": 1.1404,
+      "theoretical_loss": 3.333187100159352,
+      "tokens_seen": 3196256256
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1736478895843366e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.333176814313157,
+      "tokens_seen": 3196387328
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.169635692505216e-05,
+      "loss": 1.1411,
+      "theoretical_loss": 3.3331665290068315,
+      "tokens_seen": 3196518400
+    },
+    {
+      "epoch": 0.94,
+      "objective/train/docs_used": 1071513,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1959319114685059,
+      "objective/train/theoretical_loss": 3.3331562442403255,
+      "objective/train/tokens_used": 1567108576,
+      "theoretical_loss": 3.3331562442403255,
+      "tokens_seen": 3196649472
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.165623495426095e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.3331562442403255,
+      "tokens_seen": 3196649472
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1616112983469745e-05,
+      "loss": 1.1785,
+      "theoretical_loss": 3.3331459600135886,
+      "tokens_seen": 3196780544
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1575991012678544e-05,
+      "loss": 1.146,
+      "theoretical_loss": 3.3331356763265703,
+      "tokens_seen": 3196911616
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.153586904188734e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3331253931792197,
+      "tokens_seen": 3197042688
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1495747071096137e-05,
+      "loss": 1.2124,
+      "theoretical_loss": 3.3331151105714873,
+      "tokens_seen": 3197173760
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.145562510030492e-05,
+      "loss": 1.1736,
+      "theoretical_loss": 3.3331048285033216,
+      "tokens_seen": 3197304832
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.141550312951372e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.3330945469746727,
+      "tokens_seen": 3197435904
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1375381158722515e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.3330842659854905,
+      "tokens_seen": 3197566976
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1335259187931314e-05,
+      "loss": 1.1624,
+      "theoretical_loss": 3.333073985535724,
+      "tokens_seen": 3197698048
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.129513721714011e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.3330637056253227,
+      "tokens_seen": 3197829120
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.125501524634891e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.333053426254237,
+      "tokens_seen": 3197960192
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.121489327555769e-05,
+      "loss": 1.1685,
+      "theoretical_loss": 3.3330431474224156,
+      "tokens_seen": 3198091264
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.117477130476649e-05,
+      "loss": 1.2072,
+      "theoretical_loss": 3.3330328691298083,
+      "tokens_seen": 3198222336
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1134649333975285e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.3330225913763654,
+      "tokens_seen": 3198353408
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.109452736318408e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.333012314162036,
+      "tokens_seen": 3198484480
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.105440539239288e-05,
+      "loss": 1.1927,
+      "theoretical_loss": 3.3330020374867697,
+      "tokens_seen": 3198615552
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.101428342160167e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3329917613505167,
+      "tokens_seen": 3198746624
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.097416145081046e-05,
+      "loss": 1.1898,
+      "theoretical_loss": 3.3329814857532254,
+      "tokens_seen": 3198877696
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0934039480019255e-05,
+      "loss": 1.1336,
+      "theoretical_loss": 3.3329712106948466,
+      "tokens_seen": 3199008768
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0893917509228055e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3329609361753296,
+      "tokens_seen": 3199139840
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.085379553843685e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.332950662194624,
+      "tokens_seen": 3199270912
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.081367356764564e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.332940388752679,
+      "tokens_seen": 3199401984
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.077355159685444e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.3329301158494458,
+      "tokens_seen": 3199533056
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.073342962606323e-05,
+      "loss": 1.1839,
+      "theoretical_loss": 3.332919843484872,
+      "tokens_seen": 3199664128
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0693307655272026e-05,
+      "loss": 1.1101,
+      "theoretical_loss": 3.332909571658909,
+      "tokens_seen": 3199795200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0653185684480825e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.3328993003715057,
+      "tokens_seen": 3199926272
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.061306371368962e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.332889029622612,
+      "tokens_seen": 3200057344
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.057294174289841e-05,
+      "loss": 1.1513,
+      "theoretical_loss": 3.3328787594121776,
+      "tokens_seen": 3200188416
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.053281977210721e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.3328684897401524,
+      "tokens_seen": 3200319488
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0492697801316003e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.3328582206064854,
+      "tokens_seen": 3200450560
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0452575830524796e-05,
+      "loss": 1.1869,
+      "theoretical_loss": 3.3328479520111274,
+      "tokens_seen": 3200581632
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.041245385973359e-05,
+      "loss": 1.133,
+      "theoretical_loss": 3.3328376839540272,
+      "tokens_seen": 3200712704
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0372331888942384e-05,
+      "loss": 1.1441,
+      "theoretical_loss": 3.3328274164351352,
+      "tokens_seen": 3200843776
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.033220991815118e-05,
+      "loss": 1.1844,
+      "theoretical_loss": 3.3328171494544008,
+      "tokens_seen": 3200974848
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0292087947359973e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.3328068830117736,
+      "tokens_seen": 3201105920
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.025196597656877e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3327966171072037,
+      "tokens_seen": 3201236992
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0211844005777566e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.332786351740641,
+      "tokens_seen": 3201368064
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.017172203498636e-05,
+      "loss": 1.2348,
+      "theoretical_loss": 3.332776086912035,
+      "tokens_seen": 3201499136
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0131600064195155e-05,
+      "loss": 1.1484,
+      "theoretical_loss": 3.3327658226213352,
+      "tokens_seen": 3201630208
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.009147809340395e-05,
+      "loss": 1.1742,
+      "theoretical_loss": 3.332755558868492,
+      "tokens_seen": 3201761280
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0051356122612743e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.332745295653455,
+      "tokens_seen": 3201892352
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.001123415182154e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.3327350329761742,
+      "tokens_seen": 3202023424
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9971112181030332e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.332724770836599,
+      "tokens_seen": 3202154496
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9930990210239125e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.332714509234679,
+      "tokens_seen": 3202285568
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.989086823944792e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.332704248170365,
+      "tokens_seen": 3202416640
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9850746268656717e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.3326939876436055,
+      "tokens_seen": 3202547712
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.981062429786551e-05,
+      "loss": 1.3229,
+      "theoretical_loss": 3.3326837276543513,
+      "tokens_seen": 3202678784
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9770502327074306e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.3326734682025525,
+      "tokens_seen": 3202809856
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9730380356283102e-05,
+      "loss": 1.1825,
+      "theoretical_loss": 3.332663209288158,
+      "tokens_seen": 3202940928
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9690258385491895e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.3326529509111182,
+      "tokens_seen": 3203072000
+    },
+    {
+      "epoch": 0.94,
+      "objective/train/docs_used": 1073646,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2724738121032715,
+      "objective/train/theoretical_loss": 3.3326426930713833,
+      "objective/train/tokens_used": 1573662176,
+      "theoretical_loss": 3.3326426930713833,
+      "tokens_seen": 3203203072
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.965013641470069e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.3326426930713833,
+      "tokens_seen": 3203203072
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9610014443909487e-05,
+      "loss": 1.1972,
+      "theoretical_loss": 3.332632435768902,
+      "tokens_seen": 3203334144
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.956989247311828e-05,
+      "loss": 1.1646,
+      "theoretical_loss": 3.3326221790036255,
+      "tokens_seen": 3203465216
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9529770502327073e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.332611922775503,
+      "tokens_seen": 3203596288
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.948964853153587e-05,
+      "loss": 1.2348,
+      "theoretical_loss": 3.332601667084485,
+      "tokens_seen": 3203727360
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9449526560744662e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.33259141193052,
+      "tokens_seen": 3203858432
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9409404589953458e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.3325811573135597,
+      "tokens_seen": 3203989504
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9369282619162254e-05,
+      "loss": 1.1298,
+      "theoretical_loss": 3.332570903233553,
+      "tokens_seen": 3204120576
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9329160648371047e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.33256064969045,
+      "tokens_seen": 3204251648
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9289038677579843e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3325503966842,
+      "tokens_seen": 3204382720
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.924891670678864e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.3325401442147538,
+      "tokens_seen": 3204513792
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9208794735997432e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.332529892282061,
+      "tokens_seen": 3204644864
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9168672765206228e-05,
+      "loss": 1.1598,
+      "theoretical_loss": 3.332519640886072,
+      "tokens_seen": 3204775936
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9128550794415024e-05,
+      "loss": 1.1669,
+      "theoretical_loss": 3.332509390026736,
+      "tokens_seen": 3204907008
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9088428823623817e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.332499139704004,
+      "tokens_seen": 3205038080
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.904830685283261e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.3324888899178244,
+      "tokens_seen": 3205169152
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9008184882041406e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.332478640668149,
+      "tokens_seen": 3205300224
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.89680629112502e-05,
+      "loss": 1.2128,
+      "theoretical_loss": 3.3324683919549263,
+      "tokens_seen": 3205431296
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8927940940458995e-05,
+      "loss": 1.1519,
+      "theoretical_loss": 3.3324581437781067,
+      "tokens_seen": 3205562368
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.888781896966779e-05,
+      "loss": 1.1757,
+      "theoretical_loss": 3.3324478961376407,
+      "tokens_seen": 3205693440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8847696998876584e-05,
+      "loss": 1.1521,
+      "theoretical_loss": 3.3324376490334777,
+      "tokens_seen": 3205824512
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.880757502808538e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.3324274024655685,
+      "tokens_seen": 3205955584
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8767453057294176e-05,
+      "loss": 1.1702,
+      "theoretical_loss": 3.332417156433862,
+      "tokens_seen": 3206086656
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.872733108650297e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.332406910938309,
+      "tokens_seen": 3206217728
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8687209115711765e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.332396665978859,
+      "tokens_seen": 3206348800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.864708714492056e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.3323864215554626,
+      "tokens_seen": 3206479872
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8606965174129357e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3323761776680696,
+      "tokens_seen": 3206610944
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8566843203338146e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.33236593431663,
+      "tokens_seen": 3206742016
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8526721232546943e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.332355691501094,
+      "tokens_seen": 3206873088
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8486599261755735e-05,
+      "loss": 1.1937,
+      "theoretical_loss": 3.3323454492214113,
+      "tokens_seen": 3207004160
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.844647729096453e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.3323352074775325,
+      "tokens_seen": 3207135232
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8406355320173328e-05,
+      "loss": 1.1981,
+      "theoretical_loss": 3.3323249662694074,
+      "tokens_seen": 3207266304
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8366233349382124e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.332314725596986,
+      "tokens_seen": 3207397376
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8326111378590917e-05,
+      "loss": 1.1469,
+      "theoretical_loss": 3.3323044854602184,
+      "tokens_seen": 3207528448
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8285989407799713e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3322942458590545,
+      "tokens_seen": 3207659520
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.824586743700851e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.332284006793445,
+      "tokens_seen": 3207790592
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.82057454662173e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.33227376826334,
+      "tokens_seen": 3207921664
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8165623495426098e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.3322635302686887,
+      "tokens_seen": 3208052736
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8125501524634894e-05,
+      "loss": 1.1478,
+      "theoretical_loss": 3.3322532928094417,
+      "tokens_seen": 3208183808
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8085379553843683e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3322430558855496,
+      "tokens_seen": 3208314880
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.804525758305248e-05,
+      "loss": 1.2097,
+      "theoretical_loss": 3.332232819496962,
+      "tokens_seen": 3208445952
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8005135612261276e-05,
+      "loss": 1.1567,
+      "theoretical_loss": 3.332222583643629,
+      "tokens_seen": 3208577024
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.7965013641470068e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.332212348325502,
+      "tokens_seen": 3208708096
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.7924891670678864e-05,
+      "loss": 1.1777,
+      "theoretical_loss": 3.332202113542529,
+      "tokens_seen": 3208839168
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.788476969988766e-05,
+      "loss": 1.2067,
+      "theoretical_loss": 3.332191879294662,
+      "tokens_seen": 3208970240
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.7844647729096453e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3321816455818496,
+      "tokens_seen": 3209101312
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.780452575830525e-05,
+      "loss": 1.1764,
+      "theoretical_loss": 3.3321714124040436,
+      "tokens_seen": 3209232384
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7764403787514046e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.3321611797611927,
+      "tokens_seen": 3209363456
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.772428181672284e-05,
+      "loss": 1.166,
+      "theoretical_loss": 3.3321509476532487,
+      "tokens_seen": 3209494528
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7684159845931635e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.3321407160801604,
+      "tokens_seen": 3209625600
+    },
+    {
+      "epoch": 0.95,
+      "objective/train/docs_used": 1075876,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.237161636352539,
+      "objective/train/theoretical_loss": 3.332130485041878,
+      "objective/train/tokens_used": 1580215776,
+      "theoretical_loss": 3.332130485041878,
+      "tokens_seen": 3209756672
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7644037875140427e-05,
+      "loss": 1.1981,
+      "theoretical_loss": 3.332130485041878,
+      "tokens_seen": 3209756672
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.760391590434922e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.332120254538353,
+      "tokens_seen": 3209887744
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7563793933558016e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.332110024569534,
+      "tokens_seen": 3210018816
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7523671962766812e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3320997951353726,
+      "tokens_seen": 3210149888
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7483549991975605e-05,
+      "loss": 1.1766,
+      "theoretical_loss": 3.3320895662358185,
+      "tokens_seen": 3210280960
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.74434280211844e-05,
+      "loss": 1.184,
+      "theoretical_loss": 3.3320793378708213,
+      "tokens_seen": 3210412032
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7403306050393197e-05,
+      "loss": 1.1732,
+      "theoretical_loss": 3.3320691100403326,
+      "tokens_seen": 3210543104
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.736318407960199e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3320588827443016,
+      "tokens_seen": 3210674176
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7323062108810786e-05,
+      "loss": 1.2703,
+      "theoretical_loss": 3.3320486559826787,
+      "tokens_seen": 3210805248
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7282940138019582e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.332038429755414,
+      "tokens_seen": 3210936320
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7242818167228375e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.3320282040624587,
+      "tokens_seen": 3211067392
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.720269619643717e-05,
+      "loss": 1.2097,
+      "theoretical_loss": 3.332017978903762,
+      "tokens_seen": 3211198464
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7162574225645964e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.332007754279275,
+      "tokens_seen": 3211329536
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7122452254854757e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3319975301889473,
+      "tokens_seen": 3211460608
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7082330284063553e-05,
+      "loss": 1.1629,
+      "theoretical_loss": 3.3319873066327297,
+      "tokens_seen": 3211591680
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.704220831327235e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.331977083610572,
+      "tokens_seen": 3211722752
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7002086342481142e-05,
+      "loss": 1.1887,
+      "theoretical_loss": 3.331966861122425,
+      "tokens_seen": 3211853824
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6961964371689938e-05,
+      "loss": 1.1669,
+      "theoretical_loss": 3.3319566391682387,
+      "tokens_seen": 3211984896
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6921842400898734e-05,
+      "loss": 1.1835,
+      "theoretical_loss": 3.3319464177479636,
+      "tokens_seen": 3212115968
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6881720430107527e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.3319361968615504,
+      "tokens_seen": 3212247040
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6841598459316323e-05,
+      "loss": 1.1089,
+      "theoretical_loss": 3.3319259765089484,
+      "tokens_seen": 3212378112
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.680147648852512e-05,
+      "loss": 1.18,
+      "theoretical_loss": 3.331915756690109,
+      "tokens_seen": 3212509184
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6761354517733912e-05,
+      "loss": 1.1582,
+      "theoretical_loss": 3.3319055374049817,
+      "tokens_seen": 3212640256
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6721232546942708e-05,
+      "loss": 1.1586,
+      "theoretical_loss": 3.3318953186535176,
+      "tokens_seen": 3212771328
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.66811105761515e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3318851004356667,
+      "tokens_seen": 3212902400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6640988605360294e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.331874882751379,
+      "tokens_seen": 3213033472
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.660086663456909e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3318646656006057,
+      "tokens_seen": 3213164544
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6560744663777886e-05,
+      "loss": 1.2912,
+      "theoretical_loss": 3.3318544489832966,
+      "tokens_seen": 3213295616
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.652062269298668e-05,
+      "loss": 1.2088,
+      "theoretical_loss": 3.3318442328994022,
+      "tokens_seen": 3213426688
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6480500722195475e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.331834017348873,
+      "tokens_seen": 3213557760
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.644037875140427e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.3318238023316593,
+      "tokens_seen": 3213688832
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6400256780613064e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.3318135878477113,
+      "tokens_seen": 3213819904
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.636013480982186e-05,
+      "loss": 1.1965,
+      "theoretical_loss": 3.33180337389698,
+      "tokens_seen": 3213950976
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6320012839030656e-05,
+      "loss": 1.1475,
+      "theoretical_loss": 3.3317931604794153,
+      "tokens_seen": 3214082048
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.627989086823945e-05,
+      "loss": 1.1538,
+      "theoretical_loss": 3.331782947594968,
+      "tokens_seen": 3214213120
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6239768897448245e-05,
+      "loss": 1.1786,
+      "theoretical_loss": 3.3317727352435877,
+      "tokens_seen": 3214344192
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6199646926657038e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.331762523425226,
+      "tokens_seen": 3214475264
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.615952495586583e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.331752312139833,
+      "tokens_seen": 3214606336
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6119402985074626e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.3317421013873583,
+      "tokens_seen": 3214737408
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6079281014283423e-05,
+      "loss": 1.1744,
+      "theoretical_loss": 3.3317318911677534,
+      "tokens_seen": 3214868480
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6039159043492215e-05,
+      "loss": 1.1809,
+      "theoretical_loss": 3.3317216814809685,
+      "tokens_seen": 3214999552
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.599903707270101e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.331711472326954,
+      "tokens_seen": 3215130624
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5958915101909808e-05,
+      "loss": 1.1568,
+      "theoretical_loss": 3.33170126370566,
+      "tokens_seen": 3215261696
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.59187931311186e-05,
+      "loss": 1.2088,
+      "theoretical_loss": 3.331691055617038,
+      "tokens_seen": 3215392768
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5878671160327397e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.331680848061037,
+      "tokens_seen": 3215523840
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5838549189536193e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.3316706410376087,
+      "tokens_seen": 3215654912
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5798427218744985e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.331660434546704,
+      "tokens_seen": 3215785984
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.575830524795378e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.3316502285882716,
+      "tokens_seen": 3215917056
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5718183277162574e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3316400231622634,
+      "tokens_seen": 3216048128
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5678061306371367e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.3316298182686293,
+      "tokens_seen": 3216179200
+    },
+    {
+      "epoch": 0.95,
+      "objective/train/docs_used": 1078111,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2857028245925903,
+      "objective/train/theoretical_loss": 3.3316196139073204,
+      "objective/train/tokens_used": 1586769376,
+      "theoretical_loss": 3.3316196139073204,
+      "tokens_seen": 3216310272
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5637939335580163e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.3316196139073204,
+      "tokens_seen": 3216310272
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.559781736478896e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.331609410078287,
+      "tokens_seen": 3216441344
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5557695393997752e-05,
+      "loss": 1.179,
+      "theoretical_loss": 3.3315992067814797,
+      "tokens_seen": 3216572416
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5517573423206548e-05,
+      "loss": 1.1611,
+      "theoretical_loss": 3.331589004016849,
+      "tokens_seen": 3216703488
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5477451452415344e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.3315788017843455,
+      "tokens_seen": 3216834560
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5437329481624137e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.3315686000839198,
+      "tokens_seen": 3216965632
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5397207510832933e-05,
+      "loss": 1.2466,
+      "theoretical_loss": 3.331558398915522,
+      "tokens_seen": 3217096704
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.535708554004173e-05,
+      "loss": 1.1729,
+      "theoretical_loss": 3.331548198279103,
+      "tokens_seen": 3217227776
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5316963569250522e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.331537998174614,
+      "tokens_seen": 3217358848
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5276841598459315e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.3315277986020044,
+      "tokens_seen": 3217489920
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.523671962766811e-05,
+      "loss": 1.1622,
+      "theoretical_loss": 3.3315175995612263,
+      "tokens_seen": 3217620992
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5196597656876904e-05,
+      "loss": 1.1524,
+      "theoretical_loss": 3.3315074010522285,
+      "tokens_seen": 3217752064
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.51564756860857e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.331497203074963,
+      "tokens_seen": 3217883136
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5116353715294496e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.33148700562938,
+      "tokens_seen": 3218014208
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.507623174450329e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.33147680871543,
+      "tokens_seen": 3218145280
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5036109773712085e-05,
+      "loss": 1.1702,
+      "theoretical_loss": 3.331466612333064,
+      "tokens_seen": 3218276352
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.499598780292088e-05,
+      "loss": 1.1722,
+      "theoretical_loss": 3.331456416482232,
+      "tokens_seen": 3218407424
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4955865832129674e-05,
+      "loss": 1.1696,
+      "theoretical_loss": 3.331446221162885,
+      "tokens_seen": 3218538496
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.491574386133847e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.3314360263749743,
+      "tokens_seen": 3218669568
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4875621890547266e-05,
+      "loss": 1.1501,
+      "theoretical_loss": 3.331425832118449,
+      "tokens_seen": 3218800640
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.483549991975606e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.331415638393261,
+      "tokens_seen": 3218931712
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4795377948964852e-05,
+      "loss": 1.1808,
+      "theoretical_loss": 3.331405445199361,
+      "tokens_seen": 3219062784
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4755255978173648e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.331395252536699,
+      "tokens_seen": 3219193856
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.471513400738244e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.3313850604052266,
+      "tokens_seen": 3219324928
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4675012036591237e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.3313748688048936,
+      "tokens_seen": 3219456000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4634890065800033e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.331364677735651,
+      "tokens_seen": 3219587072
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4594768095008826e-05,
+      "loss": 1.1417,
+      "theoretical_loss": 3.331354487197449,
+      "tokens_seen": 3219718144
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4554646124217622e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.3313442971902396,
+      "tokens_seen": 3219849216
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4514524153426418e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.3313341077139724,
+      "tokens_seen": 3219980288
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.447440218263521e-05,
+      "loss": 1.173,
+      "theoretical_loss": 3.331323918768599,
+      "tokens_seen": 3220111360
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4434280211844007e-05,
+      "loss": 1.1796,
+      "theoretical_loss": 3.331313730354069,
+      "tokens_seen": 3220242432
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4394158241052803e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3313035424703337,
+      "tokens_seen": 3220373504
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4354036270261596e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.331293355117344,
+      "tokens_seen": 3220504576
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.431391429947039e-05,
+      "loss": 1.1598,
+      "theoretical_loss": 3.331283168295051,
+      "tokens_seen": 3220635648
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4273792328679185e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3312729820034046,
+      "tokens_seen": 3220766720
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4233670357887977e-05,
+      "loss": 1.1528,
+      "theoretical_loss": 3.331262796242356,
+      "tokens_seen": 3220897792
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4193548387096773e-05,
+      "loss": 1.1515,
+      "theoretical_loss": 3.3312526110118563,
+      "tokens_seen": 3221028864
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.415342641630557e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.3312424263118556,
+      "tokens_seen": 3221159936
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4113304445514366e-05,
+      "loss": 1.2013,
+      "theoretical_loss": 3.331232242142305,
+      "tokens_seen": 3221291008
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.407318247472316e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.331222058503155,
+      "tokens_seen": 3221422080
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4033060503931955e-05,
+      "loss": 1.1643,
+      "theoretical_loss": 3.331211875394357,
+      "tokens_seen": 3221553152
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.399293853314075e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.3312016928158616,
+      "tokens_seen": 3221684224
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3952816562349544e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.3311915107676198,
+      "tokens_seen": 3221815296
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.391269459155834e-05,
+      "loss": 1.1828,
+      "theoretical_loss": 3.3311813292495813,
+      "tokens_seen": 3221946368
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3872572620767136e-05,
+      "loss": 1.1449,
+      "theoretical_loss": 3.3311711482616984,
+      "tokens_seen": 3222077440
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3832450649975925e-05,
+      "loss": 1.1805,
+      "theoretical_loss": 3.331160967803921,
+      "tokens_seen": 3222208512
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.379232867918472e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3311507878762003,
+      "tokens_seen": 3222339584
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3752206708393518e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.331140608478487,
+      "tokens_seen": 3222470656
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.371208473760231e-05,
+      "loss": 1.1736,
+      "theoretical_loss": 3.331130429610732,
+      "tokens_seen": 3222601728
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3671962766811106e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3311202512728864,
+      "tokens_seen": 3222732800
+    },
+    {
+      "epoch": 0.95,
+      "objective/train/docs_used": 1080627,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1761735677719116,
+      "objective/train/theoretical_loss": 3.3311100734649006,
+      "objective/train/tokens_used": 1593322976,
+      "theoretical_loss": 3.3311100734649006,
+      "tokens_seen": 3222863872
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3631840796019903e-05,
+      "loss": 1.1545,
+      "theoretical_loss": 3.3311100734649006,
+      "tokens_seen": 3222863872
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3591718825228695e-05,
+      "loss": 1.1873,
+      "theoretical_loss": 3.3310998961867258,
+      "tokens_seen": 3222994944
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.355159685443749e-05,
+      "loss": 1.1341,
+      "theoretical_loss": 3.3310897194383124,
+      "tokens_seen": 3223126016
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3511474883646288e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.331079543219612,
+      "tokens_seen": 3223257088
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.347135291285508e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.331069367530575,
+      "tokens_seen": 3223388160
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3431230942063876e-05,
+      "loss": 1.1735,
+      "theoretical_loss": 3.3310591923711526,
+      "tokens_seen": 3223519232
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.339110897127267e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3310490177412952,
+      "tokens_seen": 3223650304
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3350987000481462e-05,
+      "loss": 1.1668,
+      "theoretical_loss": 3.3310388436409544,
+      "tokens_seen": 3223781376
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3310865029690258e-05,
+      "loss": 1.3127,
+      "theoretical_loss": 3.3310286700700806,
+      "tokens_seen": 3223912448
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3270743058899054e-05,
+      "loss": 1.3428,
+      "theoretical_loss": 3.331018497028625,
+      "tokens_seen": 3224043520
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3230621088107847e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.3310083245165383,
+      "tokens_seen": 3224174592
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3190499117316643e-05,
+      "loss": 1.1792,
+      "theoretical_loss": 3.3309981525337715,
+      "tokens_seen": 3224305664
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.315037714652544e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.330987981080276,
+      "tokens_seen": 3224436736
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3110255175734232e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3309778101560017,
+      "tokens_seen": 3224567808
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3070133204943028e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3309676397609005,
+      "tokens_seen": 3224698880
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3030011234151824e-05,
+      "loss": 1.1627,
+      "theoretical_loss": 3.330957469894923,
+      "tokens_seen": 3224829952
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2989889263360617e-05,
+      "loss": 1.141,
+      "theoretical_loss": 3.3309473005580204,
+      "tokens_seen": 3224961024
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2949767292569413e-05,
+      "loss": 1.219,
+      "theoretical_loss": 3.3309371317501437,
+      "tokens_seen": 3225092096
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2909645321778206e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.330926963471243,
+      "tokens_seen": 3225223168
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2869523350987e-05,
+      "loss": 1.2747,
+      "theoretical_loss": 3.3309167957212704,
+      "tokens_seen": 3225354240
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2829401380195795e-05,
+      "loss": 1.1654,
+      "theoretical_loss": 3.3309066285001765,
+      "tokens_seen": 3225485312
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.278927940940459e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.330896461807912,
+      "tokens_seen": 3225616384
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2749157438613384e-05,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3308862956444285,
+      "tokens_seen": 3225747456
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.270903546782218e-05,
+      "loss": 1.1863,
+      "theoretical_loss": 3.3308761300096763,
+      "tokens_seen": 3225878528
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2668913497030976e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.330865964903607,
+      "tokens_seen": 3226009600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.262879152623977e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.3308558003261712,
+      "tokens_seen": 3226140672
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2588669555448565e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.33084563627732,
+      "tokens_seen": 3226271744
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.254854758465736e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.330835472757005,
+      "tokens_seen": 3226402816
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2508425613866154e-05,
+      "loss": 1.2069,
+      "theoretical_loss": 3.330825309765177,
+      "tokens_seen": 3226533888
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.246830364307495e-05,
+      "loss": 1.0998,
+      "theoretical_loss": 3.3308151473017866,
+      "tokens_seen": 3226664960
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2428181672283743e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.330804985366785,
+      "tokens_seen": 3226796032
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2388059701492536e-05,
+      "loss": 1.197,
+      "theoretical_loss": 3.3307948239601237,
+      "tokens_seen": 3226927104
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.234793773070133e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3307846630817535,
+      "tokens_seen": 3227058176
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2307815759910128e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.330774502731625,
+      "tokens_seen": 3227189248
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.226769378911892e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3307643429096903,
+      "tokens_seen": 3227320320
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2227571818327717e-05,
+      "loss": 1.1887,
+      "theoretical_loss": 3.3307541836159,
+      "tokens_seen": 3227451392
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2187449847536513e-05,
+      "loss": 1.1762,
+      "theoretical_loss": 3.3307440248502047,
+      "tokens_seen": 3227582464
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2147327876745306e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.3307338666125563,
+      "tokens_seen": 3227713536
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2107205905954102e-05,
+      "loss": 1.1676,
+      "theoretical_loss": 3.330723708902905,
+      "tokens_seen": 3227844608
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2067083935162898e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.330713551721203,
+      "tokens_seen": 3227975680
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.202696196437169e-05,
+      "loss": 1.1301,
+      "theoretical_loss": 3.3307033950674008,
+      "tokens_seen": 3228106752
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1986839993580487e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.33069323894145,
+      "tokens_seen": 3228237824
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.194671802278928e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.3306830833433008,
+      "tokens_seen": 3228368896
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1906596051998072e-05,
+      "loss": 1.1784,
+      "theoretical_loss": 3.330672928272905,
+      "tokens_seen": 3228499968
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.186647408120687e-05,
+      "loss": 1.2078,
+      "theoretical_loss": 3.3306627737302135,
+      "tokens_seen": 3228631040
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1826352110415665e-05,
+      "loss": 1.2318,
+      "theoretical_loss": 3.330652619715178,
+      "tokens_seen": 3228762112
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1786230139624457e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.330642466227749,
+      "tokens_seen": 3228893184
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1746108168833253e-05,
+      "loss": 1.1868,
+      "theoretical_loss": 3.3306323132678783,
+      "tokens_seen": 3229024256
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.170598619804205e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.3306221608355164,
+      "tokens_seen": 3229155328
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1665864227250842e-05,
+      "loss": 1.3456,
+      "theoretical_loss": 3.330612008930615,
+      "tokens_seen": 3229286400
+    },
+    {
+      "epoch": 0.96,
+      "objective/train/docs_used": 1082871,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2117602825164795,
+      "objective/train/theoretical_loss": 3.330601857553125,
+      "objective/train/tokens_used": 1599876576,
+      "theoretical_loss": 3.330601857553125,
+      "tokens_seen": 3229417472
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.162574225645964e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.330601857553125,
+      "tokens_seen": 3229417472
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1585620285668435e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.3305917067029975,
+      "tokens_seen": 3229548544
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1545498314877227e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.330581556380184,
+      "tokens_seen": 3229679616
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1505376344086024e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.330571406584636,
+      "tokens_seen": 3229810688
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1465254373294816e-05,
+      "loss": 1.3268,
+      "theoretical_loss": 3.3305612573163037,
+      "tokens_seen": 3229941760
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.142513240250361e-05,
+      "loss": 1.1443,
+      "theoretical_loss": 3.3305511085751394,
+      "tokens_seen": 3230072832
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1385010431712405e-05,
+      "loss": 1.143,
+      "theoretical_loss": 3.330540960361094,
+      "tokens_seen": 3230203904
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.13448884609212e-05,
+      "loss": 1.2135,
+      "theoretical_loss": 3.3305308126741178,
+      "tokens_seen": 3230334976
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1304766490129994e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.3305206655141637,
+      "tokens_seen": 3230466048
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.126464451933879e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3305105188811814,
+      "tokens_seen": 3230597120
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1224522548547586e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.3305003727751235,
+      "tokens_seen": 3230728192
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.118440057775638e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.3304902271959405,
+      "tokens_seen": 3230859264
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1144278606965175e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.330480082143583,
+      "tokens_seen": 3230990336
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.110415663617397e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.330469937618004,
+      "tokens_seen": 3231121408
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1064034665382764e-05,
+      "loss": 1.1336,
+      "theoretical_loss": 3.3304597936191533,
+      "tokens_seen": 3231252480
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1023912694591557e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.3304496501469827,
+      "tokens_seen": 3231383552
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0983790723800353e-05,
+      "loss": 1.1975,
+      "theoretical_loss": 3.3304395072014437,
+      "tokens_seen": 3231514624
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0943668753009146e-05,
+      "loss": 1.1672,
+      "theoretical_loss": 3.3304293647824874,
+      "tokens_seen": 3231645696
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0903546782217942e-05,
+      "loss": 1.236,
+      "theoretical_loss": 3.330419222890065,
+      "tokens_seen": 3231776768
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0863424811426738e-05,
+      "loss": 1.1818,
+      "theoretical_loss": 3.3304090815241283,
+      "tokens_seen": 3231907840
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.082330284063553e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.3303989406846277,
+      "tokens_seen": 3232038912
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0783180869844327e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3303888003715154,
+      "tokens_seen": 3232169984
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0743058899053123e-05,
+      "loss": 1.1896,
+      "theoretical_loss": 3.330378660584742,
+      "tokens_seen": 3232301056
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0702936928261916e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.3303685213242598,
+      "tokens_seen": 3232432128
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0662814957470712e-05,
+      "loss": 1.2875,
+      "theoretical_loss": 3.330358382590019,
+      "tokens_seen": 3232563200
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0622692986679508e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.3303482443819723,
+      "tokens_seen": 3232694272
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.05825710158883e-05,
+      "loss": 1.1441,
+      "theoretical_loss": 3.3303381067000695,
+      "tokens_seen": 3232825344
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0542449045097094e-05,
+      "loss": 1.1574,
+      "theoretical_loss": 3.330327969544263,
+      "tokens_seen": 3232956416
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.050232707430589e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.330317832914504,
+      "tokens_seen": 3233087488
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0462205103514683e-05,
+      "loss": 1.1593,
+      "theoretical_loss": 3.330307696810744,
+      "tokens_seen": 3233218560
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.042208313272348e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.3302975612329337,
+      "tokens_seen": 3233349632
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0381961161932275e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.330287426181026,
+      "tokens_seen": 3233480704
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0341839191141068e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.33027729165497,
+      "tokens_seen": 3233611776
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0301717220349864e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.3302671576547187,
+      "tokens_seen": 3233742848
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.026159524955866e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.3302570241802236,
+      "tokens_seen": 3233873920
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0221473278767453e-05,
+      "loss": 1.121,
+      "theoretical_loss": 3.3302468912314356,
+      "tokens_seen": 3234004992
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.018135130797625e-05,
+      "loss": 1.1962,
+      "theoretical_loss": 3.330236758808306,
+      "tokens_seen": 3234136064
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0141229337185045e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.330226626910787,
+      "tokens_seen": 3234267136
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0101107366393838e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3302164955388287,
+      "tokens_seen": 3234398208
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.006098539560263e-05,
+      "loss": 1.1756,
+      "theoretical_loss": 3.330206364692384,
+      "tokens_seen": 3234529280
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0020863424811427e-05,
+      "loss": 1.2111,
+      "theoretical_loss": 3.330196234371403,
+      "tokens_seen": 3234660352
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.998074145402022e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.3301861045758385,
+      "tokens_seen": 3234791424
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9940619483229015e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.330175975305641,
+      "tokens_seen": 3234922496
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.990049751243781e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.330165846560762,
+      "tokens_seen": 3235053568
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9860375541646604e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.3301557183411536,
+      "tokens_seen": 3235184640
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.98202535708554e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.330145590646767,
+      "tokens_seen": 3235315712
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9780131600064197e-05,
+      "loss": 1.1975,
+      "theoretical_loss": 3.3301354634775535,
+      "tokens_seen": 3235446784
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9740009629272993e-05,
+      "loss": 1.1239,
+      "theoretical_loss": 3.3301253368334645,
+      "tokens_seen": 3235577856
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9699887658481786e-05,
+      "loss": 1.1596,
+      "theoretical_loss": 3.3301152107144514,
+      "tokens_seen": 3235708928
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.965976568769058e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.3301050851204668,
+      "tokens_seen": 3235840000
+    },
+    {
+      "epoch": 0.96,
+      "objective/train/docs_used": 1085025,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2300143241882324,
+      "objective/train/theoretical_loss": 3.330094960051461,
+      "objective/train/tokens_used": 1606430176,
+      "theoretical_loss": 3.330094960051461,
+      "tokens_seen": 3235971072
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9619643716899378e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.330094960051461,
+      "tokens_seen": 3235971072
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9579521746108167e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.3300848355073858,
+      "tokens_seen": 3236102144
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9539399775316963e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.330074711488193,
+      "tokens_seen": 3236233216
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.949927780452576e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.3300645879938338,
+      "tokens_seen": 3236364288
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9459155833734552e-05,
+      "loss": 1.1698,
+      "theoretical_loss": 3.3300544650242605,
+      "tokens_seen": 3236495360
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.941903386294335e-05,
+      "loss": 1.1156,
+      "theoretical_loss": 3.3300443425794235,
+      "tokens_seen": 3236626432
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9378911892152144e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.3300342206592752,
+      "tokens_seen": 3236757504
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9338789921360937e-05,
+      "loss": 1.1701,
+      "theoretical_loss": 3.330024099263767,
+      "tokens_seen": 3236888576
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9298667950569733e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.3300139783928495,
+      "tokens_seen": 3237019648
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.925854597977853e-05,
+      "loss": 1.1167,
+      "theoretical_loss": 3.330003858046476,
+      "tokens_seen": 3237150720
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9218424008987322e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.329993738224597,
+      "tokens_seen": 3237281792
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.917830203819612e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.329983618927164,
+      "tokens_seen": 3237412864
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.913818006740491e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.3299735001541295,
+      "tokens_seen": 3237543936
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9098058096613704e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.329963381905444,
+      "tokens_seen": 3237675008
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.90579361258225e-05,
+      "loss": 1.1393,
+      "theoretical_loss": 3.3299532641810594,
+      "tokens_seen": 3237806080
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9017814155031296e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.329943146980928,
+      "tokens_seen": 3237937152
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.897769218424009e-05,
+      "loss": 1.1387,
+      "theoretical_loss": 3.3299330303050003,
+      "tokens_seen": 3238068224
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8937570213448885e-05,
+      "loss": 1.1748,
+      "theoretical_loss": 3.329922914153229,
+      "tokens_seen": 3238199296
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.889744824265768e-05,
+      "loss": 1.1745,
+      "theoretical_loss": 3.329912798525565,
+      "tokens_seen": 3238330368
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8857326271866474e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.32990268342196,
+      "tokens_seen": 3238461440
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.881720430107527e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.329892568842366,
+      "tokens_seen": 3238592512
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8777082330284066e-05,
+      "loss": 1.1605,
+      "theoretical_loss": 3.3298824547867345,
+      "tokens_seen": 3238723584
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.873696035949286e-05,
+      "loss": 1.1462,
+      "theoretical_loss": 3.329872341255017,
+      "tokens_seen": 3238854656
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8696838388701655e-05,
+      "loss": 1.1799,
+      "theoretical_loss": 3.3298622282471655,
+      "tokens_seen": 3238985728
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8656716417910448e-05,
+      "loss": 1.17,
+      "theoretical_loss": 3.3298521157631313,
+      "tokens_seen": 3239116800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.861659444711924e-05,
+      "loss": 1.1745,
+      "theoretical_loss": 3.3298420038028667,
+      "tokens_seen": 3239247872
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8576472476328037e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.329831892366322,
+      "tokens_seen": 3239378944
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8536350505536833e-05,
+      "loss": 1.1867,
+      "theoretical_loss": 3.3298217814534503,
+      "tokens_seen": 3239510016
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8496228534745626e-05,
+      "loss": 1.1537,
+      "theoretical_loss": 3.3298116710642027,
+      "tokens_seen": 3239641088
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8456106563954422e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.329801561198531,
+      "tokens_seen": 3239772160
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8415984593163218e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.329791451856387,
+      "tokens_seen": 3239903232
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.837586262237201e-05,
+      "loss": 1.139,
+      "theoretical_loss": 3.3297813430377223,
+      "tokens_seen": 3240034304
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8335740651580807e-05,
+      "loss": 1.135,
+      "theoretical_loss": 3.3297712347424886,
+      "tokens_seen": 3240165376
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8295618680789603e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.329761126970637,
+      "tokens_seen": 3240296448
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8255496709998396e-05,
+      "loss": 1.1724,
+      "theoretical_loss": 3.3297510197221207,
+      "tokens_seen": 3240427520
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8215374739207192e-05,
+      "loss": 1.1587,
+      "theoretical_loss": 3.3297409129968902,
+      "tokens_seen": 3240558592
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8175252768415985e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.3297308067948976,
+      "tokens_seen": 3240689664
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8135130797624777e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.329720701116095,
+      "tokens_seen": 3240820736
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8095008826833574e-05,
+      "loss": 1.1124,
+      "theoretical_loss": 3.329710595960434,
+      "tokens_seen": 3240951808
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.805488685604237e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.3297004913278654,
+      "tokens_seen": 3241082880
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8014764885251163e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.3296903872183425,
+      "tokens_seen": 3241213952
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.797464291445996e-05,
+      "loss": 1.303,
+      "theoretical_loss": 3.329680283631816,
+      "tokens_seen": 3241345024
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.7934520943668755e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.3296701805682383,
+      "tokens_seen": 3241476096
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.7894398972877548e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.329660078027561,
+      "tokens_seen": 3241607168
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.7854277002086344e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.3296499760097356,
+      "tokens_seen": 3241738240
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.781415503129514e-05,
+      "loss": 1.1251,
+      "theoretical_loss": 3.329639874514714,
+      "tokens_seen": 3241869312
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.7774033060503933e-05,
+      "loss": 1.1703,
+      "theoretical_loss": 3.3296297735424485,
+      "tokens_seen": 3242000384
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.773391108971273e-05,
+      "loss": 1.1883,
+      "theoretical_loss": 3.3296196730928904,
+      "tokens_seen": 3242131456
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.769378911892152e-05,
+      "loss": 1.1481,
+      "theoretical_loss": 3.3296095731659916,
+      "tokens_seen": 3242262528
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7653667148130314e-05,
+      "loss": 1.1467,
+      "theoretical_loss": 3.329599473761704,
+      "tokens_seen": 3242393600
+    },
+    {
+      "epoch": 0.97,
+      "objective/train/docs_used": 1087175,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2063442468643188,
+      "objective/train/theoretical_loss": 3.32958937487998,
+      "objective/train/tokens_used": 1612983776,
+      "theoretical_loss": 3.32958937487998,
+      "tokens_seen": 3242524672
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.761354517733911e-05,
+      "loss": 1.1812,
+      "theoretical_loss": 3.32958937487998,
+      "tokens_seen": 3242524672
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7573423206547907e-05,
+      "loss": 1.1577,
+      "theoretical_loss": 3.32957927652077,
+      "tokens_seen": 3242655744
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.75333012357567e-05,
+      "loss": 1.1527,
+      "theoretical_loss": 3.329569178684027,
+      "tokens_seen": 3242786816
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7493179264965495e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3295590813697027,
+      "tokens_seen": 3242917888
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.745305729417429e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.329548984577749,
+      "tokens_seen": 3243048960
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7412935323383084e-05,
+      "loss": 1.197,
+      "theoretical_loss": 3.329538888308117,
+      "tokens_seen": 3243180032
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.737281335259188e-05,
+      "loss": 1.1717,
+      "theoretical_loss": 3.32952879256076,
+      "tokens_seen": 3243311104
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7332691381800677e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.3295186973356286,
+      "tokens_seen": 3243442176
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.729256941100947e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.329508602632675,
+      "tokens_seen": 3243573248
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7252447440218265e-05,
+      "loss": 1.2184,
+      "theoretical_loss": 3.3294985084518514,
+      "tokens_seen": 3243704320
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7212325469427058e-05,
+      "loss": 1.1586,
+      "theoretical_loss": 3.3294884147931096,
+      "tokens_seen": 3243835392
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.717220349863585e-05,
+      "loss": 1.1396,
+      "theoretical_loss": 3.329478321656401,
+      "tokens_seen": 3243966464
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7132081527844647e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3294682290416784,
+      "tokens_seen": 3244097536
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7091959557053443e-05,
+      "loss": 1.1823,
+      "theoretical_loss": 3.329458136948893,
+      "tokens_seen": 3244228608
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7051837586262236e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.329448045377997,
+      "tokens_seen": 3244359680
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7011715615471032e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.3294379543289425,
+      "tokens_seen": 3244490752
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.697159364467983e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.329427863801681,
+      "tokens_seen": 3244621824
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.693147167388862e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.3294177737961648,
+      "tokens_seen": 3244752896
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6891349703097417e-05,
+      "loss": 1.1058,
+      "theoretical_loss": 3.3294076843123457,
+      "tokens_seen": 3244883968
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6851227732306213e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3293975953501755,
+      "tokens_seen": 3245015040
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6811105761515006e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3293875069096064,
+      "tokens_seen": 3245146112
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.67709837907238e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.3293774189905907,
+      "tokens_seen": 3245277184
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6730861819932595e-05,
+      "loss": 1.1799,
+      "theoretical_loss": 3.3293673315930796,
+      "tokens_seen": 3245408256
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6690739849141388e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.3293572447170257,
+      "tokens_seen": 3245539328
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6650617878350184e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.3293471583623804,
+      "tokens_seen": 3245670400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.661049590755898e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.3293370725290963,
+      "tokens_seen": 3245801472
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6570373936767773e-05,
+      "loss": 1.1718,
+      "theoretical_loss": 3.329326987217125,
+      "tokens_seen": 3245932544
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.653025196597657e-05,
+      "loss": 1.1589,
+      "theoretical_loss": 3.329316902426419,
+      "tokens_seen": 3246063616
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6490129995185365e-05,
+      "loss": 1.2139,
+      "theoretical_loss": 3.3293068181569296,
+      "tokens_seen": 3246194688
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6450008024394158e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.3292967344086093,
+      "tokens_seen": 3246325760
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6409886053602954e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.32928665118141,
+      "tokens_seen": 3246456832
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.636976408281175e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3292765684752834,
+      "tokens_seen": 3246587904
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6329642112020543e-05,
+      "loss": 1.2156,
+      "theoretical_loss": 3.3292664862901824,
+      "tokens_seen": 3246718976
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6289520141229336e-05,
+      "loss": 1.1504,
+      "theoretical_loss": 3.329256404626058,
+      "tokens_seen": 3246850048
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6249398170438132e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.329246323482863,
+      "tokens_seen": 3246981120
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6209276199646925e-05,
+      "loss": 1.1585,
+      "theoretical_loss": 3.3292362428605493,
+      "tokens_seen": 3247112192
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.616915422885572e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.3292261627590687,
+      "tokens_seen": 3247243264
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6129032258064517e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.3292160831783737,
+      "tokens_seen": 3247374336
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.608891028727331e-05,
+      "loss": 1.1902,
+      "theoretical_loss": 3.329206004118416,
+      "tokens_seen": 3247505408
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6048788316482106e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.329195925579148,
+      "tokens_seen": 3247636480
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6008666345690902e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.329185847560521,
+      "tokens_seen": 3247767552
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5968544374899695e-05,
+      "loss": 1.1783,
+      "theoretical_loss": 3.3291757700624887,
+      "tokens_seen": 3247898624
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.592842240410849e-05,
+      "loss": 1.1556,
+      "theoretical_loss": 3.329165693085001,
+      "tokens_seen": 3248029696
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5888300433317287e-05,
+      "loss": 1.1577,
+      "theoretical_loss": 3.329155616628012,
+      "tokens_seen": 3248160768
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.584817846252608e-05,
+      "loss": 1.1485,
+      "theoretical_loss": 3.3291455406914725,
+      "tokens_seen": 3248291840
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5808056491734872e-05,
+      "loss": 1.1707,
+      "theoretical_loss": 3.3291354652753355,
+      "tokens_seen": 3248422912
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.576793452094367e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.329125390379553,
+      "tokens_seen": 3248553984
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.572781255015246e-05,
+      "loss": 1.1813,
+      "theoretical_loss": 3.3291153160040765,
+      "tokens_seen": 3248685056
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5687690579361257e-05,
+      "loss": 1.1253,
+      "theoretical_loss": 3.3291052421488585,
+      "tokens_seen": 3248816128
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5647568608570054e-05,
+      "loss": 1.1725,
+      "theoretical_loss": 3.329095168813851,
+      "tokens_seen": 3248947200
+    },
+    {
+      "epoch": 0.97,
+      "objective/train/docs_used": 1089234,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2247605323791504,
+      "objective/train/theoretical_loss": 3.3290850959990066,
+      "objective/train/tokens_used": 1619537376,
+      "theoretical_loss": 3.3290850959990066,
+      "tokens_seen": 3249078272
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5607446637778846e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.3290850959990066,
+      "tokens_seen": 3249078272
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5567324666987642e-05,
+      "loss": 1.153,
+      "theoretical_loss": 3.329075023704277,
+      "tokens_seen": 3249209344
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.552720269619644e-05,
+      "loss": 1.1292,
+      "theoretical_loss": 3.329064951929615,
+      "tokens_seen": 3249340416
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.548708072540523e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.329054880674972,
+      "tokens_seen": 3249471488
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5446958754614027e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.3290448099403003,
+      "tokens_seen": 3249602560
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.540683678382282e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.3290347397255524,
+      "tokens_seen": 3249733632
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5366714813031616e-05,
+      "loss": 1.1712,
+      "theoretical_loss": 3.3290246700306803,
+      "tokens_seen": 3249864704
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5326592842240413e-05,
+      "loss": 1.1435,
+      "theoretical_loss": 3.3290146008556363,
+      "tokens_seen": 3249995776
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5286470871449205e-05,
+      "loss": 1.1542,
+      "theoretical_loss": 3.329004532200373,
+      "tokens_seen": 3250126848
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5246348900658001e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.3289944640648415,
+      "tokens_seen": 3250257920
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5206226929866794e-05,
+      "loss": 1.211,
+      "theoretical_loss": 3.3289843964489947,
+      "tokens_seen": 3250388992
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.516610495907559e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.3289743293527856,
+      "tokens_seen": 3250520064
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5125982988284385e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.328964262776165,
+      "tokens_seen": 3250651136
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.508586101749318e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.328954196719086,
+      "tokens_seen": 3250782208
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5045739046701975e-05,
+      "loss": 1.3133,
+      "theoretical_loss": 3.3289441311815002,
+      "tokens_seen": 3250913280
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.500561707591077e-05,
+      "loss": 1.174,
+      "theoretical_loss": 3.3289340661633604,
+      "tokens_seen": 3251044352
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4965495105119563e-05,
+      "loss": 1.2852,
+      "theoretical_loss": 3.328924001664619,
+      "tokens_seen": 3251175424
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4925373134328359e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.3289139376852277,
+      "tokens_seen": 3251306496
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4885251163537153e-05,
+      "loss": 1.18,
+      "theoretical_loss": 3.328903874225139,
+      "tokens_seen": 3251437568
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4845129192745948e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3288938112843054,
+      "tokens_seen": 3251568640
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4805007221954744e-05,
+      "loss": 1.0579,
+      "theoretical_loss": 3.328883748862679,
+      "tokens_seen": 3251699712
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4764885251163536e-05,
+      "loss": 1.1865,
+      "theoretical_loss": 3.3288736869602116,
+      "tokens_seen": 3251830784
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4724763280372331e-05,
+      "loss": 1.1515,
+      "theoretical_loss": 3.3288636255768562,
+      "tokens_seen": 3251961856
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4684641309581127e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.328853564712565,
+      "tokens_seen": 3252092928
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4644519338789922e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.32884350436729,
+      "tokens_seen": 3252224000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4604397367998716e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.328833444540984,
+      "tokens_seen": 3252355072
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4564275397207512e-05,
+      "loss": 1.1537,
+      "theoretical_loss": 3.3288233852335987,
+      "tokens_seen": 3252486144
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4524153426416305e-05,
+      "loss": 1.1477,
+      "theoretical_loss": 3.3288133264450863,
+      "tokens_seen": 3252617216
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.44840314556251e-05,
+      "loss": 1.1412,
+      "theoretical_loss": 3.3288032681754,
+      "tokens_seen": 3252748288
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4443909484833895e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3287932104244917,
+      "tokens_seen": 3252879360
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.440378751404269e-05,
+      "loss": 1.1401,
+      "theoretical_loss": 3.3287831531923135,
+      "tokens_seen": 3253010432
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4363665543251484e-05,
+      "loss": 1.2156,
+      "theoretical_loss": 3.328773096478818,
+      "tokens_seen": 3253141504
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.432354357246028e-05,
+      "loss": 1.1395,
+      "theoretical_loss": 3.3287630402839574,
+      "tokens_seen": 3253272576
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4283421601669073e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.3287529846076844,
+      "tokens_seen": 3253403648
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4243299630877868e-05,
+      "loss": 1.1672,
+      "theoretical_loss": 3.328742929449951,
+      "tokens_seen": 3253534720
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4203177660086664e-05,
+      "loss": 1.1617,
+      "theoretical_loss": 3.3287328748107097,
+      "tokens_seen": 3253665792
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4163055689295458e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.328722820689913,
+      "tokens_seen": 3253796864
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4122933718504254e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.328712767087513,
+      "tokens_seen": 3253927936
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4082811747713049e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.328702714003462,
+      "tokens_seen": 3254059008
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4042689776921842e-05,
+      "loss": 1.1514,
+      "theoretical_loss": 3.328692661437713,
+      "tokens_seen": 3254190080
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4002567806130638e-05,
+      "loss": 1.2135,
+      "theoretical_loss": 3.328682609390218,
+      "tokens_seen": 3254321152
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3962445835339432e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.3286725578609295,
+      "tokens_seen": 3254452224
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3922323864548227e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.3286625068498,
+      "tokens_seen": 3254583296
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3882201893757023e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.3286524563567816,
+      "tokens_seen": 3254714368
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3842079922965817e-05,
+      "loss": 1.1809,
+      "theoretical_loss": 3.3286424063818267,
+      "tokens_seen": 3254845440
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.380195795217461e-05,
+      "loss": 1.1374,
+      "theoretical_loss": 3.3286323569248886,
+      "tokens_seen": 3254976512
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3761835981383406e-05,
+      "loss": 1.2013,
+      "theoretical_loss": 3.3286223079859183,
+      "tokens_seen": 3255107584
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.37217140105922e-05,
+      "loss": 1.1642,
+      "theoretical_loss": 3.3286122595648697,
+      "tokens_seen": 3255238656
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3681592039800995e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.3286022116616945,
+      "tokens_seen": 3255369728
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3641470069009791e-05,
+      "loss": 1.1572,
+      "theoretical_loss": 3.328592164276345,
+      "tokens_seen": 3255500800
+    },
+    {
+      "epoch": 0.97,
+      "objective/train/docs_used": 1091411,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1839964389801025,
+      "objective/train/theoretical_loss": 3.328582117408774,
+      "objective/train/tokens_used": 1626090976,
+      "theoretical_loss": 3.328582117408774,
+      "tokens_seen": 3255631872
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3601348098218586e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.328582117408774,
+      "tokens_seen": 3255631872
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3561226127427378e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.328572071058934,
+      "tokens_seen": 3255762944
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3521104156636175e-05,
+      "loss": 1.1789,
+      "theoretical_loss": 3.3285620252267774,
+      "tokens_seen": 3255894016
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3480982185844969e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.3285519799122567,
+      "tokens_seen": 3256025088
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3440860215053763e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.3285419351153243,
+      "tokens_seen": 3256156160
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.340073824426256e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.328531890835933,
+      "tokens_seen": 3256287232
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3360616273471354e-05,
+      "loss": 1.2318,
+      "theoretical_loss": 3.3285218470740348,
+      "tokens_seen": 3256418304
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3320494302680147e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.3285118038295822,
+      "tokens_seen": 3256549376
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3280372331888943e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3285017611025283,
+      "tokens_seen": 3256680448
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3240250361097737e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.3284917188928254,
+      "tokens_seen": 3256811520
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3200128390306532e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.328481677200426,
+      "tokens_seen": 3256942592
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3160006419515328e-05,
+      "loss": 1.1325,
+      "theoretical_loss": 3.328471636025282,
+      "tokens_seen": 3257073664
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3119884448724122e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.3284615953673473,
+      "tokens_seen": 3257204736
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3079762477932915e-05,
+      "loss": 1.1903,
+      "theoretical_loss": 3.3284515552265734,
+      "tokens_seen": 3257335808
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3039640507141711e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.328441515602913,
+      "tokens_seen": 3257466880
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2999518536350506e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.328431476496319,
+      "tokens_seen": 3257597952
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.29593965655593e-05,
+      "loss": 1.1308,
+      "theoretical_loss": 3.3284214379067434,
+      "tokens_seen": 3257729024
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2919274594768096e-05,
+      "loss": 1.1463,
+      "theoretical_loss": 3.328411399834139,
+      "tokens_seen": 3257860096
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.287915262397689e-05,
+      "loss": 1.1627,
+      "theoretical_loss": 3.328401362278459,
+      "tokens_seen": 3257991168
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2839030653185684e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.3283913252396555,
+      "tokens_seen": 3258122240
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.279890868239448e-05,
+      "loss": 1.1789,
+      "theoretical_loss": 3.328381288717681,
+      "tokens_seen": 3258253312
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2758786711603274e-05,
+      "loss": 1.1681,
+      "theoretical_loss": 3.328371252712488,
+      "tokens_seen": 3258384384
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2718664740812069e-05,
+      "loss": 1.1761,
+      "theoretical_loss": 3.328361217224029,
+      "tokens_seen": 3258515456
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2678542770020865e-05,
+      "loss": 1.2362,
+      "theoretical_loss": 3.328351182252258,
+      "tokens_seen": 3258646528
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2638420799229657e-05,
+      "loss": 1.1997,
+      "theoretical_loss": 3.3283411477971256,
+      "tokens_seen": 3258777600
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2598298828438452e-05,
+      "loss": 1.1584,
+      "theoretical_loss": 3.328331113858585,
+      "tokens_seen": 3258908672
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2558176857647248e-05,
+      "loss": 1.1196,
+      "theoretical_loss": 3.32832108043659,
+      "tokens_seen": 3259039744
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2518054886856043e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.328311047531092,
+      "tokens_seen": 3259170816
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2477932916064837e-05,
+      "loss": 1.1863,
+      "theoretical_loss": 3.3283010151420442,
+      "tokens_seen": 3259301888
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2437810945273633e-05,
+      "loss": 1.1272,
+      "theoretical_loss": 3.328290983269399,
+      "tokens_seen": 3259432960
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2397688974482426e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.328280951913109,
+      "tokens_seen": 3259564032
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.235756700369122e-05,
+      "loss": 1.1836,
+      "theoretical_loss": 3.3282709210731274,
+      "tokens_seen": 3259695104
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2317445032900016e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.328260890749406,
+      "tokens_seen": 3259826176
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2277323062108811e-05,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3282508609418984,
+      "tokens_seen": 3259957248
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2237201091317605e-05,
+      "loss": 1.1187,
+      "theoretical_loss": 3.3282408316505565,
+      "tokens_seen": 3260088320
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2197079120526401e-05,
+      "loss": 1.1409,
+      "theoretical_loss": 3.3282308028753333,
+      "tokens_seen": 3260219392
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2156957149735194e-05,
+      "loss": 1.1614,
+      "theoretical_loss": 3.328220774616182,
+      "tokens_seen": 3260350464
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2116835178943989e-05,
+      "loss": 1.1514,
+      "theoretical_loss": 3.328210746873054,
+      "tokens_seen": 3260481536
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2076713208152785e-05,
+      "loss": 1.1235,
+      "theoretical_loss": 3.3282007196459036,
+      "tokens_seen": 3260612608
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.203659123736158e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.3281906929346823,
+      "tokens_seen": 3260743680
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1996469266570375e-05,
+      "loss": 1.1605,
+      "theoretical_loss": 3.328180666739343,
+      "tokens_seen": 3260874752
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.195634729577917e-05,
+      "loss": 1.1101,
+      "theoretical_loss": 3.328170641059839,
+      "tokens_seen": 3261005824
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1916225324987963e-05,
+      "loss": 1.1655,
+      "theoretical_loss": 3.3281606158961226,
+      "tokens_seen": 3261136896
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1876103354196759e-05,
+      "loss": 1.1578,
+      "theoretical_loss": 3.3281505912481464,
+      "tokens_seen": 3261267968
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1835981383405553e-05,
+      "loss": 1.1837,
+      "theoretical_loss": 3.328140567115864,
+      "tokens_seen": 3261399040
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1795859412614348e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.3281305434992268,
+      "tokens_seen": 3261530112
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1755737441823144e-05,
+      "loss": 1.1941,
+      "theoretical_loss": 3.3281205203981887,
+      "tokens_seen": 3261661184
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1715615471031938e-05,
+      "loss": 1.1864,
+      "theoretical_loss": 3.328110497812702,
+      "tokens_seen": 3261792256
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1675493500240731e-05,
+      "loss": 1.1845,
+      "theoretical_loss": 3.328100475742719,
+      "tokens_seen": 3261923328
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1635371529449527e-05,
+      "loss": 1.1682,
+      "theoretical_loss": 3.3280904541881933,
+      "tokens_seen": 3262054400
+    },
+    {
+      "epoch": 0.98,
+      "objective/train/docs_used": 1093665,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.083878993988037,
+      "objective/train/theoretical_loss": 3.3280804331490774,
+      "objective/train/tokens_used": 1632644576,
+      "theoretical_loss": 3.3280804331490774,
+      "tokens_seen": 3262185472
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1595249558658322e-05,
+      "loss": 1.1711,
+      "theoretical_loss": 3.3280804331490774,
+      "tokens_seen": 3262185472
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1555127587867116e-05,
+      "loss": 1.1811,
+      "theoretical_loss": 3.328070412625324,
+      "tokens_seen": 3262316544
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1515005617075912e-05,
+      "loss": 1.1112,
+      "theoretical_loss": 3.328060392616886,
+      "tokens_seen": 3262447616
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1474883646284707e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.328050373123716,
+      "tokens_seen": 3262578688
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.14347616754935e-05,
+      "loss": 1.1567,
+      "theoretical_loss": 3.3280403541457666,
+      "tokens_seen": 3262709760
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1394639704702296e-05,
+      "loss": 1.1384,
+      "theoretical_loss": 3.3280303356829917,
+      "tokens_seen": 3262840832
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.135451773391109e-05,
+      "loss": 1.1834,
+      "theoretical_loss": 3.328020317735343,
+      "tokens_seen": 3262971904
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1314395763119884e-05,
+      "loss": 1.1035,
+      "theoretical_loss": 3.3280103003027732,
+      "tokens_seen": 3263102976
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.127427379232868e-05,
+      "loss": 1.1657,
+      "theoretical_loss": 3.328000283385236,
+      "tokens_seen": 3263234048
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1234151821537475e-05,
+      "loss": 1.1838,
+      "theoretical_loss": 3.3279902669826837,
+      "tokens_seen": 3263365120
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1194029850746268e-05,
+      "loss": 1.1768,
+      "theoretical_loss": 3.32798025109507,
+      "tokens_seen": 3263496192
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1153907879955064e-05,
+      "loss": 1.123,
+      "theoretical_loss": 3.327970235722346,
+      "tokens_seen": 3263627264
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1113785909163858e-05,
+      "loss": 1.1329,
+      "theoretical_loss": 3.327960220864466,
+      "tokens_seen": 3263758336
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1073663938372653e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.3279502065213826,
+      "tokens_seen": 3263889408
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1033541967581449e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.327940192693048,
+      "tokens_seen": 3264020480
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0993419996790243e-05,
+      "loss": 1.1632,
+      "theoretical_loss": 3.327930179379416,
+      "tokens_seen": 3264151552
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0953298025999036e-05,
+      "loss": 1.101,
+      "theoretical_loss": 3.327920166580439,
+      "tokens_seen": 3264282624
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0913176055207832e-05,
+      "loss": 1.1511,
+      "theoretical_loss": 3.32791015429607,
+      "tokens_seen": 3264413696
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0873054084416627e-05,
+      "loss": 1.183,
+      "theoretical_loss": 3.3279001425262615,
+      "tokens_seen": 3264544768
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0832932113625421e-05,
+      "loss": 1.2112,
+      "theoretical_loss": 3.327890131270967,
+      "tokens_seen": 3264675840
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0792810142834217e-05,
+      "loss": 1.1855,
+      "theoretical_loss": 3.327880120530139,
+      "tokens_seen": 3264806912
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0752688172043012e-05,
+      "loss": 1.1896,
+      "theoretical_loss": 3.3278701103037305,
+      "tokens_seen": 3264937984
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0712566201251805e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.327860100591695,
+      "tokens_seen": 3265069056
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.06724442304606e-05,
+      "loss": 1.1852,
+      "theoretical_loss": 3.327850091393984,
+      "tokens_seen": 3265200128
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0632322259669395e-05,
+      "loss": 1.1772,
+      "theoretical_loss": 3.327840082710552,
+      "tokens_seen": 3265331200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.059220028887819e-05,
+      "loss": 1.1612,
+      "theoretical_loss": 3.3278300745413505,
+      "tokens_seen": 3265462272
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0552078318086986e-05,
+      "loss": 1.1837,
+      "theoretical_loss": 3.327820066886334,
+      "tokens_seen": 3265593344
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0511956347295778e-05,
+      "loss": 1.1972,
+      "theoretical_loss": 3.3278100597454543,
+      "tokens_seen": 3265724416
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0471834376504573e-05,
+      "loss": 1.178,
+      "theoretical_loss": 3.3278000531186644,
+      "tokens_seen": 3265855488
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0431712405713369e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3277900470059176,
+      "tokens_seen": 3265986560
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0391590434922163e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.327780041407167,
+      "tokens_seen": 3266117632
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0351468464130958e-05,
+      "loss": 1.145,
+      "theoretical_loss": 3.3277700363223657,
+      "tokens_seen": 3266248704
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0311346493339754e-05,
+      "loss": 1.1147,
+      "theoretical_loss": 3.327760031751466,
+      "tokens_seen": 3266379776
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0271224522548547e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.327750027694421,
+      "tokens_seen": 3266510848
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0231102551757341e-05,
+      "loss": 1.207,
+      "theoretical_loss": 3.327740024151184,
+      "tokens_seen": 3266641920
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0190980580966137e-05,
+      "loss": 1.1532,
+      "theoretical_loss": 3.3277300211217082,
+      "tokens_seen": 3266772992
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0150858610174932e-05,
+      "loss": 1.1771,
+      "theoretical_loss": 3.3277200186059464,
+      "tokens_seen": 3266904064
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0110736639383726e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.3277100166038514,
+      "tokens_seen": 3267035136
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0070614668592522e-05,
+      "loss": 1.1207,
+      "theoretical_loss": 3.327700015115376,
+      "tokens_seen": 3267166208
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0030492697801315e-05,
+      "loss": 1.1438,
+      "theoretical_loss": 3.327690014140474,
+      "tokens_seen": 3267297280
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.99037072701011e-06,
+      "loss": 1.175,
+      "theoretical_loss": 3.327680013679098,
+      "tokens_seen": 3267428352
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.950248756218906e-06,
+      "loss": 1.1796,
+      "theoretical_loss": 3.3276700137312005,
+      "tokens_seen": 3267559424
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.9101267854277e-06,
+      "loss": 1.1773,
+      "theoretical_loss": 3.3276600142967356,
+      "tokens_seen": 3267690496
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.870004814636496e-06,
+      "loss": 1.198,
+      "theoretical_loss": 3.3276500153756556,
+      "tokens_seen": 3267821568
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.82988284384529e-06,
+      "loss": 1.1807,
+      "theoretical_loss": 3.327640016967914,
+      "tokens_seen": 3267952640
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.789760873054084e-06,
+      "loss": 1.1891,
+      "theoretical_loss": 3.3276300190734633,
+      "tokens_seen": 3268083712
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.74963890226288e-06,
+      "loss": 1.2136,
+      "theoretical_loss": 3.327620021692257,
+      "tokens_seen": 3268214784
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.709516931471674e-06,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3276100248242484,
+      "tokens_seen": 3268345856
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.669394960680469e-06,
+      "loss": 1.1794,
+      "theoretical_loss": 3.32760002846939,
+      "tokens_seen": 3268476928
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.629272989889265e-06,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3275900326276346,
+      "tokens_seen": 3268608000
+    },
+    {
+      "epoch": 0.98,
+      "objective/train/docs_used": 1095836,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1877013444900513,
+      "objective/train/theoretical_loss": 3.327580037298936,
+      "objective/train/tokens_used": 1639198176,
+      "theoretical_loss": 3.327580037298936,
+      "tokens_seen": 3268739072
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.58915101909806e-06,
+      "loss": 1.1852,
+      "theoretical_loss": 3.327580037298936,
+      "tokens_seen": 3268739072
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.549029048306852e-06,
+      "loss": 1.2258,
+      "theoretical_loss": 3.3275700424832477,
+      "tokens_seen": 3268870144
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.508907077515648e-06,
+      "loss": 1.2238,
+      "theoretical_loss": 3.327560048180522,
+      "tokens_seen": 3269001216
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.468785106724443e-06,
+      "loss": 1.2344,
+      "theoretical_loss": 3.327550054390712,
+      "tokens_seen": 3269132288
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.428663135933237e-06,
+      "loss": 1.2332,
+      "theoretical_loss": 3.327540061113771,
+      "tokens_seen": 3269263360
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.388541165142033e-06,
+      "loss": 1.2271,
+      "theoretical_loss": 3.327530068349652,
+      "tokens_seen": 3269394432
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.348419194350828e-06,
+      "loss": 1.2224,
+      "theoretical_loss": 3.3275200760983084,
+      "tokens_seen": 3269525504
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.30829722355962e-06,
+      "loss": 1.1648,
+      "theoretical_loss": 3.3275100843596936,
+      "tokens_seen": 3269656576
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.268175252768416e-06,
+      "loss": 1.152,
+      "theoretical_loss": 3.32750009313376,
+      "tokens_seen": 3269787648
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.228053281977211e-06,
+      "loss": 1.1078,
+      "theoretical_loss": 3.327490102420461,
+      "tokens_seen": 3269918720
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.187931311186005e-06,
+      "loss": 1.1748,
+      "theoretical_loss": 3.32748011221975,
+      "tokens_seen": 3270049792
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.147809340394802e-06,
+      "loss": 1.1395,
+      "theoretical_loss": 3.3274701225315795,
+      "tokens_seen": 3270180864
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.107687369603596e-06,
+      "loss": 1.1906,
+      "theoretical_loss": 3.327460133355904,
+      "tokens_seen": 3270311936
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.067565398812389e-06,
+      "loss": 1.1417,
+      "theoretical_loss": 3.3274501446926754,
+      "tokens_seen": 3270443008
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.027443428021185e-06,
+      "loss": 1.2016,
+      "theoretical_loss": 3.327440156541847,
+      "tokens_seen": 3270574080
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.98732145722998e-06,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3274301689033727,
+      "tokens_seen": 3270705152
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.947199486438774e-06,
+      "loss": 1.1891,
+      "theoretical_loss": 3.327420181777205,
+      "tokens_seen": 3270836224
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.90707751564757e-06,
+      "loss": 1.1937,
+      "theoretical_loss": 3.3274101951632975,
+      "tokens_seen": 3270967296
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.866955544856364e-06,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3274002090616035,
+      "tokens_seen": 3271098368
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.826833574065157e-06,
+      "loss": 1.2014,
+      "theoretical_loss": 3.3273902234720754,
+      "tokens_seen": 3271229440
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.786711603273953e-06,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3273802383946673,
+      "tokens_seen": 3271360512
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.746589632482748e-06,
+      "loss": 1.2107,
+      "theoretical_loss": 3.327370253829332,
+      "tokens_seen": 3271491584
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.706467661691542e-06,
+      "loss": 1.1574,
+      "theoretical_loss": 3.327360269776023,
+      "tokens_seen": 3271622656
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.666345690900338e-06,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3273502862346933,
+      "tokens_seen": 3271753728
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.626223720109133e-06,
+      "loss": 1.2114,
+      "theoretical_loss": 3.327340303205296,
+      "tokens_seen": 3271884800
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.586101749317925e-06,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3273303206877847,
+      "tokens_seen": 3272015872
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.545979778526722e-06,
+      "loss": 1.1386,
+      "theoretical_loss": 3.327320338682113,
+      "tokens_seen": 3272146944
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.505857807735516e-06,
+      "loss": 1.1858,
+      "theoretical_loss": 3.3273103571882325,
+      "tokens_seen": 3272278016
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.46573583694431e-06,
+      "loss": 1.254,
+      "theoretical_loss": 3.3273003762060984,
+      "tokens_seen": 3272409088
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.425613866153107e-06,
+      "loss": 1.1831,
+      "theoretical_loss": 3.327290395735663,
+      "tokens_seen": 3272540160
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.3854918953619e-06,
+      "loss": 1.1353,
+      "theoretical_loss": 3.3272804157768796,
+      "tokens_seen": 3272671232
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.345369924570694e-06,
+      "loss": 1.201,
+      "theoretical_loss": 3.3272704363297017,
+      "tokens_seen": 3272802304
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.30524795377949e-06,
+      "loss": 1.2141,
+      "theoretical_loss": 3.3272604573940825,
+      "tokens_seen": 3272933376
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.265125982988284e-06,
+      "loss": 1.1651,
+      "theoretical_loss": 3.3272504789699755,
+      "tokens_seen": 3273064448
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.225004012197079e-06,
+      "loss": 1.1677,
+      "theoretical_loss": 3.3272405010573336,
+      "tokens_seen": 3273195520
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.184882041405875e-06,
+      "loss": 1.1396,
+      "theoretical_loss": 3.32723052365611,
+      "tokens_seen": 3273326592
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.144760070614668e-06,
+      "loss": 1.201,
+      "theoretical_loss": 3.3272205467662586,
+      "tokens_seen": 3273457664
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.104638099823462e-06,
+      "loss": 1.1813,
+      "theoretical_loss": 3.3272105703877326,
+      "tokens_seen": 3273588736
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.064516129032258e-06,
+      "loss": 1.1482,
+      "theoretical_loss": 3.3272005945204848,
+      "tokens_seen": 3273719808
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.024394158241053e-06,
+      "loss": 1.2299,
+      "theoretical_loss": 3.327190619164469,
+      "tokens_seen": 3273850880
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.984272187449847e-06,
+      "loss": 1.2202,
+      "theoretical_loss": 3.3271806443196383,
+      "tokens_seen": 3273981952
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.944150216658643e-06,
+      "loss": 1.251,
+      "theoretical_loss": 3.3271706699859465,
+      "tokens_seen": 3274113024
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.904028245867436e-06,
+      "loss": 1.1888,
+      "theoretical_loss": 3.3271606961633466,
+      "tokens_seen": 3274244096
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.86390627507623e-06,
+      "loss": 1.1298,
+      "theoretical_loss": 3.3271507228517914,
+      "tokens_seen": 3274375168
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.823784304285027e-06,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3271407500512353,
+      "tokens_seen": 3274506240
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.783662333493821e-06,
+      "loss": 1.2282,
+      "theoretical_loss": 3.327130777761631,
+      "tokens_seen": 3274637312
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.743540362702616e-06,
+      "loss": 1.1557,
+      "theoretical_loss": 3.327120805982932,
+      "tokens_seen": 3274768384
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.70341839191141e-06,
+      "loss": 1.1807,
+      "theoretical_loss": 3.327110834715092,
+      "tokens_seen": 3274899456
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.663296421120206e-06,
+      "loss": 1.1997,
+      "theoretical_loss": 3.327100863958064,
+      "tokens_seen": 3275030528
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.623174450329001e-06,
+      "loss": 1.1745,
+      "theoretical_loss": 3.327090893711801,
+      "tokens_seen": 3275161600
+    },
+    {
+      "epoch": 0.98,
+      "objective/train/docs_used": 1097820,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1068789958953857,
+      "objective/train/theoretical_loss": 3.3270809239762573,
+      "objective/train/tokens_used": 1645751776,
+      "theoretical_loss": 3.3270809239762573,
+      "tokens_seen": 3275292672
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.583052479537795e-06,
+      "loss": 1.1259,
+      "theoretical_loss": 3.3270809239762573,
+      "tokens_seen": 3275292672
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.54293050874659e-06,
+      "loss": 1.2403,
+      "theoretical_loss": 3.327070954751386,
+      "tokens_seen": 3275423744
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.502808537955385e-06,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3270609860371403,
+      "tokens_seen": 3275554816
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.462686567164179e-06,
+      "loss": 1.1759,
+      "theoretical_loss": 3.327051017833474,
+      "tokens_seen": 3275685888
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.422564596372974e-06,
+      "loss": 1.1715,
+      "theoretical_loss": 3.3270410501403394,
+      "tokens_seen": 3275816960
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.382442625581768e-06,
+      "loss": 1.2143,
+      "theoretical_loss": 3.3270310829576912,
+      "tokens_seen": 3275948032
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.3423206547905635e-06,
+      "loss": 1.1998,
+      "theoretical_loss": 3.327021116285483,
+      "tokens_seen": 3276079104
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.302198683999358e-06,
+      "loss": 1.1974,
+      "theoretical_loss": 3.327011150123667,
+      "tokens_seen": 3276210176
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.2620767132081524e-06,
+      "loss": 1.1906,
+      "theoretical_loss": 3.3270011844721976,
+      "tokens_seen": 3276341248
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.221954742416948e-06,
+      "loss": 1.1282,
+      "theoretical_loss": 3.3269912193310276,
+      "tokens_seen": 3276472320
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.181832771625742e-06,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3269812547001107,
+      "tokens_seen": 3276603392
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.141710800834537e-06,
+      "loss": 1.2014,
+      "theoretical_loss": 3.326971290579401,
+      "tokens_seen": 3276734464
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.101588830043332e-06,
+      "loss": 1.1929,
+      "theoretical_loss": 3.326961326968851,
+      "tokens_seen": 3276865536
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.061466859252127e-06,
+      "loss": 1.1547,
+      "theoretical_loss": 3.3269513638684147,
+      "tokens_seen": 3276996608
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.021344888460921e-06,
+      "loss": 1.1598,
+      "theoretical_loss": 3.3269414012780456,
+      "tokens_seen": 3277127680
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.981222917669716e-06,
+      "loss": 1.1482,
+      "theoretical_loss": 3.326931439197697,
+      "tokens_seen": 3277258752
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.941100946878511e-06,
+      "loss": 1.223,
+      "theoretical_loss": 3.326921477627322,
+      "tokens_seen": 3277389824
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.900978976087305e-06,
+      "loss": 1.1872,
+      "theoretical_loss": 3.3269115165668754,
+      "tokens_seen": 3277520896
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.8608570052961e-06,
+      "loss": 1.2739,
+      "theoretical_loss": 3.3269015560163093,
+      "tokens_seen": 3277651968
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.820735034504896e-06,
+      "loss": 1.2019,
+      "theoretical_loss": 3.326891595975578,
+      "tokens_seen": 3277783040
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.780613063713689e-06,
+      "loss": 1.2268,
+      "theoretical_loss": 3.326881636444635,
+      "tokens_seen": 3277914112
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.7404910929224845e-06,
+      "loss": 1.2271,
+      "theoretical_loss": 3.326871677423433,
+      "tokens_seen": 3278045184
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.70036912213128e-06,
+      "loss": 1.1962,
+      "theoretical_loss": 3.3268617189119265,
+      "tokens_seen": 3278176256
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.660247151340073e-06,
+      "loss": 1.2904,
+      "theoretical_loss": 3.3268517609100687,
+      "tokens_seen": 3278307328
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.620125180548869e-06,
+      "loss": 1.2206,
+      "theoretical_loss": 3.326841803417813,
+      "tokens_seen": 3278438400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.580003209757664e-06,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3268318464351134,
+      "tokens_seen": 3278569472
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.539881238966458e-06,
+      "loss": 1.2085,
+      "theoretical_loss": 3.326821889961923,
+      "tokens_seen": 3278700544
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.499759268175253e-06,
+      "loss": 1.2855,
+      "theoretical_loss": 3.3268119339981954,
+      "tokens_seen": 3278831616
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.459637297384048e-06,
+      "loss": 1.2265,
+      "theoretical_loss": 3.3268019785438847,
+      "tokens_seen": 3278962688
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.419515326592842e-06,
+      "loss": 1.1268,
+      "theoretical_loss": 3.3267920235989434,
+      "tokens_seen": 3279093760
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.379393355801637e-06,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3267820691633263,
+      "tokens_seen": 3279224832
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.339271385010432e-06,
+      "loss": 1.1979,
+      "theoretical_loss": 3.326772115236986,
+      "tokens_seen": 3279355904
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.299149414219226e-06,
+      "loss": 1.1744,
+      "theoretical_loss": 3.326762161819876,
+      "tokens_seen": 3279486976
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.259027443428021e-06,
+      "loss": 1.1982,
+      "theoretical_loss": 3.3267522089119512,
+      "tokens_seen": 3279618048
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.2189054726368165e-06,
+      "loss": 1.181,
+      "theoretical_loss": 3.3267422565131644,
+      "tokens_seen": 3279749120
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.17878350184561e-06,
+      "loss": 1.1833,
+      "theoretical_loss": 3.326732304623469,
+      "tokens_seen": 3279880192
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.1386615310544054e-06,
+      "loss": 1.1527,
+      "theoretical_loss": 3.3267223532428187,
+      "tokens_seen": 3280011264
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.098539560263201e-06,
+      "loss": 1.1806,
+      "theoretical_loss": 3.326712402371167,
+      "tokens_seen": 3280142336
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.058417589471994e-06,
+      "loss": 1.3159,
+      "theoretical_loss": 3.326702452008468,
+      "tokens_seen": 3280273408
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.01829561868079e-06,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3266925021546756,
+      "tokens_seen": 3280404480
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.978173647889585e-06,
+      "loss": 1.2544,
+      "theoretical_loss": 3.3266825528097423,
+      "tokens_seen": 3280535552
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.938051677098379e-06,
+      "loss": 1.1922,
+      "theoretical_loss": 3.3266726039736225,
+      "tokens_seen": 3280666624
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.897929706307174e-06,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3266626556462704,
+      "tokens_seen": 3280797696
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.857807735515969e-06,
+      "loss": 1.2143,
+      "theoretical_loss": 3.326652707827638,
+      "tokens_seen": 3280928768
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.8176857647247636e-06,
+      "loss": 1.1743,
+      "theoretical_loss": 3.3266427605176805,
+      "tokens_seen": 3281059840
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.777563793933558e-06,
+      "loss": 1.1627,
+      "theoretical_loss": 3.3266328137163512,
+      "tokens_seen": 3281190912
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.737441823142353e-06,
+      "loss": 1.2048,
+      "theoretical_loss": 3.326622867423603,
+      "tokens_seen": 3281321984
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.697319852351148e-06,
+      "loss": 1.2241,
+      "theoretical_loss": 3.326612921639391,
+      "tokens_seen": 3281453056
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.657197881559942e-06,
+      "loss": 1.1712,
+      "theoretical_loss": 3.3266029763636675,
+      "tokens_seen": 3281584128
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.6170759107687375e-06,
+      "loss": 1.2515,
+      "theoretical_loss": 3.3265930315963868,
+      "tokens_seen": 3281715200
+    },
+    {
+      "epoch": 0.99,
+      "objective/train/docs_used": 1099909,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1352242231369019,
+      "objective/train/theoretical_loss": 3.326583087337503,
+      "objective/train/tokens_used": 1652305376,
+      "theoretical_loss": 3.326583087337503,
+      "tokens_seen": 3281846272
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.576953939977532e-06,
+      "loss": 1.1836,
+      "theoretical_loss": 3.326583087337503,
+      "tokens_seen": 3281846272
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.536831969186326e-06,
+      "loss": 1.1465,
+      "theoretical_loss": 3.3265731435869688,
+      "tokens_seen": 3281977344
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.496709998395122e-06,
+      "loss": 1.1578,
+      "theoretical_loss": 3.3265632003447387,
+      "tokens_seen": 3282108416
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.456588027603916e-06,
+      "loss": 1.1937,
+      "theoretical_loss": 3.326553257610766,
+      "tokens_seen": 3282239488
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.416466056812711e-06,
+      "loss": 1.1894,
+      "theoretical_loss": 3.3265433153850053,
+      "tokens_seen": 3282370560
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.376344086021506e-06,
+      "loss": 1.1732,
+      "theoretical_loss": 3.3265333736674094,
+      "tokens_seen": 3282501632
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.3362221152303e-06,
+      "loss": 1.1779,
+      "theoretical_loss": 3.326523432457932,
+      "tokens_seen": 3282632704
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.296100144439095e-06,
+      "loss": 1.2682,
+      "theoretical_loss": 3.3265134917565273,
+      "tokens_seen": 3282763776
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.255978173647889e-06,
+      "loss": 1.2484,
+      "theoretical_loss": 3.3265035515631487,
+      "tokens_seen": 3282894848
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.2158562028566845e-06,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3264936118777504,
+      "tokens_seen": 3283025920
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.175734232065479e-06,
+      "loss": 1.2432,
+      "theoretical_loss": 3.326483672700286,
+      "tokens_seen": 3283156992
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.135612261274273e-06,
+      "loss": 1.1676,
+      "theoretical_loss": 3.326473734030709,
+      "tokens_seen": 3283288064
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.095490290483069e-06,
+      "loss": 1.1894,
+      "theoretical_loss": 3.326463795868974,
+      "tokens_seen": 3283419136
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.055368319691863e-06,
+      "loss": 1.2549,
+      "theoretical_loss": 3.326453858215033,
+      "tokens_seen": 3283550208
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.015246348900658e-06,
+      "loss": 1.1756,
+      "theoretical_loss": 3.3264439210688415,
+      "tokens_seen": 3283681280
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.975124378109453e-06,
+      "loss": 1.216,
+      "theoretical_loss": 3.326433984430353,
+      "tokens_seen": 3283812352
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.935002407318248e-06,
+      "loss": 1.2038,
+      "theoretical_loss": 3.3264240482995207,
+      "tokens_seen": 3283943424
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.894880436527042e-06,
+      "loss": 1.1589,
+      "theoretical_loss": 3.3264141126762983,
+      "tokens_seen": 3284074496
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.854758465735837e-06,
+      "loss": 1.1546,
+      "theoretical_loss": 3.3264041775606406,
+      "tokens_seen": 3284205568
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.814636494944632e-06,
+      "loss": 1.2192,
+      "theoretical_loss": 3.326394242952501,
+      "tokens_seen": 3284336640
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.774514524153426e-06,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3263843088518326,
+      "tokens_seen": 3284467712
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.734392553362221e-06,
+      "loss": 1.2099,
+      "theoretical_loss": 3.32637437525859,
+      "tokens_seen": 3284598784
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.6942705825710166e-06,
+      "loss": 1.173,
+      "theoretical_loss": 3.3263644421727268,
+      "tokens_seen": 3284729856
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.65414861177981e-06,
+      "loss": 1.2284,
+      "theoretical_loss": 3.326354509594197,
+      "tokens_seen": 3284860928
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.6140266409886055e-06,
+      "loss": 1.1553,
+      "theoretical_loss": 3.3263445775229545,
+      "tokens_seen": 3284992000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.573904670197401e-06,
+      "loss": 1.291,
+      "theoretical_loss": 3.3263346459589527,
+      "tokens_seen": 3285123072
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.533782699406194e-06,
+      "loss": 1.179,
+      "theoretical_loss": 3.3263247149021455,
+      "tokens_seen": 3285254144
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.49366072861499e-06,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3263147843524874,
+      "tokens_seen": 3285385216
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.453538757823785e-06,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3263048543099316,
+      "tokens_seen": 3285516288
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.4134167870325786e-06,
+      "loss": 1.2817,
+      "theoretical_loss": 3.326294924774432,
+      "tokens_seen": 3285647360
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.373294816241374e-06,
+      "loss": 1.2089,
+      "theoretical_loss": 3.326284995745943,
+      "tokens_seen": 3285778432
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.333172845450169e-06,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3262750672244183,
+      "tokens_seen": 3285909504
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.293050874658963e-06,
+      "loss": 1.1624,
+      "theoretical_loss": 3.3262651392098115,
+      "tokens_seen": 3286040576
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.252928903867758e-06,
+      "loss": 1.2047,
+      "theoretical_loss": 3.326255211702077,
+      "tokens_seen": 3286171648
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.212806933076553e-06,
+      "loss": 1.2029,
+      "theoretical_loss": 3.326245284701168,
+      "tokens_seen": 3286302720
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.172684962285347e-06,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3262353582070388,
+      "tokens_seen": 3286433792
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.132562991494142e-06,
+      "loss": 1.2538,
+      "theoretical_loss": 3.3262254322196436,
+      "tokens_seen": 3286564864
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.0924410207029375e-06,
+      "loss": 1.1885,
+      "theoretical_loss": 3.326215506738936,
+      "tokens_seen": 3286695936
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.052319049911731e-06,
+      "loss": 1.2224,
+      "theoretical_loss": 3.32620558176487,
+      "tokens_seen": 3286827008
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.012197079120526e-06,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3261956572973994,
+      "tokens_seen": 3286958080
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.972075108329322e-06,
+      "loss": 1.2207,
+      "theoretical_loss": 3.3261857333364784,
+      "tokens_seen": 3287089152
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.931953137538115e-06,
+      "loss": 1.1941,
+      "theoretical_loss": 3.3261758098820606,
+      "tokens_seen": 3287220224
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.891831166746911e-06,
+      "loss": 1.178,
+      "theoretical_loss": 3.3261658869341004,
+      "tokens_seen": 3287351296
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.851709195955705e-06,
+      "loss": 1.2713,
+      "theoretical_loss": 3.3261559644925516,
+      "tokens_seen": 3287482368
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.8115872251645004e-06,
+      "loss": 1.3234,
+      "theoretical_loss": 3.326146042557368,
+      "tokens_seen": 3287613440
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.771465254373295e-06,
+      "loss": 1.2354,
+      "theoretical_loss": 3.326136121128503,
+      "tokens_seen": 3287744512
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.7313432835820897e-06,
+      "loss": 1.2467,
+      "theoretical_loss": 3.326126200205912,
+      "tokens_seen": 3287875584
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.691221312790884e-06,
+      "loss": 1.2352,
+      "theoretical_loss": 3.3261162797895483,
+      "tokens_seen": 3288006656
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.651099341999679e-06,
+      "loss": 1.2158,
+      "theoretical_loss": 3.3261063598793656,
+      "tokens_seen": 3288137728
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.610977371208474e-06,
+      "loss": 1.2054,
+      "theoretical_loss": 3.326096440475318,
+      "tokens_seen": 3288268800
+    },
+    {
+      "epoch": 0.99,
+      "objective/train/docs_used": 1102163,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1918718814849854,
+      "objective/train/theoretical_loss": 3.3260865215773596,
+      "objective/train/tokens_used": 1658858976,
+      "theoretical_loss": 3.3260865215773596,
+      "tokens_seen": 3288399872
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.5708554004172683e-06,
+      "loss": 1.2124,
+      "theoretical_loss": 3.3260865215773596,
+      "tokens_seen": 3288399872
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.5307334296260636e-06,
+      "loss": 1.2497,
+      "theoretical_loss": 3.3260766031854447,
+      "tokens_seen": 3288530944
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.490611458834858e-06,
+      "loss": 1.2223,
+      "theoretical_loss": 3.326066685299527,
+      "tokens_seen": 3288662016
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.4504894880436525e-06,
+      "loss": 1.1354,
+      "theoretical_loss": 3.3260567679195603,
+      "tokens_seen": 3288793088
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.410367517252448e-06,
+      "loss": 1.2445,
+      "theoretical_loss": 3.3260468510454992,
+      "tokens_seen": 3288924160
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.3702455464612422e-06,
+      "loss": 1.2233,
+      "theoretical_loss": 3.326036934677297,
+      "tokens_seen": 3289055232
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.3301235756700367e-06,
+      "loss": 1.2224,
+      "theoretical_loss": 3.3260270188149086,
+      "tokens_seen": 3289186304
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.290001604878832e-06,
+      "loss": 1.1617,
+      "theoretical_loss": 3.3260171034582875,
+      "tokens_seen": 3289317376
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.2498796340876264e-06,
+      "loss": 1.2389,
+      "theoretical_loss": 3.3260071886073876,
+      "tokens_seen": 3289448448
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.209757663296421e-06,
+      "loss": 1.2019,
+      "theoretical_loss": 3.3259972742621637,
+      "tokens_seen": 3289579520
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.169635692505216e-06,
+      "loss": 1.1321,
+      "theoretical_loss": 3.325987360422569,
+      "tokens_seen": 3289710592
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.1295137217140106e-06,
+      "loss": 1.218,
+      "theoretical_loss": 3.325977447088558,
+      "tokens_seen": 3289841664
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.089391750922805e-06,
+      "loss": 1.2117,
+      "theoretical_loss": 3.325967534260085,
+      "tokens_seen": 3289972736
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.0492697801316004e-06,
+      "loss": 1.2512,
+      "theoretical_loss": 3.3259576219371034,
+      "tokens_seen": 3290103808
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.009147809340395e-06,
+      "loss": 1.2102,
+      "theoretical_loss": 3.325947710119568,
+      "tokens_seen": 3290234880
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.9690258385491897e-06,
+      "loss": 1.2251,
+      "theoretical_loss": 3.3259377988074323,
+      "tokens_seen": 3290365952
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.9289038677579846e-06,
+      "loss": 1.2616,
+      "theoretical_loss": 3.325927888000651,
+      "tokens_seen": 3290497024
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.888781896966779e-06,
+      "loss": 1.2622,
+      "theoretical_loss": 3.325917977699177,
+      "tokens_seen": 3290628096
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.848659926175574e-06,
+      "loss": 1.1699,
+      "theoretical_loss": 3.3259080679029664,
+      "tokens_seen": 3290759168
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.8085379553843687e-06,
+      "loss": 1.1331,
+      "theoretical_loss": 3.3258981586119716,
+      "tokens_seen": 3290890240
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.768415984593163e-06,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3258882498261477,
+      "tokens_seen": 3291021312
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.728294013801958e-06,
+      "loss": 1.1863,
+      "theoretical_loss": 3.3258783415454483,
+      "tokens_seen": 3291152384
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.688172043010753e-06,
+      "loss": 1.1711,
+      "theoretical_loss": 3.325868433769828,
+      "tokens_seen": 3291283456
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.6480500722195474e-06,
+      "loss": 1.2749,
+      "theoretical_loss": 3.32585852649924,
+      "tokens_seen": 3291414528
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.6079281014283423e-06,
+      "loss": 1.273,
+      "theoretical_loss": 3.325848619733639,
+      "tokens_seen": 3291545600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.5678061306371367e-06,
+      "loss": 1.2009,
+      "theoretical_loss": 3.32583871347298,
+      "tokens_seen": 3291676672
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.5276841598459316e-06,
+      "loss": 1.2063,
+      "theoretical_loss": 3.3258288077172162,
+      "tokens_seen": 3291807744
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.4875621890547264e-06,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3258189024663016,
+      "tokens_seen": 3291938816
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.447440218263521e-06,
+      "loss": 1.2252,
+      "theoretical_loss": 3.325808997720191,
+      "tokens_seen": 3292069888
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.407318247472316e-06,
+      "loss": 1.2443,
+      "theoretical_loss": 3.325799093478838,
+      "tokens_seen": 3292200960
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.3671962766811106e-06,
+      "loss": 1.2288,
+      "theoretical_loss": 3.3257891897421974,
+      "tokens_seen": 3292332032
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.327074305889905e-06,
+      "loss": 1.2131,
+      "theoretical_loss": 3.325779286510223,
+      "tokens_seen": 3292463104
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.2869523350987004e-06,
+      "loss": 1.2278,
+      "theoretical_loss": 3.3257693837828692,
+      "tokens_seen": 3292594176
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.246830364307495e-06,
+      "loss": 1.2638,
+      "theoretical_loss": 3.3257594815600897,
+      "tokens_seen": 3292725248
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.2067083935162893e-06,
+      "loss": 1.1884,
+      "theoretical_loss": 3.3257495798418395,
+      "tokens_seen": 3292856320
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.1665864227250846e-06,
+      "loss": 1.2551,
+      "theoretical_loss": 3.325739678628072,
+      "tokens_seen": 3292987392
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.126464451933879e-06,
+      "loss": 1.2429,
+      "theoretical_loss": 3.3257297779187422,
+      "tokens_seen": 3293118464
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.0863424811426735e-06,
+      "loss": 1.1823,
+      "theoretical_loss": 3.3257198777138037,
+      "tokens_seen": 3293249536
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.0462205103514688e-06,
+      "loss": 1.2281,
+      "theoretical_loss": 3.325709978013211,
+      "tokens_seen": 3293380608
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.006098539560263e-06,
+      "loss": 1.2144,
+      "theoretical_loss": 3.325700078816918,
+      "tokens_seen": 3293511680
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.9659765687690577e-06,
+      "loss": 1.2189,
+      "theoretical_loss": 3.32569018012488,
+      "tokens_seen": 3293642752
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.9258545979778525e-06,
+      "loss": 1.1737,
+      "theoretical_loss": 3.32568028193705,
+      "tokens_seen": 3293773824
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.8857326271866474e-06,
+      "loss": 1.2865,
+      "theoretical_loss": 3.325670384253383,
+      "tokens_seen": 3293904896
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.845610656395442e-06,
+      "loss": 1.1564,
+      "theoretical_loss": 3.325660487073833,
+      "tokens_seen": 3294035968
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.805488685604237e-06,
+      "loss": 1.2146,
+      "theoretical_loss": 3.3256505903983538,
+      "tokens_seen": 3294167040
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.7653667148130318e-06,
+      "loss": 1.2087,
+      "theoretical_loss": 3.3256406942269003,
+      "tokens_seen": 3294298112
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.7252447440218263e-06,
+      "loss": 1.2871,
+      "theoretical_loss": 3.325630798559427,
+      "tokens_seen": 3294429184
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6851227732306211e-06,
+      "loss": 1.2144,
+      "theoretical_loss": 3.3256209033958877,
+      "tokens_seen": 3294560256
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.645000802439416e-06,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3256110087362365,
+      "tokens_seen": 3294691328
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6048788316482104e-06,
+      "loss": 1.2827,
+      "theoretical_loss": 3.325601114580428,
+      "tokens_seen": 3294822400
+    },
+    {
+      "epoch": 1.0,
+      "objective/train/docs_used": 1104299,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1488850116729736,
+      "objective/train/theoretical_loss": 3.325591220928417,
+      "objective/train/tokens_used": 1665412576,
+      "theoretical_loss": 3.325591220928417,
+      "tokens_seen": 3294953472
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.5647568608570053e-06,
+      "loss": 1.1317,
+      "theoretical_loss": 3.325591220928417,
+      "tokens_seen": 3294953472
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.5246348900658002e-06,
+      "loss": 1.219,
+      "theoretical_loss": 3.325581327780157,
+      "tokens_seen": 3295084544
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.4845129192745948e-06,
+      "loss": 1.1606,
+      "theoretical_loss": 3.3255714351356023,
+      "tokens_seen": 3295215616
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.4443909484833895e-06,
+      "loss": 1.2236,
+      "theoretical_loss": 3.325561542994708,
+      "tokens_seen": 3295346688
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.4042689776921844e-06,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3255516513574275,
+      "tokens_seen": 3295477760
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.364147006900979e-06,
+      "loss": 1.1897,
+      "theoretical_loss": 3.3255417602237163,
+      "tokens_seen": 3295608832
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.3240250361097737e-06,
+      "loss": 1.1702,
+      "theoretical_loss": 3.325531869593527,
+      "tokens_seen": 3295739904
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2839030653185684e-06,
+      "loss": 1.2037,
+      "theoretical_loss": 3.325521979466816,
+      "tokens_seen": 3295870976
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2437810945273632e-06,
+      "loss": 1.2984,
+      "theoretical_loss": 3.325512089843536,
+      "tokens_seen": 3296002048
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.203659123736158e-06,
+      "loss": 1.2045,
+      "theoretical_loss": 3.3255022007236423,
+      "tokens_seen": 3296133120
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.1635371529449525e-06,
+      "loss": 1.1411,
+      "theoretical_loss": 3.325492312107089,
+      "tokens_seen": 3296264192
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.1234151821537474e-06,
+      "loss": 1.1967,
+      "theoretical_loss": 3.3254824239938303,
+      "tokens_seen": 3296395264
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0832932113625423e-06,
+      "loss": 1.1671,
+      "theoretical_loss": 3.3254725363838205,
+      "tokens_seen": 3296526336
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0431712405713367e-06,
+      "loss": 1.1713,
+      "theoretical_loss": 3.3254626492770143,
+      "tokens_seen": 3296657408
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0030492697801316e-06,
+      "loss": 1.2395,
+      "theoretical_loss": 3.325452762673366,
+      "tokens_seen": 3296788480
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 9.629272989889263e-07,
+      "loss": 1.2322,
+      "theoretical_loss": 3.32544287657283,
+      "tokens_seen": 3296919552
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 9.22805328197721e-07,
+      "loss": 1.1909,
+      "theoretical_loss": 3.3254329909753606,
+      "tokens_seen": 3297050624
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.826833574065159e-07,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3254231058809123,
+      "tokens_seen": 3297181696
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.425613866153106e-07,
+      "loss": 1.1914,
+      "theoretical_loss": 3.3254132212894394,
+      "tokens_seen": 3297312768
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.024394158241052e-07,
+      "loss": 1.2128,
+      "theoretical_loss": 3.325403337200896,
+      "tokens_seen": 3297443840
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 7.623174450329001e-07,
+      "loss": 1.1373,
+      "theoretical_loss": 3.3253934536152374,
+      "tokens_seen": 3297574912
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 7.221954742416948e-07,
+      "loss": 1.2719,
+      "theoretical_loss": 3.3253835705324173,
+      "tokens_seen": 3297705984
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 6.820735034504895e-07,
+      "loss": 1.2314,
+      "theoretical_loss": 3.3253736879523905,
+      "tokens_seen": 3297837056
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 6.419515326592842e-07,
+      "loss": 1.1858,
+      "theoretical_loss": 3.325363805875111,
+      "tokens_seen": 3297968128
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 6.01829561868079e-07,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3253539243005337,
+      "tokens_seen": 3298099200
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 5.617075910768737e-07,
+      "loss": 1.1663,
+      "theoretical_loss": 3.325344043228613,
+      "tokens_seen": 3298230272
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 5.215856202856684e-07,
+      "loss": 1.2496,
+      "theoretical_loss": 3.325334162659303,
+      "tokens_seen": 3298361344
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.814636494944631e-07,
+      "loss": 1.2662,
+      "theoretical_loss": 3.3253242825925584,
+      "tokens_seen": 3298492416
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.4134167870325795e-07,
+      "loss": 1.1859,
+      "theoretical_loss": 3.325314403028334,
+      "tokens_seen": 3298623488
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.012197079120526e-07,
+      "loss": 1.225,
+      "theoretical_loss": 3.325304523966584,
+      "tokens_seen": 3298754560
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.610977371208474e-07,
+      "loss": 1.1533,
+      "theoretical_loss": 3.3252946454072623,
+      "tokens_seen": 3298885632
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.209757663296421e-07,
+      "loss": 1.1474,
+      "theoretical_loss": 3.325284767350324,
+      "tokens_seen": 3299016704
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.8085379553843685e-07,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3252748897957236,
+      "tokens_seen": 3299147776
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.4073182474723157e-07,
+      "loss": 1.1644,
+      "theoretical_loss": 3.3252650127434156,
+      "tokens_seen": 3299278848
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.006098539560263e-07,
+      "loss": 1.203,
+      "theoretical_loss": 3.3252551361933547,
+      "tokens_seen": 3299409920
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6048788316482104e-07,
+      "loss": 1.1228,
+      "theoretical_loss": 3.3252452601454947,
+      "tokens_seen": 3299540992
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2036591237361578e-07,
+      "loss": 1.2079,
+      "theoretical_loss": 3.3252353845997904,
+      "tokens_seen": 3299672064
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.024394158241052e-08,
+      "loss": 1.1876,
+      "theoretical_loss": 3.325225509556197,
+      "tokens_seen": 3299803136
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.012197079120526e-08,
+      "loss": 1.1816,
+      "theoretical_loss": 3.325215635014668,
+      "tokens_seen": 3299934208
+    }
+  ],
+  "max_steps": 12588,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 8.42020889296896e+17,
+  "trial_name": null,
+  "trial_params": null
+}