diff --git "a/train_logs.json" "b/train_logs.json"
new file mode 100644--- /dev/null
+++ "b/train_logs.json"
@@ -0,0 +1,3593 @@
+[
+    {
+        "step": 50,
+        "train_loss": 6.0813891887664795,
+        "val_loss": 6.026689910888672,
+        "lr": 0.01,
+        "tokens_per_sec": 794214.7164234697,
+        "mfu": 7.466395208164367,
+        "total_flops": 1266401163608064.0
+    },
+    {
+        "step": 100,
+        "train_loss": 5.087921142578125,
+        "val_loss": 5.089700198173523,
+        "lr": 0.02,
+        "tokens_per_sec": 736508.4501651466,
+        "mfu": 6.923899859032017,
+        "total_flops": 2507970931851264.0
+    },
+    {
+        "step": 150,
+        "train_loss": 4.558597803115845,
+        "val_loss": 4.58702654838562,
+        "lr": 0.02,
+        "tokens_per_sec": 635169.419700884,
+        "mfu": 5.97121384628007,
+        "total_flops": 3749540700094464.0
+    },
+    {
+        "step": 200,
+        "train_loss": 4.243805885314941,
+        "val_loss": 4.244970810413361,
+        "lr": 0.02,
+        "tokens_per_sec": 740626.4277880366,
+        "mfu": 6.9626128767526305,
+        "total_flops": 4991110468337664.0
+    },
+    {
+        "step": 250,
+        "train_loss": 4.045148968696594,
+        "val_loss": 4.084274005889893,
+        "lr": 0.02,
+        "tokens_per_sec": 729753.6828782216,
+        "mfu": 6.860398439251636,
+        "total_flops": 6232680236580864.0
+    },
+    {
+        "step": 300,
+        "train_loss": 3.926572263240814,
+        "val_loss": 3.9035560965538023,
+        "lr": 0.02,
+        "tokens_per_sec": 587099.4249975935,
+        "mfu": 5.5193088756376385,
+        "total_flops": 7474250004824064.0
+    },
+    {
+        "step": 350,
+        "train_loss": 3.7291558384895325,
+        "val_loss": 3.7622887253761292,
+        "lr": 0.02,
+        "tokens_per_sec": 789797.325586338,
+        "mfu": 7.424867413354072,
+        "total_flops": 8715819773067264.0
+    },
+    {
+        "step": 400,
+        "train_loss": 3.615461826324463,
+        "val_loss": 3.6281559228897096,
+        "lr": 0.02,
+        "tokens_per_sec": 302900.2753542377,
+        "mfu": 2.847558875062059,
+        "total_flops": 9957389541310464.0
+    },
+    {
+        "step": 450,
+        "train_loss": 3.5205406546592712,
+        "val_loss": 3.5723757147789,
+        "lr": 0.02,
+        "tokens_per_sec": 607157.4423502305,
+        "mfu": 5.707873858821802,
+        "total_flops": 1.1198959309553664e+16
+    },
+    {
+        "step": 500,
+        "train_loss": 3.4816104769706726,
+        "val_loss": 3.532227611541748,
+        "lr": 0.02,
+        "tokens_per_sec": 703708.9583817514,
+        "mfu": 6.615552552922427,
+        "total_flops": 1.2440529077796864e+16
+    },
+    {
+        "step": 550,
+        "train_loss": 3.3761273622512817,
+        "val_loss": 3.5005128383636475,
+        "lr": 0.02,
+        "tokens_per_sec": 722074.6273645067,
+        "mfu": 6.788207806032169,
+        "total_flops": 1.3682098846040064e+16
+    },
+    {
+        "step": 600,
+        "train_loss": 3.4218554496765137,
+        "val_loss": 3.471066892147064,
+        "lr": 0.02,
+        "tokens_per_sec": 621526.2112982491,
+        "mfu": 5.842954341973545,
+        "total_flops": 1.4923668614283264e+16
+    },
+    {
+        "step": 650,
+        "train_loss": 3.4142528772354126,
+        "val_loss": 3.455700898170471,
+        "lr": 0.02,
+        "tokens_per_sec": 760036.6436618323,
+        "mfu": 7.145087892378344,
+        "total_flops": 1.6165238382526464e+16
+    },
+    {
+        "step": 700,
+        "train_loss": 3.377539098262787,
+        "val_loss": 3.436308777332306,
+        "lr": 0.02,
+        "tokens_per_sec": 722331.7487977484,
+        "mfu": 6.79062499901207,
+        "total_flops": 1.7406808150769664e+16
+    },
+    {
+        "step": 750,
+        "train_loss": 3.3675426840782166,
+        "val_loss": 3.4219574451446535,
+        "lr": 0.02,
+        "tokens_per_sec": 710436.7041358028,
+        "mfu": 6.678799943862251,
+        "total_flops": 1.8648377919012864e+16
+    },
+    {
+        "step": 800,
+        "train_loss": 3.331727921962738,
+        "val_loss": 3.40491441488266,
+        "lr": 0.02,
+        "tokens_per_sec": 809757.6478994827,
+        "mfu": 7.612513967605048,
+        "total_flops": 1.9889947687256064e+16
+    },
+    {
+        "step": 850,
+        "train_loss": 3.3138641715049744,
+        "val_loss": 3.3908426761627197,
+        "lr": 0.02,
+        "tokens_per_sec": 742484.9786147004,
+        "mfu": 6.980085072494384,
+        "total_flops": 2.1131517455499264e+16
+    },
+    {
+        "step": 900,
+        "train_loss": 3.314534604549408,
+        "val_loss": 3.372792398929596,
+        "lr": 0.02,
+        "tokens_per_sec": 699743.0740374597,
+        "mfu": 6.5782693607931,
+        "total_flops": 2.2373087223742464e+16
+    },
+    {
+        "step": 950,
+        "train_loss": 3.2452561259269714,
+        "val_loss": 3.362290894985199,
+        "lr": 0.02,
+        "tokens_per_sec": 739832.2356642535,
+        "mfu": 6.955146693937383,
+        "total_flops": 2.3614656991985664e+16
+    },
+    {
+        "step": 1000,
+        "train_loss": 3.262390971183777,
+        "val_loss": 3.3680756092071533,
+        "lr": 0.02,
+        "tokens_per_sec": 736792.1990218526,
+        "mfu": 6.926567375838519,
+        "total_flops": 2.4856226760228864e+16
+    },
+    {
+        "step": 1050,
+        "train_loss": 3.3225939869880676,
+        "val_loss": 3.335637629032135,
+        "lr": 0.02,
+        "tokens_per_sec": 729562.5144698353,
+        "mfu": 6.858601269218378,
+        "total_flops": 2.6097796528472064e+16
+    },
+    {
+        "step": 1100,
+        "train_loss": 3.300910174846649,
+        "val_loss": 3.316732621192932,
+        "lr": 0.02,
+        "tokens_per_sec": 802942.0066579218,
+        "mfu": 7.5484402731062605,
+        "total_flops": 2.7339366296715264e+16
+    },
+    {
+        "step": 1150,
+        "train_loss": 3.2660632729530334,
+        "val_loss": 3.3214237689971924,
+        "lr": 0.02,
+        "tokens_per_sec": 753021.8553813706,
+        "mfu": 7.0791420209150395,
+        "total_flops": 2.8580936064958464e+16
+    },
+    {
+        "step": 1200,
+        "train_loss": 3.2546098232269287,
+        "val_loss": 3.314812517166138,
+        "lr": 0.02,
+        "tokens_per_sec": 761949.7264553182,
+        "mfu": 7.163072741949536,
+        "total_flops": 2.9822505833201664e+16
+    },
+    {
+        "step": 1250,
+        "train_loss": 3.200866460800171,
+        "val_loss": 3.3129363536834715,
+        "lr": 0.02,
+        "tokens_per_sec": 759970.4128061022,
+        "mfu": 7.144465257549713,
+        "total_flops": 3.1064075601444864e+16
+    },
+    {
+        "step": 1300,
+        "train_loss": 3.130142331123352,
+        "val_loss": 3.3093976616859435,
+        "lr": 0.02,
+        "tokens_per_sec": 763491.4059958718,
+        "mfu": 7.1775660376491555,
+        "total_flops": 3.2305645369688064e+16
+    },
+    {
+        "step": 1350,
+        "train_loss": 3.259128153324127,
+        "val_loss": 3.3080609321594237,
+        "lr": 0.02,
+        "tokens_per_sec": 737350.9208228615,
+        "mfu": 6.931819907290637,
+        "total_flops": 3.3547215137931264e+16
+    },
+    {
+        "step": 1400,
+        "train_loss": 3.2849743962287903,
+        "val_loss": 3.3059651374816896,
+        "lr": 0.02,
+        "tokens_per_sec": 523486.77008353075,
+        "mfu": 4.921287695713131,
+        "total_flops": 3.4788784906174464e+16
+    },
+    {
+        "step": 1450,
+        "train_loss": 3.06045800447464,
+        "val_loss": 3.3061444997787475,
+        "lr": 0.02,
+        "tokens_per_sec": 735041.2803046936,
+        "mfu": 6.9101070272081735,
+        "total_flops": 3.603035467441766e+16
+    },
+    {
+        "step": 1500,
+        "train_loss": 3.2446532249450684,
+        "val_loss": 3.291266882419586,
+        "lr": 0.02,
+        "tokens_per_sec": 724220.7811134176,
+        "mfu": 6.808383750566429,
+        "total_flops": 3.727192444266086e+16
+    },
+    {
+        "step": 1550,
+        "train_loss": 3.244351804256439,
+        "val_loss": 3.2926158547401427,
+        "lr": 0.02,
+        "tokens_per_sec": 779334.7750054352,
+        "mfu": 7.326509203782974,
+        "total_flops": 3.851349421090406e+16
+    },
+    {
+        "step": 1600,
+        "train_loss": 3.2528223395347595,
+        "val_loss": 3.2792186975479125,
+        "lr": 0.02,
+        "tokens_per_sec": 732232.7445099982,
+        "mfu": 6.883704043523984,
+        "total_flops": 3.975506397914726e+16
+    },
+    {
+        "step": 1650,
+        "train_loss": 3.2681846022605896,
+        "val_loss": 3.2697198152542115,
+        "lr": 0.02,
+        "tokens_per_sec": 727952.4135197375,
+        "mfu": 6.843464745341551,
+        "total_flops": 4.099663374739046e+16
+    },
+    {
+        "step": 1700,
+        "train_loss": 3.257525682449341,
+        "val_loss": 3.2683892846107483,
+        "lr": 0.02,
+        "tokens_per_sec": 727843.2189831685,
+        "mfu": 6.842438209887426,
+        "total_flops": 4.223820351563366e+16
+    },
+    {
+        "step": 1750,
+        "train_loss": 3.1968674063682556,
+        "val_loss": 3.277073395252228,
+        "lr": 0.02,
+        "tokens_per_sec": 715307.3754703364,
+        "mfu": 6.724589018731667,
+        "total_flops": 4.347977328387686e+16
+    },
+    {
+        "step": 1800,
+        "train_loss": 3.2354595065116882,
+        "val_loss": 3.276608467102051,
+        "lr": 0.02,
+        "tokens_per_sec": 745560.8215328843,
+        "mfu": 7.009001004610087,
+        "total_flops": 4.472134305212006e+16
+    },
+    {
+        "step": 1850,
+        "train_loss": 3.219004511833191,
+        "val_loss": 3.267551374435425,
+        "lr": 0.02,
+        "tokens_per_sec": 706164.9282135066,
+        "mfu": 6.638641071686961,
+        "total_flops": 4.596291282036326e+16
+    },
+    {
+        "step": 1900,
+        "train_loss": 3.205766499042511,
+        "val_loss": 3.2662167072296144,
+        "lr": 0.02,
+        "tokens_per_sec": 682567.1752142893,
+        "mfu": 6.416799110976101,
+        "total_flops": 4.720448258860646e+16
+    },
+    {
+        "step": 1950,
+        "train_loss": 3.321563243865967,
+        "val_loss": 3.2645433068275453,
+        "lr": 0.02,
+        "tokens_per_sec": 713166.0692443526,
+        "mfu": 6.704458645654604,
+        "total_flops": 4.844605235684966e+16
+    },
+    {
+        "step": 2000,
+        "train_loss": 3.162742018699646,
+        "val_loss": 3.263501501083374,
+        "lr": 0.02,
+        "tokens_per_sec": 736771.9689582054,
+        "mfu": 6.926377193451875,
+        "total_flops": 4.968762212509286e+16
+    },
+    {
+        "step": 2050,
+        "train_loss": 3.239263594150543,
+        "val_loss": 3.260008931159973,
+        "lr": 0.02,
+        "tokens_per_sec": 701897.3676516128,
+        "mfu": 6.5985218280227755,
+        "total_flops": 5.092919189333606e+16
+    },
+    {
+        "step": 2100,
+        "train_loss": 3.1417112350463867,
+        "val_loss": 3.2654417872428896,
+        "lr": 0.02,
+        "tokens_per_sec": 724962.1222182881,
+        "mfu": 6.81535308210683,
+        "total_flops": 5.217076166157926e+16
+    },
+    {
+        "step": 2150,
+        "train_loss": 3.1347965002059937,
+        "val_loss": 3.259656322002411,
+        "lr": 0.02,
+        "tokens_per_sec": 779163.2670647042,
+        "mfu": 7.324896861376877,
+        "total_flops": 5.341233142982246e+16
+    },
+    {
+        "step": 2200,
+        "train_loss": 3.1822943687438965,
+        "val_loss": 3.253624069690704,
+        "lr": 0.02,
+        "tokens_per_sec": 760755.4181808244,
+        "mfu": 7.151845075937621,
+        "total_flops": 5.465390119806566e+16
+    },
+    {
+        "step": 2250,
+        "train_loss": 3.1740533113479614,
+        "val_loss": 3.2457565784454347,
+        "lr": 0.02,
+        "tokens_per_sec": 769730.274715895,
+        "mfu": 7.236217506266253,
+        "total_flops": 5.589547096630886e+16
+    },
+    {
+        "step": 2300,
+        "train_loss": 3.198862850666046,
+        "val_loss": 3.2464448690414427,
+        "lr": 0.02,
+        "tokens_per_sec": 720703.044629105,
+        "mfu": 6.775313586683879,
+        "total_flops": 5.713704073455206e+16
+    },
+    {
+        "step": 2350,
+        "train_loss": 3.1597362756729126,
+        "val_loss": 3.242673170566559,
+        "lr": 0.02,
+        "tokens_per_sec": 738083.676306573,
+        "mfu": 6.938708525593981,
+        "total_flops": 5.837861050279526e+16
+    },
+    {
+        "step": 2400,
+        "train_loss": 3.1868541836738586,
+        "val_loss": 3.237907588481903,
+        "lr": 0.02,
+        "tokens_per_sec": 769817.1624990003,
+        "mfu": 7.237034336418113,
+        "total_flops": 5.962018027103846e+16
+    },
+    {
+        "step": 2450,
+        "train_loss": 3.153138518333435,
+        "val_loss": 3.2368793964385985,
+        "lr": 0.02,
+        "tokens_per_sec": 767427.2208379519,
+        "mfu": 7.214566547044729,
+        "total_flops": 6.086175003928166e+16
+    },
+    {
+        "step": 2500,
+        "train_loss": 3.247396469116211,
+        "val_loss": 3.2282278895378114,
+        "lr": 0.02,
+        "tokens_per_sec": 780143.0316463324,
+        "mfu": 7.334107606816523,
+        "total_flops": 6.210331980752486e+16
+    },
+    {
+        "step": 2550,
+        "train_loss": 3.122944414615631,
+        "val_loss": 3.2200260877609255,
+        "lr": 0.02,
+        "tokens_per_sec": 742767.58170015,
+        "mfu": 6.9827418179303224,
+        "total_flops": 6.334488957576806e+16
+    },
+    {
+        "step": 2600,
+        "train_loss": 3.1630945801734924,
+        "val_loss": 3.216521847248077,
+        "lr": 0.02,
+        "tokens_per_sec": 704446.6323657001,
+        "mfu": 6.62248740993911,
+        "total_flops": 6.458645934401126e+16
+    },
+    {
+        "step": 2650,
+        "train_loss": 3.0957342982292175,
+        "val_loss": 3.21864767074585,
+        "lr": 0.02,
+        "tokens_per_sec": 191314.34620024994,
+        "mfu": 1.7985419914594185,
+        "total_flops": 6.582802911225446e+16
+    },
+    {
+        "step": 2700,
+        "train_loss": 3.173651099205017,
+        "val_loss": 3.224730205535889,
+        "lr": 0.02,
+        "tokens_per_sec": 709829.4426204753,
+        "mfu": 6.673091091615642,
+        "total_flops": 6.706959888049766e+16
+    },
+    {
+        "step": 2750,
+        "train_loss": 3.1663557291030884,
+        "val_loss": 3.2234278440475466,
+        "lr": 0.02,
+        "tokens_per_sec": 724416.7606760697,
+        "mfu": 6.810226150155891,
+        "total_flops": 6.831116864874086e+16
+    },
+    {
+        "step": 2800,
+        "train_loss": 3.147324025630951,
+        "val_loss": 3.2159315824508665,
+        "lr": 0.02,
+        "tokens_per_sec": 734666.1504854651,
+        "mfu": 6.906580439968213,
+        "total_flops": 6.955273841698406e+16
+    },
+    {
+        "step": 2850,
+        "train_loss": 3.224787950515747,
+        "val_loss": 3.2105109453201295,
+        "lr": 0.02,
+        "tokens_per_sec": 699516.5324807317,
+        "mfu": 6.576139648564623,
+        "total_flops": 7.079430818522726e+16
+    },
+    {
+        "step": 2900,
+        "train_loss": 3.2367559671401978,
+        "val_loss": 3.231448769569397,
+        "lr": 0.02,
+        "tokens_per_sec": 662728.9877131225,
+        "mfu": 6.230300743425831,
+        "total_flops": 7.203587795347046e+16
+    },
+    {
+        "step": 2950,
+        "train_loss": 3.2191177010536194,
+        "val_loss": 3.2184744119644164,
+        "lr": 0.02,
+        "tokens_per_sec": 672224.1362785365,
+        "mfu": 6.319564427771635,
+        "total_flops": 7.327744772171366e+16
+    },
+    {
+        "step": 3000,
+        "train_loss": 3.131461203098297,
+        "val_loss": 3.213130033016205,
+        "lr": 0.02,
+        "tokens_per_sec": 717274.5614815141,
+        "mfu": 6.743082491471089,
+        "total_flops": 7.451901748995686e+16
+    },
+    {
+        "step": 3050,
+        "train_loss": 3.1307908296585083,
+        "val_loss": 3.228156554698944,
+        "lr": 0.02,
+        "tokens_per_sec": 699103.3039631982,
+        "mfu": 6.572254896293749,
+        "total_flops": 7.576058725820006e+16
+    },
+    {
+        "step": 3100,
+        "train_loss": 3.102234125137329,
+        "val_loss": 3.2135570764541628,
+        "lr": 0.02,
+        "tokens_per_sec": 708978.8172659309,
+        "mfu": 6.665094381230177,
+        "total_flops": 7.700215702644326e+16
+    },
+    {
+        "step": 3150,
+        "train_loss": 3.1781490445137024,
+        "val_loss": 3.206849551200867,
+        "lr": 0.02,
+        "tokens_per_sec": 710505.0044441634,
+        "mfu": 6.679442033569876,
+        "total_flops": 7.824372679468646e+16
+    },
+    {
+        "step": 3200,
+        "train_loss": 3.020294487476349,
+        "val_loss": 3.218398690223694,
+        "lr": 0.02,
+        "tokens_per_sec": 700330.5278482502,
+        "mfu": 6.583792001241841,
+        "total_flops": 7.948529656292966e+16
+    },
+    {
+        "step": 3250,
+        "train_loss": 3.1484155654907227,
+        "val_loss": 3.195547950267792,
+        "lr": 0.02,
+        "tokens_per_sec": 803595.1037902995,
+        "mfu": 7.554580024987982,
+        "total_flops": 8.072686633117286e+16
+    },
+    {
+        "step": 3300,
+        "train_loss": 3.2074788212776184,
+        "val_loss": 3.202925610542297,
+        "lr": 0.02,
+        "tokens_per_sec": 725059.8870553945,
+        "mfu": 6.816272167205863,
+        "total_flops": 8.196843609941606e+16
+    },
+    {
+        "step": 3350,
+        "train_loss": 3.1220479011535645,
+        "val_loss": 3.2095597267150877,
+        "lr": 0.02,
+        "tokens_per_sec": 730116.0178445905,
+        "mfu": 6.863804742359225,
+        "total_flops": 8.321000586765926e+16
+    },
+    {
+        "step": 3400,
+        "train_loss": 3.133055806159973,
+        "val_loss": 3.2004257798194886,
+        "lr": 0.02,
+        "tokens_per_sec": 684447.9707085511,
+        "mfu": 6.434480428352253,
+        "total_flops": 8.445157563590246e+16
+    },
+    {
+        "step": 3450,
+        "train_loss": 3.142967641353607,
+        "val_loss": 3.2058743834495544,
+        "lr": 0.02,
+        "tokens_per_sec": 674727.5005119084,
+        "mfu": 6.343098500270955,
+        "total_flops": 8.569314540414566e+16
+    },
+    {
+        "step": 3500,
+        "train_loss": 3.111777603626251,
+        "val_loss": 3.2019073963165283,
+        "lr": 0.02,
+        "tokens_per_sec": 733620.1779752404,
+        "mfu": 6.896747275237419,
+        "total_flops": 8.693471517238886e+16
+    },
+    {
+        "step": 3550,
+        "train_loss": 3.0948435068130493,
+        "val_loss": 3.208818733692169,
+        "lr": 0.02,
+        "tokens_per_sec": 752006.8155076118,
+        "mfu": 7.069599653224275,
+        "total_flops": 8.817628494063206e+16
+    },
+    {
+        "step": 3600,
+        "train_loss": 3.1297584176063538,
+        "val_loss": 3.20062255859375,
+        "lr": 0.02,
+        "tokens_per_sec": 757551.0811660548,
+        "mfu": 7.1217211736254615,
+        "total_flops": 8.941785470887526e+16
+    },
+    {
+        "step": 3650,
+        "train_loss": 3.097672760486603,
+        "val_loss": 3.195528542995453,
+        "lr": 0.02,
+        "tokens_per_sec": 753421.1065528493,
+        "mfu": 7.082895372460834,
+        "total_flops": 9.065942447711846e+16
+    },
+    {
+        "step": 3700,
+        "train_loss": 3.239434063434601,
+        "val_loss": 3.1988398909568785,
+        "lr": 0.02,
+        "tokens_per_sec": 663844.9946939754,
+        "mfu": 6.2407923006858645,
+        "total_flops": 9.190099424536166e+16
+    },
+    {
+        "step": 3750,
+        "train_loss": 3.1775227189064026,
+        "val_loss": 3.197325074672699,
+        "lr": 0.02,
+        "tokens_per_sec": 770276.3805974862,
+        "mfu": 7.241351435735383,
+        "total_flops": 9.314256401360486e+16
+    },
+    {
+        "step": 3800,
+        "train_loss": 3.133751928806305,
+        "val_loss": 3.198042118549347,
+        "lr": 0.02,
+        "tokens_per_sec": 701977.7272873463,
+        "mfu": 6.599277287203729,
+        "total_flops": 9.438413378184806e+16
+    },
+    {
+        "step": 3850,
+        "train_loss": 3.067799389362335,
+        "val_loss": 3.1975177764892577,
+        "lr": 0.02,
+        "tokens_per_sec": 681804.8338186503,
+        "mfu": 6.409632356160707,
+        "total_flops": 9.562570355009126e+16
+    },
+    {
+        "step": 3900,
+        "train_loss": 3.1261569261550903,
+        "val_loss": 3.199464535713196,
+        "lr": 0.02,
+        "tokens_per_sec": 700615.0151335543,
+        "mfu": 6.586466459999472,
+        "total_flops": 9.686727331833446e+16
+    },
+    {
+        "step": 3950,
+        "train_loss": 3.1253642439842224,
+        "val_loss": 3.1979977607727053,
+        "lr": 0.02,
+        "tokens_per_sec": 764910.951011311,
+        "mfu": 7.190911149344868,
+        "total_flops": 9.810884308657766e+16
+    },
+    {
+        "step": 4000,
+        "train_loss": 3.0701352953910828,
+        "val_loss": 3.1961832523345945,
+        "lr": 0.02,
+        "tokens_per_sec": 763747.9314474192,
+        "mfu": 7.17997762781816,
+        "total_flops": 9.935041285482086e+16
+    },
+    {
+        "step": 4050,
+        "train_loss": 3.127575993537903,
+        "val_loss": 3.1980692744255066,
+        "lr": 0.02,
+        "tokens_per_sec": 711749.536967804,
+        "mfu": 6.691141856651423,
+        "total_flops": 1.0059198262306406e+17
+    },
+    {
+        "step": 4100,
+        "train_loss": 3.1469481587409973,
+        "val_loss": 3.1901233553886414,
+        "lr": 0.02,
+        "tokens_per_sec": 780514.0241170248,
+        "mfu": 7.3375952989332305,
+        "total_flops": 1.0183355239130726e+17
+    },
+    {
+        "step": 4150,
+        "train_loss": 3.2410582304000854,
+        "val_loss": 3.195158803462982,
+        "lr": 0.02,
+        "tokens_per_sec": 747323.0486882784,
+        "mfu": 7.025567663621373,
+        "total_flops": 1.0307512215955046e+17
+    },
+    {
+        "step": 4200,
+        "train_loss": 3.158669590950012,
+        "val_loss": 3.2082596898078917,
+        "lr": 0.02,
+        "tokens_per_sec": 762441.6464148208,
+        "mfu": 7.167697270748211,
+        "total_flops": 1.0431669192779366e+17
+    },
+    {
+        "step": 4250,
+        "train_loss": 3.171230673789978,
+        "val_loss": 3.193644082546234,
+        "lr": 0.02,
+        "tokens_per_sec": 789190.9245840942,
+        "mfu": 7.419166650772175,
+        "total_flops": 1.0555826169603686e+17
+    },
+    {
+        "step": 4300,
+        "train_loss": 3.1072226762771606,
+        "val_loss": 3.1926248908042907,
+        "lr": 0.02,
+        "tokens_per_sec": 798558.8382577032,
+        "mfu": 7.507234202678174,
+        "total_flops": 1.0679983146428006e+17
+    },
+    {
+        "step": 4350,
+        "train_loss": 3.197103798389435,
+        "val_loss": 3.1995687246322633,
+        "lr": 0.02,
+        "tokens_per_sec": 797254.8530887661,
+        "mfu": 7.494975466576287,
+        "total_flops": 1.0804140123252326e+17
+    },
+    {
+        "step": 4400,
+        "train_loss": 3.1892601251602173,
+        "val_loss": 3.1967344164848326,
+        "lr": 0.02,
+        "tokens_per_sec": 755555.0768235743,
+        "mfu": 7.102956780382684,
+        "total_flops": 1.0928297100076646e+17
+    },
+    {
+        "step": 4450,
+        "train_loss": 3.1196552515029907,
+        "val_loss": 3.1996314406394957,
+        "lr": 0.02,
+        "tokens_per_sec": 799516.4178640184,
+        "mfu": 7.516236387649291,
+        "total_flops": 1.1052454076900966e+17
+    },
+    {
+        "step": 4500,
+        "train_loss": 3.112544298171997,
+        "val_loss": 3.1933774828910826,
+        "lr": 0.02,
+        "tokens_per_sec": 728543.854373343,
+        "mfu": 6.849024867892742,
+        "total_flops": 1.1176611053725286e+17
+    },
+    {
+        "step": 4550,
+        "train_loss": 3.0423641204833984,
+        "val_loss": 3.1891826510429384,
+        "lr": 0.02,
+        "tokens_per_sec": 750392.8790864372,
+        "mfu": 7.054427072167598,
+        "total_flops": 1.1300768030549606e+17
+    },
+    {
+        "step": 4600,
+        "train_loss": 3.125426948070526,
+        "val_loss": 3.1997652888298034,
+        "lr": 0.02,
+        "tokens_per_sec": 661473.8626517142,
+        "mfu": 6.218501340128009,
+        "total_flops": 1.1424925007373926e+17
+    },
+    {
+        "step": 4650,
+        "train_loss": 3.1154977679252625,
+        "val_loss": 3.191564977169037,
+        "lr": 0.02,
+        "tokens_per_sec": 670619.464242622,
+        "mfu": 6.304478940998509,
+        "total_flops": 1.1549081984198246e+17
+    },
+    {
+        "step": 4700,
+        "train_loss": 3.0956615805625916,
+        "val_loss": 3.1927812576293944,
+        "lr": 0.02,
+        "tokens_per_sec": 703655.0511273266,
+        "mfu": 6.615045772000593,
+        "total_flops": 1.1673238961022566e+17
+    },
+    {
+        "step": 4750,
+        "train_loss": 3.169421136379242,
+        "val_loss": 3.2010963797569274,
+        "lr": 0.02,
+        "tokens_per_sec": 498538.56643634004,
+        "mfu": 4.686750177946495,
+        "total_flops": 1.1797395937846886e+17
+    },
+    {
+        "step": 4800,
+        "train_loss": 3.1868945956230164,
+        "val_loss": 3.1932270765304565,
+        "lr": 0.02,
+        "tokens_per_sec": 697798.546786271,
+        "mfu": 6.559988902561603,
+        "total_flops": 1.1921552914671206e+17
+    },
+    {
+        "step": 4850,
+        "train_loss": 3.1466269493103027,
+        "val_loss": 3.197691464424133,
+        "lr": 0.02,
+        "tokens_per_sec": 702094.5409157231,
+        "mfu": 6.600375449573584,
+        "total_flops": 1.2045709891495526e+17
+    },
+    {
+        "step": 4900,
+        "train_loss": 3.1853439807891846,
+        "val_loss": 3.190361273288727,
+        "lr": 0.02,
+        "tokens_per_sec": 673496.7381930096,
+        "mfu": 6.331528130583528,
+        "total_flops": 1.2169866868319846e+17
+    },
+    {
+        "step": 4950,
+        "train_loss": 3.1415867805480957,
+        "val_loss": 3.18849755525589,
+        "lr": 0.02,
+        "tokens_per_sec": 775784.3591892282,
+        "mfu": 7.293131822214809,
+        "total_flops": 1.2294023845144166e+17
+    },
+    {
+        "step": 5000,
+        "train_loss": 3.132832705974579,
+        "val_loss": 3.1886184215545654,
+        "lr": 0.02,
+        "tokens_per_sec": 705061.2508035857,
+        "mfu": 6.628265424454055,
+        "total_flops": 1.2418180821968486e+17
+    },
+    {
+        "step": 5050,
+        "train_loss": 3.13687264919281,
+        "val_loss": 3.192574620246887,
+        "lr": 0.02,
+        "tokens_per_sec": 766733.9904356612,
+        "mfu": 7.20804950316883,
+        "total_flops": 1.2542337798792806e+17
+    },
+    {
+        "step": 5100,
+        "train_loss": 3.1279391646385193,
+        "val_loss": 3.1825401544570924,
+        "lr": 0.02,
+        "tokens_per_sec": 783440.1023898001,
+        "mfu": 7.365103296887419,
+        "total_flops": 1.2666494775617126e+17
+    },
+    {
+        "step": 5150,
+        "train_loss": 3.1994088888168335,
+        "val_loss": 3.1858301997184753,
+        "lr": 0.02,
+        "tokens_per_sec": 732295.3383307715,
+        "mfu": 6.884292486666397,
+        "total_flops": 1.2790651752441446e+17
+    },
+    {
+        "step": 5200,
+        "train_loss": 3.2014970183372498,
+        "val_loss": 3.1864626884460447,
+        "lr": 0.02,
+        "tokens_per_sec": 698731.1794154128,
+        "mfu": 6.5687565615450065,
+        "total_flops": 1.2914808729265766e+17
+    },
+    {
+        "step": 5250,
+        "train_loss": 3.09488445520401,
+        "val_loss": 3.183492195606232,
+        "lr": 0.02,
+        "tokens_per_sec": 728472.9470492065,
+        "mfu": 6.848358269686729,
+        "total_flops": 1.3038965706090086e+17
+    },
+    {
+        "step": 5300,
+        "train_loss": 3.119004726409912,
+        "val_loss": 3.182840895652771,
+        "lr": 0.02,
+        "tokens_per_sec": 717396.2762640028,
+        "mfu": 6.744226729483736,
+        "total_flops": 1.3163122682914406e+17
+    },
+    {
+        "step": 5350,
+        "train_loss": 3.2293498516082764,
+        "val_loss": 3.1889729499816895,
+        "lr": 0.02,
+        "tokens_per_sec": 725808.5296261577,
+        "mfu": 6.8233101396677,
+        "total_flops": 1.3287279659738726e+17
+    },
+    {
+        "step": 5400,
+        "train_loss": 3.2103021144866943,
+        "val_loss": 3.179009199142456,
+        "lr": 0.02,
+        "tokens_per_sec": 662681.5853182896,
+        "mfu": 6.229855114546993,
+        "total_flops": 1.3411436636563046e+17
+    },
+    {
+        "step": 5450,
+        "train_loss": 3.1303423047065735,
+        "val_loss": 3.188521647453308,
+        "lr": 0.02,
+        "tokens_per_sec": 728308.2940295547,
+        "mfu": 6.846810370244026,
+        "total_flops": 1.3535593613387366e+17
+    },
+    {
+        "step": 5500,
+        "train_loss": 3.1481414437294006,
+        "val_loss": 3.18265780210495,
+        "lr": 0.02,
+        "tokens_per_sec": 782476.503243071,
+        "mfu": 7.356044522348301,
+        "total_flops": 1.3659750590211686e+17
+    },
+    {
+        "step": 5550,
+        "train_loss": 3.103872239589691,
+        "val_loss": 3.1814547777175903,
+        "lr": 0.02,
+        "tokens_per_sec": 693542.3596234568,
+        "mfu": 6.519976579973838,
+        "total_flops": 1.3783907567036006e+17
+    },
+    {
+        "step": 5600,
+        "train_loss": 3.1177070140838623,
+        "val_loss": 3.1881037831306456,
+        "lr": 0.02,
+        "tokens_per_sec": 670932.9462096259,
+        "mfu": 6.307425978125727,
+        "total_flops": 1.3908064543860326e+17
+    },
+    {
+        "step": 5650,
+        "train_loss": 3.0816861987113953,
+        "val_loss": 3.1840782165527344,
+        "lr": 0.02,
+        "tokens_per_sec": 696515.0198524478,
+        "mfu": 6.5479224938813365,
+        "total_flops": 1.4032221520684646e+17
+    },
+    {
+        "step": 5700,
+        "train_loss": 3.09510201215744,
+        "val_loss": 3.184128928184509,
+        "lr": 0.02,
+        "tokens_per_sec": 687919.7057449103,
+        "mfu": 6.4671181336269905,
+        "total_flops": 1.4156378497508966e+17
+    },
+    {
+        "step": 5750,
+        "train_loss": 3.1403180360794067,
+        "val_loss": 3.1881123423576354,
+        "lr": 0.02,
+        "tokens_per_sec": 648873.811017929,
+        "mfu": 6.100048529829092,
+        "total_flops": 1.4280535474333286e+17
+    },
+    {
+        "step": 5800,
+        "train_loss": 3.130989968776703,
+        "val_loss": 3.1858367681503297,
+        "lr": 0.02,
+        "tokens_per_sec": 727039.472701932,
+        "mfu": 6.834882208646567,
+        "total_flops": 1.4404692451157606e+17
+    },
+    {
+        "step": 5850,
+        "train_loss": 3.069033682346344,
+        "val_loss": 3.17846132516861,
+        "lr": 0.02,
+        "tokens_per_sec": 706744.8880096792,
+        "mfu": 6.644093261067901,
+        "total_flops": 1.4528849427981926e+17
+    },
+    {
+        "step": 5900,
+        "train_loss": 3.13921719789505,
+        "val_loss": 3.1820234179496767,
+        "lr": 0.02,
+        "tokens_per_sec": 726040.7135660015,
+        "mfu": 6.825492895816653,
+        "total_flops": 1.4653006404806246e+17
+    },
+    {
+        "step": 5950,
+        "train_loss": 3.0790565609931946,
+        "val_loss": 3.201737642288208,
+        "lr": 0.02,
+        "tokens_per_sec": 642554.8551086267,
+        "mfu": 6.04064416329421,
+        "total_flops": 1.4777163381630566e+17
+    },
+    {
+        "step": 6000,
+        "train_loss": 3.0759347081184387,
+        "val_loss": 3.1834460496902466,
+        "lr": 0.02,
+        "tokens_per_sec": 740776.0605106918,
+        "mfu": 6.964019570711222,
+        "total_flops": 1.4901320358454886e+17
+    },
+    {
+        "step": 6050,
+        "train_loss": 3.1061267256736755,
+        "val_loss": 3.1822536826133727,
+        "lr": 0.02,
+        "tokens_per_sec": 699657.8714128656,
+        "mfu": 6.577468372779611,
+        "total_flops": 1.5025477335279206e+17
+    },
+    {
+        "step": 6100,
+        "train_loss": 3.151519775390625,
+        "val_loss": 3.18179247379303,
+        "lr": 0.02,
+        "tokens_per_sec": 557667.215608958,
+        "mfu": 5.242617317799736,
+        "total_flops": 1.5149634312103526e+17
+    },
+    {
+        "step": 6150,
+        "train_loss": 3.080659031867981,
+        "val_loss": 3.18075053691864,
+        "lr": 0.02,
+        "tokens_per_sec": 726509.7037583116,
+        "mfu": 6.829901862374601,
+        "total_flops": 1.5273791288927846e+17
+    },
+    {
+        "step": 6200,
+        "train_loss": 3.0890342593193054,
+        "val_loss": 3.179175615310669,
+        "lr": 0.02,
+        "tokens_per_sec": 699340.2421265863,
+        "mfu": 6.574482346794437,
+        "total_flops": 1.5397948265752166e+17
+    },
+    {
+        "step": 6250,
+        "train_loss": 3.1207754611968994,
+        "val_loss": 3.1796735525131226,
+        "lr": 0.02,
+        "tokens_per_sec": 757885.2205318413,
+        "mfu": 7.1248624105076015,
+        "total_flops": 1.5522105242576486e+17
+    },
+    {
+        "step": 6300,
+        "train_loss": 3.0962599515914917,
+        "val_loss": 3.180558133125305,
+        "lr": 0.02,
+        "tokens_per_sec": 667449.8589065681,
+        "mfu": 6.274681550439623,
+        "total_flops": 1.5646262219400806e+17
+    },
+    {
+        "step": 6350,
+        "train_loss": 2.9786906838417053,
+        "val_loss": 3.18046555519104,
+        "lr": 0.02,
+        "tokens_per_sec": 694050.5334623031,
+        "mfu": 6.524753911137333,
+        "total_flops": 1.5770419196225126e+17
+    },
+    {
+        "step": 6400,
+        "train_loss": 3.049709677696228,
+        "val_loss": 3.1755032181739806,
+        "lr": 0.02,
+        "tokens_per_sec": 665420.704705452,
+        "mfu": 6.255605516099607,
+        "total_flops": 1.5894576173049446e+17
+    },
+    {
+        "step": 6450,
+        "train_loss": 3.163714349269867,
+        "val_loss": 3.177147853374481,
+        "lr": 0.02,
+        "tokens_per_sec": 671713.0309627261,
+        "mfu": 6.314759537857205,
+        "total_flops": 1.6018733149873766e+17
+    },
+    {
+        "step": 6500,
+        "train_loss": 3.1299668550491333,
+        "val_loss": 3.179553759098053,
+        "lr": 0.02,
+        "tokens_per_sec": 750298.6979698391,
+        "mfu": 7.053541677546817,
+        "total_flops": 1.6142890126698086e+17
+    },
+    {
+        "step": 6550,
+        "train_loss": 3.1561201214790344,
+        "val_loss": 3.176179659366608,
+        "lr": 0.02,
+        "tokens_per_sec": 680751.6339691349,
+        "mfu": 6.399731247371044,
+        "total_flops": 1.6267047103522406e+17
+    },
+    {
+        "step": 6600,
+        "train_loss": 3.204793870449066,
+        "val_loss": 3.174552595615387,
+        "lr": 0.02,
+        "tokens_per_sec": 711036.3609291689,
+        "mfu": 6.684437304283743,
+        "total_flops": 1.6391204080346726e+17
+    },
+    {
+        "step": 6650,
+        "train_loss": 3.143680512905121,
+        "val_loss": 3.178517699241638,
+        "lr": 0.02,
+        "tokens_per_sec": 716431.4505102517,
+        "mfu": 6.735156423638787,
+        "total_flops": 1.6515361057171046e+17
+    },
+    {
+        "step": 6700,
+        "train_loss": 3.075408637523651,
+        "val_loss": 3.1751816272735596,
+        "lr": 0.02,
+        "tokens_per_sec": 704070.4786472794,
+        "mfu": 6.618951197045199,
+        "total_flops": 1.6639518033995366e+17
+    },
+    {
+        "step": 6750,
+        "train_loss": 3.104284882545471,
+        "val_loss": 3.1782999992370606,
+        "lr": 0.02,
+        "tokens_per_sec": 696606.8731664469,
+        "mfu": 6.548786004880711,
+        "total_flops": 1.6763675010819686e+17
+    },
+    {
+        "step": 6800,
+        "train_loss": 3.0922266840934753,
+        "val_loss": 3.1723532557487486,
+        "lr": 0.02,
+        "tokens_per_sec": 769887.0884655679,
+        "mfu": 7.237691708903047,
+        "total_flops": 1.6887831987644006e+17
+    },
+    {
+        "step": 6850,
+        "train_loss": 3.156929671764374,
+        "val_loss": 3.17406679391861,
+        "lr": 0.02,
+        "tokens_per_sec": 674543.6381873306,
+        "mfu": 6.34137001457206,
+        "total_flops": 1.7011988964468326e+17
+    },
+    {
+        "step": 6900,
+        "train_loss": 3.13032203912735,
+        "val_loss": 3.181139957904816,
+        "lr": 0.02,
+        "tokens_per_sec": 711158.9804319512,
+        "mfu": 6.685590047552117,
+        "total_flops": 1.7136145941292646e+17
+    },
+    {
+        "step": 6950,
+        "train_loss": 3.0561442375183105,
+        "val_loss": 3.171230399608612,
+        "lr": 0.02,
+        "tokens_per_sec": 657218.3994794192,
+        "mfu": 6.178495823759307,
+        "total_flops": 1.7260302918116966e+17
+    },
+    {
+        "step": 7000,
+        "train_loss": 3.1265305876731873,
+        "val_loss": 3.170287549495697,
+        "lr": 0.02,
+        "tokens_per_sec": 705210.176723673,
+        "mfu": 6.6296654737771386,
+        "total_flops": 1.7384459894941286e+17
+    },
+    {
+        "step": 7050,
+        "train_loss": 3.104595959186554,
+        "val_loss": 3.1692514777183534,
+        "lr": 0.02,
+        "tokens_per_sec": 674145.6150938105,
+        "mfu": 6.337628208160338,
+        "total_flops": 1.7508616871765606e+17
+    },
+    {
+        "step": 7100,
+        "train_loss": 3.152307689189911,
+        "val_loss": 3.177652657032013,
+        "lr": 0.02,
+        "tokens_per_sec": 659974.2035204765,
+        "mfu": 6.204403077378892,
+        "total_flops": 1.7632773848589926e+17
+    },
+    {
+        "step": 7150,
+        "train_loss": 3.120695650577545,
+        "val_loss": 3.166973686218262,
+        "lr": 0.02,
+        "tokens_per_sec": 756319.1076230536,
+        "mfu": 7.110139417246694,
+        "total_flops": 1.7756930825414246e+17
+    },
+    {
+        "step": 7200,
+        "train_loss": 3.116680860519409,
+        "val_loss": 3.1675556182861326,
+        "lr": 0.02,
+        "tokens_per_sec": 736554.6306746806,
+        "mfu": 6.924334000993837,
+        "total_flops": 1.7881087802238566e+17
+    },
+    {
+        "step": 7250,
+        "train_loss": 3.1216148138046265,
+        "val_loss": 3.176751661300659,
+        "lr": 0.02,
+        "tokens_per_sec": 768497.8671271454,
+        "mfu": 7.224631669432881,
+        "total_flops": 1.8005244779062886e+17
+    },
+    {
+        "step": 7300,
+        "train_loss": 3.1001328229904175,
+        "val_loss": 3.1768130898475646,
+        "lr": 0.02,
+        "tokens_per_sec": 701846.6496135799,
+        "mfu": 6.598045028854608,
+        "total_flops": 1.8129401755887206e+17
+    },
+    {
+        "step": 7350,
+        "train_loss": 3.0882458090782166,
+        "val_loss": 3.1811372876167296,
+        "lr": 0.02,
+        "tokens_per_sec": 664789.1802584612,
+        "mfu": 6.249668568562189,
+        "total_flops": 1.8253558732711526e+17
+    },
+    {
+        "step": 7400,
+        "train_loss": 3.0628254413604736,
+        "val_loss": 3.167879915237427,
+        "lr": 0.02,
+        "tokens_per_sec": 674304.7162599304,
+        "mfu": 6.339123914749206,
+        "total_flops": 1.8377715709535846e+17
+    },
+    {
+        "step": 7450,
+        "train_loss": 3.06341552734375,
+        "val_loss": 3.172988545894623,
+        "lr": 0.02,
+        "tokens_per_sec": 679328.3677658916,
+        "mfu": 6.386351152870034,
+        "total_flops": 1.8501872686360166e+17
+    },
+    {
+        "step": 7500,
+        "train_loss": 3.0606775879859924,
+        "val_loss": 3.1777786135673525,
+        "lr": 0.02,
+        "tokens_per_sec": 782756.6463094545,
+        "mfu": 7.358678141198712,
+        "total_flops": 1.8626029663184486e+17
+    },
+    {
+        "step": 7550,
+        "train_loss": 3.0959790349006653,
+        "val_loss": 3.180156445503235,
+        "lr": 0.02,
+        "tokens_per_sec": 710747.656465127,
+        "mfu": 6.681723199920876,
+        "total_flops": 1.8750186640008806e+17
+    },
+    {
+        "step": 7600,
+        "train_loss": 3.049330413341522,
+        "val_loss": 3.1722224354743958,
+        "lr": 0.02,
+        "tokens_per_sec": 686499.064916994,
+        "mfu": 6.453762720222107,
+        "total_flops": 1.8874343616833126e+17
+    },
+    {
+        "step": 7650,
+        "train_loss": 3.0719059705734253,
+        "val_loss": 3.167457950115204,
+        "lr": 0.02,
+        "tokens_per_sec": 711523.1675032718,
+        "mfu": 6.68901376225792,
+        "total_flops": 1.8998500593657446e+17
+    },
+    {
+        "step": 7700,
+        "train_loss": 3.0947948694229126,
+        "val_loss": 3.1769357442855837,
+        "lr": 0.02,
+        "tokens_per_sec": 701649.4700183949,
+        "mfu": 6.596191347785505,
+        "total_flops": 1.9122657570481766e+17
+    },
+    {
+        "step": 7750,
+        "train_loss": 3.044970750808716,
+        "val_loss": 3.183099925518036,
+        "lr": 0.02,
+        "tokens_per_sec": 697595.9668251104,
+        "mfu": 6.558084452769876,
+        "total_flops": 1.9246814547306086e+17
+    },
+    {
+        "step": 7800,
+        "train_loss": 3.085705578327179,
+        "val_loss": 3.1682560086250304,
+        "lr": 0.02,
+        "tokens_per_sec": 690917.0614373354,
+        "mfu": 6.495296209047032,
+        "total_flops": 1.9370971524130406e+17
+    },
+    {
+        "step": 7850,
+        "train_loss": 3.047266960144043,
+        "val_loss": 3.1687360525131227,
+        "lr": 0.02,
+        "tokens_per_sec": 718745.4816144382,
+        "mfu": 6.756910579524549,
+        "total_flops": 1.9495128500954726e+17
+    },
+    {
+        "step": 7900,
+        "train_loss": 3.1841835379600525,
+        "val_loss": 3.169699013233185,
+        "lr": 0.02,
+        "tokens_per_sec": 705554.4630634915,
+        "mfu": 6.632902102140603,
+        "total_flops": 1.9619285477779046e+17
+    },
+    {
+        "step": 7950,
+        "train_loss": 3.1347849369049072,
+        "val_loss": 3.1597500681877135,
+        "lr": 0.02,
+        "tokens_per_sec": 752479.9128405473,
+        "mfu": 7.074047230921565,
+        "total_flops": 1.9743442454603366e+17
+    },
+    {
+        "step": 8000,
+        "train_loss": 3.133264899253845,
+        "val_loss": 3.169278311729431,
+        "lr": 0.02,
+        "tokens_per_sec": 714196.2511140726,
+        "mfu": 6.7141433629187866,
+        "total_flops": 1.9867599431427686e+17
+    },
+    {
+        "step": 8050,
+        "train_loss": 3.0986027121543884,
+        "val_loss": 3.1637734532356263,
+        "lr": 0.02,
+        "tokens_per_sec": 717463.4166620161,
+        "mfu": 6.744857914899511,
+        "total_flops": 1.9991756408252006e+17
+    },
+    {
+        "step": 8100,
+        "train_loss": 3.2134180665016174,
+        "val_loss": 3.157232141494751,
+        "lr": 0.02,
+        "tokens_per_sec": 688577.3122537801,
+        "mfu": 6.4733002780586535,
+        "total_flops": 2.0115913385076326e+17
+    },
+    {
+        "step": 8150,
+        "train_loss": 3.1356531381607056,
+        "val_loss": 3.156856894493103,
+        "lr": 0.02,
+        "tokens_per_sec": 707653.2779380869,
+        "mfu": 6.6526330149510216,
+        "total_flops": 2.0240070361900646e+17
+    },
+    {
+        "step": 8200,
+        "train_loss": 3.1011769771575928,
+        "val_loss": 3.1655483722686766,
+        "lr": 0.02,
+        "tokens_per_sec": 705070.7567580029,
+        "mfu": 6.6283547897239785,
+        "total_flops": 2.0364227338724966e+17
+    },
+    {
+        "step": 8250,
+        "train_loss": 3.043505847454071,
+        "val_loss": 3.1673601865768433,
+        "lr": 0.02,
+        "tokens_per_sec": 712346.4170891232,
+        "mfu": 6.696753113639627,
+        "total_flops": 2.0488384315549286e+17
+    },
+    {
+        "step": 8300,
+        "train_loss": 3.0787187814712524,
+        "val_loss": 3.167344641685486,
+        "lr": 0.02,
+        "tokens_per_sec": 726796.4044192191,
+        "mfu": 6.832597129027953,
+        "total_flops": 2.0612541292373606e+17
+    },
+    {
+        "step": 8350,
+        "train_loss": 3.04863178730011,
+        "val_loss": 3.163574528694153,
+        "lr": 0.02,
+        "tokens_per_sec": 684120.0113477479,
+        "mfu": 6.431397289562027,
+        "total_flops": 2.0736698269197926e+17
+    },
+    {
+        "step": 8400,
+        "train_loss": 3.118406891822815,
+        "val_loss": 3.1581183552742003,
+        "lr": 0.02,
+        "tokens_per_sec": 714684.0160097927,
+        "mfu": 6.718728830053563,
+        "total_flops": 2.0860855246022246e+17
+    },
+    {
+        "step": 8450,
+        "train_loss": 3.094644069671631,
+        "val_loss": 3.1536608576774596,
+        "lr": 0.02,
+        "tokens_per_sec": 757979.1997237524,
+        "mfu": 7.125745906838811,
+        "total_flops": 2.0985012222846566e+17
+    },
+    {
+        "step": 8500,
+        "train_loss": 3.061720073223114,
+        "val_loss": 3.147541642189026,
+        "lr": 0.02,
+        "tokens_per_sec": 727306.0241525556,
+        "mfu": 6.837388053013979,
+        "total_flops": 2.1109169199670886e+17
+    },
+    {
+        "step": 8550,
+        "train_loss": 3.11970978975296,
+        "val_loss": 3.1589664101600645,
+        "lr": 0.02,
+        "tokens_per_sec": 704990.4558965408,
+        "mfu": 6.627599883078666,
+        "total_flops": 2.1233326176495206e+17
+    },
+    {
+        "step": 8600,
+        "train_loss": 3.0975873470306396,
+        "val_loss": 3.1601911067962645,
+        "lr": 0.02,
+        "tokens_per_sec": 740458.8535430809,
+        "mfu": 6.961037514934626,
+        "total_flops": 2.1357483153319526e+17
+    },
+    {
+        "step": 8650,
+        "train_loss": 3.055800497531891,
+        "val_loss": 3.158919131755829,
+        "lr": 0.02,
+        "tokens_per_sec": 422942.260395141,
+        "mfu": 3.9760709554275504,
+        "total_flops": 2.1481640130143846e+17
+    },
+    {
+        "step": 8700,
+        "train_loss": 3.0930920839309692,
+        "val_loss": 3.166372525691986,
+        "lr": 0.02,
+        "tokens_per_sec": 737549.19874948,
+        "mfu": 6.933683913749565,
+        "total_flops": 2.1605797106968166e+17
+    },
+    {
+        "step": 8750,
+        "train_loss": 3.1593551635742188,
+        "val_loss": 3.1717628717422484,
+        "lr": 0.02,
+        "tokens_per_sec": 763392.7556223898,
+        "mfu": 7.1766386276419905,
+        "total_flops": 2.1729954083792486e+17
+    },
+    {
+        "step": 8800,
+        "train_loss": 3.046756327152252,
+        "val_loss": 3.166207802295685,
+        "lr": 0.02,
+        "tokens_per_sec": 742377.0576086775,
+        "mfu": 6.979070509473116,
+        "total_flops": 2.1854111060616806e+17
+    },
+    {
+        "step": 8850,
+        "train_loss": 3.075532078742981,
+        "val_loss": 3.158578038215637,
+        "lr": 0.02,
+        "tokens_per_sec": 709000.5443841416,
+        "mfu": 6.665298637394083,
+        "total_flops": 2.1978268037441126e+17
+    },
+    {
+        "step": 8900,
+        "train_loss": 3.062712073326111,
+        "val_loss": 3.1605077266693113,
+        "lr": 0.02,
+        "tokens_per_sec": 713368.5260956465,
+        "mfu": 6.7063619380931625,
+        "total_flops": 2.2102425014265446e+17
+    },
+    {
+        "step": 8950,
+        "train_loss": 3.1221720576286316,
+        "val_loss": 3.1749680519104,
+        "lr": 0.02,
+        "tokens_per_sec": 703279.3709544374,
+        "mfu": 6.611514010897886,
+        "total_flops": 2.2226581991089766e+17
+    },
+    {
+        "step": 9000,
+        "train_loss": 3.106475830078125,
+        "val_loss": 3.1677522897720336,
+        "lr": 0.02,
+        "tokens_per_sec": 735564.6727283602,
+        "mfu": 6.915027427955281,
+        "total_flops": 2.2350738967914086e+17
+    },
+    {
+        "step": 9050,
+        "train_loss": 3.0899603962898254,
+        "val_loss": 3.160882556438446,
+        "lr": 0.02,
+        "tokens_per_sec": 681783.8537221702,
+        "mfu": 6.40943512273178,
+        "total_flops": 2.2474895944738406e+17
+    },
+    {
+        "step": 9100,
+        "train_loss": 3.0604807138442993,
+        "val_loss": 3.156177818775177,
+        "lr": 0.02,
+        "tokens_per_sec": 712196.8495005193,
+        "mfu": 6.69534703200483,
+        "total_flops": 2.2599052921562726e+17
+    },
+    {
+        "step": 9150,
+        "train_loss": 3.1883918046951294,
+        "val_loss": 3.1617040753364565,
+        "lr": 0.02,
+        "tokens_per_sec": 757768.7291387941,
+        "mfu": 7.123767277465046,
+        "total_flops": 2.2723209898387046e+17
+    },
+    {
+        "step": 9200,
+        "train_loss": 3.1464973092079163,
+        "val_loss": 3.1746654391288756,
+        "lr": 0.02,
+        "tokens_per_sec": 689372.4842663524,
+        "mfu": 6.480775672787012,
+        "total_flops": 2.2847366875211366e+17
+    },
+    {
+        "step": 9250,
+        "train_loss": 3.0570514798164368,
+        "val_loss": 3.1565617084503175,
+        "lr": 0.02,
+        "tokens_per_sec": 744331.5530949961,
+        "mfu": 6.997444678865422,
+        "total_flops": 2.2971523852035686e+17
+    },
+    {
+        "step": 9300,
+        "train_loss": 3.1816789507865906,
+        "val_loss": 3.161880099773407,
+        "lr": 0.02,
+        "tokens_per_sec": 712816.8001758846,
+        "mfu": 6.701175174767898,
+        "total_flops": 2.3095680828860006e+17
+    },
+    {
+        "step": 9350,
+        "train_loss": 3.117702007293701,
+        "val_loss": 3.1647464752197267,
+        "lr": 0.02,
+        "tokens_per_sec": 735756.5505862661,
+        "mfu": 6.916831267507982,
+        "total_flops": 2.3219837805684326e+17
+    },
+    {
+        "step": 9400,
+        "train_loss": 3.143872082233429,
+        "val_loss": 3.16401606798172,
+        "lr": 0.02,
+        "tokens_per_sec": 717769.7278227371,
+        "mfu": 6.747737539433432,
+        "total_flops": 2.3343994782508646e+17
+    },
+    {
+        "step": 9450,
+        "train_loss": 3.0404112339019775,
+        "val_loss": 3.158752131462097,
+        "lr": 0.02,
+        "tokens_per_sec": 742365.9928105028,
+        "mfu": 6.978966489547064,
+        "total_flops": 2.3468151759332966e+17
+    },
+    {
+        "step": 9500,
+        "train_loss": 3.1219199895858765,
+        "val_loss": 3.1590757966041565,
+        "lr": 0.02,
+        "tokens_per_sec": 625727.6886516663,
+        "mfu": 5.882452338837707,
+        "total_flops": 2.3592308736157286e+17
+    },
+    {
+        "step": 9550,
+        "train_loss": 3.1075311303138733,
+        "val_loss": 3.161842477321625,
+        "lr": 0.02,
+        "tokens_per_sec": 721577.0335884815,
+        "mfu": 6.783529937808249,
+        "total_flops": 2.3716465712981606e+17
+    },
+    {
+        "step": 9600,
+        "train_loss": 3.1048957109451294,
+        "val_loss": 3.163857471942902,
+        "lr": 0.02,
+        "tokens_per_sec": 806568.6400450236,
+        "mfu": 7.5825341743942705,
+        "total_flops": 2.3840622689805926e+17
+    },
+    {
+        "step": 9650,
+        "train_loss": 2.975189983844757,
+        "val_loss": 3.1734387636184693,
+        "lr": 0.02,
+        "tokens_per_sec": 745568.3417033429,
+        "mfu": 7.009071701568373,
+        "total_flops": 2.3964779666630246e+17
+    },
+    {
+        "step": 9700,
+        "train_loss": 3.0660216212272644,
+        "val_loss": 3.171565854549408,
+        "lr": 0.02,
+        "tokens_per_sec": 654774.0789957069,
+        "mfu": 6.1555168202613695,
+        "total_flops": 2.4088936643454566e+17
+    },
+    {
+        "step": 9750,
+        "train_loss": 3.099682331085205,
+        "val_loss": 3.159890115261078,
+        "lr": 0.02,
+        "tokens_per_sec": 748786.3525892427,
+        "mfu": 7.03932415164447,
+        "total_flops": 2.4213093620278886e+17
+    },
+    {
+        "step": 9800,
+        "train_loss": 3.0244967341423035,
+        "val_loss": 3.1675970196723937,
+        "lr": 0.02,
+        "tokens_per_sec": 728149.514021465,
+        "mfu": 6.8453176828547875,
+        "total_flops": 2.4337250597103206e+17
+    },
+    {
+        "step": 9850,
+        "train_loss": 3.16311913728714,
+        "val_loss": 3.1630403995513916,
+        "lr": 0.02,
+        "tokens_per_sec": 762955.907496443,
+        "mfu": 7.172531827948133,
+        "total_flops": 2.4461407573927526e+17
+    },
+    {
+        "step": 9900,
+        "train_loss": 3.0646278262138367,
+        "val_loss": 3.166551649570465,
+        "lr": 0.02,
+        "tokens_per_sec": 699854.4342206167,
+        "mfu": 6.579316255443516,
+        "total_flops": 2.4585564550751846e+17
+    },
+    {
+        "step": 9950,
+        "train_loss": 3.0938143134117126,
+        "val_loss": 3.170255482196808,
+        "lr": 0.02,
+        "tokens_per_sec": 590216.7351573871,
+        "mfu": 5.548614640386318,
+        "total_flops": 2.4709721527576166e+17
+    },
+    {
+        "step": 10000,
+        "train_loss": 3.0690277814865112,
+        "val_loss": 3.1615793347358703,
+        "lr": 0.02,
+        "tokens_per_sec": 753254.4144864467,
+        "mfu": 7.081328303984148,
+        "total_flops": 2.4833878504400486e+17
+    },
+    {
+        "step": 10050,
+        "train_loss": 3.0185793042182922,
+        "val_loss": 3.160174882411957,
+        "lr": 0.02,
+        "tokens_per_sec": 696998.2878720823,
+        "mfu": 6.5524656859822175,
+        "total_flops": 2.4958035481224806e+17
+    },
+    {
+        "step": 10100,
+        "train_loss": 3.115284562110901,
+        "val_loss": 3.1621490120887756,
+        "lr": 0.02,
+        "tokens_per_sec": 571145.0607667634,
+        "mfu": 5.369322245852169,
+        "total_flops": 2.5082192458049126e+17
+    },
+    {
+        "step": 10150,
+        "train_loss": 3.1012810468673706,
+        "val_loss": 3.160170817375183,
+        "lr": 0.02,
+        "tokens_per_sec": 703584.5720835248,
+        "mfu": 6.614383200048655,
+        "total_flops": 2.5206349434873446e+17
+    },
+    {
+        "step": 10200,
+        "train_loss": 3.054893910884857,
+        "val_loss": 3.1524683117866514,
+        "lr": 0.02,
+        "tokens_per_sec": 733004.6487976534,
+        "mfu": 6.890960698878425,
+        "total_flops": 2.5330506411697766e+17
+    },
+    {
+        "step": 10250,
+        "train_loss": 3.079162895679474,
+        "val_loss": 3.1491398572921754,
+        "lr": 0.02,
+        "tokens_per_sec": 737557.201350517,
+        "mfu": 6.933759146027185,
+        "total_flops": 2.5454663388522086e+17
+    },
+    {
+        "step": 10300,
+        "train_loss": 3.1038126945495605,
+        "val_loss": 3.1571540594100953,
+        "lr": 0.02,
+        "tokens_per_sec": 720052.58965927,
+        "mfu": 6.769198673714534,
+        "total_flops": 2.5578820365346406e+17
+    },
+    {
+        "step": 10350,
+        "train_loss": 3.0655335187911987,
+        "val_loss": 3.155700421333313,
+        "lr": 0.02,
+        "tokens_per_sec": 689044.8419944294,
+        "mfu": 6.4776955149424245,
+        "total_flops": 2.5702977342170726e+17
+    },
+    {
+        "step": 10400,
+        "train_loss": 3.1391899585723877,
+        "val_loss": 3.154104471206665,
+        "lr": 0.02,
+        "tokens_per_sec": 731350.8642784627,
+        "mfu": 6.875413506722346,
+        "total_flops": 2.5827134318995046e+17
+    },
+    {
+        "step": 10450,
+        "train_loss": 3.118773400783539,
+        "val_loss": 3.1520774245262144,
+        "lr": 0.02,
+        "tokens_per_sec": 719848.7605407503,
+        "mfu": 6.767282480621748,
+        "total_flops": 2.5951291295819366e+17
+    },
+    {
+        "step": 10500,
+        "train_loss": 3.1492371559143066,
+        "val_loss": 3.152772569656372,
+        "lr": 0.02,
+        "tokens_per_sec": 367770.51127847325,
+        "mfu": 3.4574025466051,
+        "total_flops": 2.6075448272643686e+17
+    },
+    {
+        "step": 10550,
+        "train_loss": 3.1020652055740356,
+        "val_loss": 3.1449167728424072,
+        "lr": 0.02,
+        "tokens_per_sec": 751263.41527877,
+        "mfu": 7.06261096390438,
+        "total_flops": 2.6199605249468006e+17
+    },
+    {
+        "step": 10600,
+        "train_loss": 3.041691780090332,
+        "val_loss": 3.1420316219329836,
+        "lr": 0.02,
+        "tokens_per_sec": 741753.7205190589,
+        "mfu": 6.973210531103592,
+        "total_flops": 2.6323762226292326e+17
+    },
+    {
+        "step": 10650,
+        "train_loss": 3.117504835128784,
+        "val_loss": 3.1396547675132753,
+        "lr": 0.02,
+        "tokens_per_sec": 773011.1429140648,
+        "mfu": 7.267060876562572,
+        "total_flops": 2.6447919203116646e+17
+    },
+    {
+        "step": 10700,
+        "train_loss": 3.0745006799697876,
+        "val_loss": 3.1497706532478333,
+        "lr": 0.02,
+        "tokens_per_sec": 733934.356207618,
+        "mfu": 6.899700857940215,
+        "total_flops": 2.6572076179940966e+17
+    },
+    {
+        "step": 10750,
+        "train_loss": 3.128191292285919,
+        "val_loss": 3.1601159811019897,
+        "lr": 0.02,
+        "tokens_per_sec": 714878.0629070438,
+        "mfu": 6.720553060697789,
+        "total_flops": 2.6696233156765286e+17
+    },
+    {
+        "step": 10800,
+        "train_loss": 2.9438751339912415,
+        "val_loss": 3.1561400055885316,
+        "lr": 0.02,
+        "tokens_per_sec": 698684.514108107,
+        "mfu": 6.568317862009919,
+        "total_flops": 2.6820390133589606e+17
+    },
+    {
+        "step": 10850,
+        "train_loss": 3.1292802691459656,
+        "val_loss": 3.1578634858131407,
+        "lr": 0.02,
+        "tokens_per_sec": 732158.2530349453,
+        "mfu": 6.883003750793463,
+        "total_flops": 2.6944547110413926e+17
+    },
+    {
+        "step": 10900,
+        "train_loss": 3.072293758392334,
+        "val_loss": 3.1496147394180296,
+        "lr": 0.02,
+        "tokens_per_sec": 744635.3570570046,
+        "mfu": 7.0003007332789915,
+        "total_flops": 2.7068704087238246e+17
+    },
+    {
+        "step": 10950,
+        "train_loss": 3.1238223910331726,
+        "val_loss": 3.1552544713020323,
+        "lr": 0.02,
+        "tokens_per_sec": 773699.7554751125,
+        "mfu": 7.27353450821377,
+        "total_flops": 2.7192861064062566e+17
+    },
+    {
+        "step": 11000,
+        "train_loss": 3.1469348073005676,
+        "val_loss": 3.160993766784668,
+        "lr": 0.02,
+        "tokens_per_sec": 721327.0859497564,
+        "mfu": 6.781180185513967,
+        "total_flops": 2.7317018040886886e+17
+    },
+    {
+        "step": 11050,
+        "train_loss": 3.070757746696472,
+        "val_loss": 3.1459980487823485,
+        "lr": 0.02,
+        "tokens_per_sec": 711585.7360416645,
+        "mfu": 6.689601967721225,
+        "total_flops": 2.7441175017711206e+17
+    },
+    {
+        "step": 11100,
+        "train_loss": 3.065983474254608,
+        "val_loss": 3.1586222529411314,
+        "lr": 0.02,
+        "tokens_per_sec": 715606.969297743,
+        "mfu": 6.727405493761767,
+        "total_flops": 2.7565331994535526e+17
+    },
+    {
+        "step": 11150,
+        "train_loss": 3.1281445622444153,
+        "val_loss": 3.1559403777122497,
+        "lr": 0.02,
+        "tokens_per_sec": 697343.4573019198,
+        "mfu": 6.555710616255669,
+        "total_flops": 2.7689488971359846e+17
+    },
+    {
+        "step": 11200,
+        "train_loss": 3.0846157670021057,
+        "val_loss": 3.151843547821045,
+        "lr": 0.02,
+        "tokens_per_sec": 698598.7589954758,
+        "mfu": 6.567511680068455,
+        "total_flops": 2.7813645948184166e+17
+    },
+    {
+        "step": 11250,
+        "train_loss": 3.005524456501007,
+        "val_loss": 3.1599945425987244,
+        "lr": 0.02,
+        "tokens_per_sec": 644463.4567446798,
+        "mfu": 6.058586885602221,
+        "total_flops": 2.7937802925008486e+17
+    },
+    {
+        "step": 11300,
+        "train_loss": 3.1120718121528625,
+        "val_loss": 3.1506922483444213,
+        "lr": 0.02,
+        "tokens_per_sec": 703597.1449334478,
+        "mfu": 6.614501397136266,
+        "total_flops": 2.8061959901832806e+17
+    },
+    {
+        "step": 11350,
+        "train_loss": 3.0928125381469727,
+        "val_loss": 3.171697747707367,
+        "lr": 0.02,
+        "tokens_per_sec": 790605.0705730701,
+        "mfu": 7.432461006337987,
+        "total_flops": 2.8186116878657126e+17
+    },
+    {
+        "step": 11400,
+        "train_loss": 3.0579270720481873,
+        "val_loss": 3.155070924758911,
+        "lr": 0.02,
+        "tokens_per_sec": 674691.2002795199,
+        "mfu": 6.342757242578852,
+        "total_flops": 2.8310273855481446e+17
+    },
+    {
+        "step": 11450,
+        "train_loss": 3.107338547706604,
+        "val_loss": 3.154102897644043,
+        "lr": 0.02,
+        "tokens_per_sec": 749453.6631067256,
+        "mfu": 7.045597523249228,
+        "total_flops": 2.8434430832305766e+17
+    },
+    {
+        "step": 11500,
+        "train_loss": 3.034080922603607,
+        "val_loss": 3.1530386328697206,
+        "lr": 0.02,
+        "tokens_per_sec": 711996.9935045289,
+        "mfu": 6.693468190150191,
+        "total_flops": 2.8558587809130086e+17
+    },
+    {
+        "step": 11550,
+        "train_loss": 3.0200106501579285,
+        "val_loss": 3.163648450374603,
+        "lr": 0.02,
+        "tokens_per_sec": 699272.63712686,
+        "mfu": 6.5738467936681015,
+        "total_flops": 2.8682744785954406e+17
+    },
+    {
+        "step": 11600,
+        "train_loss": 3.127845346927643,
+        "val_loss": 3.164807939529419,
+        "lr": 0.02,
+        "tokens_per_sec": 705079.2456450074,
+        "mfu": 6.628434593565359,
+        "total_flops": 2.8806901762778726e+17
+    },
+    {
+        "step": 11650,
+        "train_loss": 3.0393056869506836,
+        "val_loss": 3.160170352458954,
+        "lr": 0.02,
+        "tokens_per_sec": 716858.8806831492,
+        "mfu": 6.739174685361659,
+        "total_flops": 2.8931058739603046e+17
+    },
+    {
+        "step": 11700,
+        "train_loss": 3.0907124280929565,
+        "val_loss": 3.155959928035736,
+        "lr": 0.02,
+        "tokens_per_sec": 749071.7197254659,
+        "mfu": 7.042006881861382,
+        "total_flops": 2.9055215716427366e+17
+    },
+    {
+        "step": 11750,
+        "train_loss": 3.0880022048950195,
+        "val_loss": 3.150728130340576,
+        "lr": 0.02,
+        "tokens_per_sec": 694510.9299758312,
+        "mfu": 6.529082088708685,
+        "total_flops": 2.9179372693251686e+17
+    },
+    {
+        "step": 11800,
+        "train_loss": 3.127938985824585,
+        "val_loss": 3.1598729610443117,
+        "lr": 0.02,
+        "tokens_per_sec": 775348.4312721618,
+        "mfu": 7.289033673384543,
+        "total_flops": 2.9303529670076006e+17
+    },
+    {
+        "step": 11850,
+        "train_loss": 3.1025193333625793,
+        "val_loss": 3.1526888370513917,
+        "lr": 0.02,
+        "tokens_per_sec": 299671.53479248524,
+        "mfu": 2.8172055555375426,
+        "total_flops": 2.9427686646900326e+17
+    },
+    {
+        "step": 11900,
+        "train_loss": 3.1594666242599487,
+        "val_loss": 3.1516989946365355,
+        "lr": 0.02,
+        "tokens_per_sec": 833475.199704364,
+        "mfu": 7.835482154272261,
+        "total_flops": 2.9551843623724646e+17
+    },
+    {
+        "step": 11950,
+        "train_loss": 3.0817718505859375,
+        "val_loss": 3.152645695209503,
+        "lr": 0.02,
+        "tokens_per_sec": 749447.2775898139,
+        "mfu": 7.045537493144163,
+        "total_flops": 2.9676000600548966e+17
+    },
+    {
+        "step": 12000,
+        "train_loss": 3.145545721054077,
+        "val_loss": 3.151441919803619,
+        "lr": 0.02,
+        "tokens_per_sec": 747379.828585372,
+        "mfu": 7.026101450194214,
+        "total_flops": 2.9800157577373286e+17
+    },
+    {
+        "step": 12050,
+        "train_loss": 3.094115436077118,
+        "val_loss": 3.1536296248435973,
+        "lr": 0.02,
+        "tokens_per_sec": 615306.9817176713,
+        "mfu": 5.784487500477572,
+        "total_flops": 2.9924314554197606e+17
+    },
+    {
+        "step": 12100,
+        "train_loss": 3.088757812976837,
+        "val_loss": 3.147271144390106,
+        "lr": 0.02,
+        "tokens_per_sec": 704596.9580772287,
+        "mfu": 6.6239006186709695,
+        "total_flops": 3.0048471531021926e+17
+    },
+    {
+        "step": 12150,
+        "train_loss": 3.090001881122589,
+        "val_loss": 3.1499510526657106,
+        "lr": 0.02,
+        "tokens_per_sec": 698254.1863021854,
+        "mfu": 6.564272359702253,
+        "total_flops": 3.0172628507846246e+17
+    },
+    {
+        "step": 12200,
+        "train_loss": 3.1208192706108093,
+        "val_loss": 3.155332291126251,
+        "lr": 0.02,
+        "tokens_per_sec": 720833.5932935321,
+        "mfu": 6.77654087182775,
+        "total_flops": 3.0296785484670566e+17
+    },
+    {
+        "step": 12250,
+        "train_loss": 3.0598087310791016,
+        "val_loss": 3.1506024837493896,
+        "lr": 0.02,
+        "tokens_per_sec": 652741.7328201098,
+        "mfu": 6.136410778238953,
+        "total_flops": 3.0420942461494886e+17
+    },
+    {
+        "step": 12300,
+        "train_loss": 3.106735408306122,
+        "val_loss": 3.146341931819916,
+        "lr": 0.02,
+        "tokens_per_sec": 709632.3556119555,
+        "mfu": 6.671238280951758,
+        "total_flops": 3.0545099438319206e+17
+    },
+    {
+        "step": 12350,
+        "train_loss": 3.059627652168274,
+        "val_loss": 3.147243654727936,
+        "lr": 0.02,
+        "tokens_per_sec": 721662.8154768226,
+        "mfu": 6.784336371467576,
+        "total_flops": 3.0669256415143526e+17
+    },
+    {
+        "step": 12400,
+        "train_loss": 3.074007570743561,
+        "val_loss": 3.1499706149101256,
+        "lr": 0.02,
+        "tokens_per_sec": 635126.2628783196,
+        "mfu": 5.97080812993343,
+        "total_flops": 3.0793413391967846e+17
+    },
+    {
+        "step": 12450,
+        "train_loss": 3.0849021077156067,
+        "val_loss": 3.155875825881958,
+        "lr": 0.02,
+        "tokens_per_sec": 378615.5670563863,
+        "mfu": 3.55935667918165,
+        "total_flops": 3.0917570368792166e+17
+    },
+    {
+        "step": 12500,
+        "train_loss": 3.0780173540115356,
+        "val_loss": 3.1482291340827944,
+        "lr": 0.02,
+        "tokens_per_sec": 674222.0916978031,
+        "mfu": 6.338347163044679,
+        "total_flops": 3.1041727345616486e+17
+    },
+    {
+        "step": 12550,
+        "train_loss": 3.05746990442276,
+        "val_loss": 3.146533954143524,
+        "lr": 0.02,
+        "tokens_per_sec": 718523.8440063655,
+        "mfu": 6.754826969210305,
+        "total_flops": 3.1165884322440806e+17
+    },
+    {
+        "step": 12600,
+        "train_loss": 3.0032397508621216,
+        "val_loss": 3.15649356842041,
+        "lr": 0.02,
+        "tokens_per_sec": 715966.0417387835,
+        "mfu": 6.730781125939977,
+        "total_flops": 3.1290041299265126e+17
+    },
+    {
+        "step": 12650,
+        "train_loss": 3.0083073377609253,
+        "val_loss": 3.1501465797424317,
+        "lr": 0.02,
+        "tokens_per_sec": 726098.2613042835,
+        "mfu": 6.826033900847742,
+        "total_flops": 3.1414198276089446e+17
+    },
+    {
+        "step": 12700,
+        "train_loss": 3.0275267362594604,
+        "val_loss": 3.148030436038971,
+        "lr": 0.02,
+        "tokens_per_sec": 692495.3758321449,
+        "mfu": 6.510133908211622,
+        "total_flops": 3.1538355252913766e+17
+    },
+    {
+        "step": 12750,
+        "train_loss": 3.120089828968048,
+        "val_loss": 3.147168529033661,
+        "lr": 0.02,
+        "tokens_per_sec": 734716.3837822445,
+        "mfu": 6.907052682094441,
+        "total_flops": 3.1662512229738086e+17
+    },
+    {
+        "step": 12800,
+        "train_loss": 3.1195261478424072,
+        "val_loss": 3.1519615650177,
+        "lr": 0.02,
+        "tokens_per_sec": 734447.1703783188,
+        "mfu": 6.904521812762156,
+        "total_flops": 3.1786669206562406e+17
+    },
+    {
+        "step": 12850,
+        "train_loss": 3.092920422554016,
+        "val_loss": 3.1470771193504334,
+        "lr": 0.02,
+        "tokens_per_sec": 722598.539243233,
+        "mfu": 6.793133090164952,
+        "total_flops": 3.1910826183386726e+17
+    },
+    {
+        "step": 12900,
+        "train_loss": 2.993108630180359,
+        "val_loss": 3.1456865310668944,
+        "lr": 0.02,
+        "tokens_per_sec": 697838.9152378561,
+        "mfu": 6.560368405493547,
+        "total_flops": 3.2034983160211046e+17
+    },
+    {
+        "step": 12950,
+        "train_loss": 2.9819764494895935,
+        "val_loss": 3.1511509299278258,
+        "lr": 0.02,
+        "tokens_per_sec": 731947.2335040426,
+        "mfu": 6.88101996079086,
+        "total_flops": 3.2159140137035366e+17
+    },
+    {
+        "step": 13000,
+        "train_loss": 3.0769078731536865,
+        "val_loss": 3.1605732798576356,
+        "lr": 0.02,
+        "tokens_per_sec": 712998.6797673736,
+        "mfu": 6.702885020836316,
+        "total_flops": 3.2283297113859686e+17
+    },
+    {
+        "step": 13050,
+        "train_loss": 3.067193925380707,
+        "val_loss": 3.145203471183777,
+        "lr": 0.02,
+        "tokens_per_sec": 741974.4137104355,
+        "mfu": 6.975285262977095,
+        "total_flops": 3.2407454090684006e+17
+    },
+    {
+        "step": 13100,
+        "train_loss": 3.037872314453125,
+        "val_loss": 3.14087290763855,
+        "lr": 0.02,
+        "tokens_per_sec": 719613.4782381974,
+        "mfu": 6.765070596832616,
+        "total_flops": 3.2531611067508326e+17
+    },
+    {
+        "step": 13150,
+        "train_loss": 3.0800358653068542,
+        "val_loss": 3.149751353263855,
+        "lr": 0.02,
+        "tokens_per_sec": 806705.9466868389,
+        "mfu": 7.583824991136015,
+        "total_flops": 3.2655768044332646e+17
+    },
+    {
+        "step": 13200,
+        "train_loss": 3.0346257090568542,
+        "val_loss": 3.149294650554657,
+        "lr": 0.02,
+        "tokens_per_sec": 759897.65491187,
+        "mfu": 7.143781262174636,
+        "total_flops": 3.2779925021156966e+17
+    },
+    {
+        "step": 13250,
+        "train_loss": 3.115667760372162,
+        "val_loss": 3.141120672225952,
+        "lr": 0.02,
+        "tokens_per_sec": 760821.1147015138,
+        "mfu": 7.1524626874942,
+        "total_flops": 3.2904081997981286e+17
+    },
+    {
+        "step": 13300,
+        "train_loss": 3.044420897960663,
+        "val_loss": 3.1427155256271364,
+        "lr": 0.02,
+        "tokens_per_sec": 652259.2522328668,
+        "mfu": 6.1318749887727915,
+        "total_flops": 3.3028238974805606e+17
+    },
+    {
+        "step": 13350,
+        "train_loss": 3.0882224440574646,
+        "val_loss": 3.1480340123176576,
+        "lr": 0.02,
+        "tokens_per_sec": 612838.8821541268,
+        "mfu": 5.761284950369327,
+        "total_flops": 3.3152395951629926e+17
+    },
+    {
+        "step": 13400,
+        "train_loss": 3.0521661043167114,
+        "val_loss": 3.15398633480072,
+        "lr": 0.02,
+        "tokens_per_sec": 689524.5090688368,
+        "mfu": 6.4822048546358495,
+        "total_flops": 3.3276552928454246e+17
+    },
+    {
+        "step": 13450,
+        "train_loss": 3.0448684692382812,
+        "val_loss": 3.148828995227814,
+        "lr": 0.02,
+        "tokens_per_sec": 685100.4687505038,
+        "mfu": 6.440614548197968,
+        "total_flops": 3.3400709905278566e+17
+    },
+    {
+        "step": 13500,
+        "train_loss": 3.056766629219055,
+        "val_loss": 3.146988809108734,
+        "lr": 0.02,
+        "tokens_per_sec": 613608.7624056519,
+        "mfu": 5.768522577804301,
+        "total_flops": 3.3524866882102886e+17
+    },
+    {
+        "step": 13550,
+        "train_loss": 3.039847433567047,
+        "val_loss": 3.150434172153473,
+        "lr": 0.02,
+        "tokens_per_sec": 690282.3138646273,
+        "mfu": 6.48932896097512,
+        "total_flops": 3.3649023858927206e+17
+    },
+    {
+        "step": 13600,
+        "train_loss": 3.047086179256439,
+        "val_loss": 3.1425138473510743,
+        "lr": 0.02,
+        "tokens_per_sec": 703774.056657341,
+        "mfu": 6.616164540389883,
+        "total_flops": 3.3773180835751526e+17
+    },
+    {
+        "step": 13650,
+        "train_loss": 3.163117825984955,
+        "val_loss": 3.153716003894806,
+        "lr": 0.02,
+        "tokens_per_sec": 562963.0033541002,
+        "mfu": 5.292402902763263,
+        "total_flops": 3.3897337812575846e+17
+    },
+    {
+        "step": 13700,
+        "train_loss": 3.0215771198272705,
+        "val_loss": 3.1455086708068847,
+        "lr": 0.02,
+        "tokens_per_sec": 726098.8486954446,
+        "mfu": 6.826039422899221,
+        "total_flops": 3.4021494789400166e+17
+    },
+    {
+        "step": 13750,
+        "train_loss": 3.0746082067489624,
+        "val_loss": 3.1506766438484193,
+        "lr": 0.02,
+        "tokens_per_sec": 387258.1708015886,
+        "mfu": 3.6406056082871583,
+        "total_flops": 3.4145651766224486e+17
+    },
+    {
+        "step": 13800,
+        "train_loss": 3.1453649401664734,
+        "val_loss": 3.1505769014358522,
+        "lr": 0.02,
+        "tokens_per_sec": 740524.9813837727,
+        "mfu": 6.961659181321131,
+        "total_flops": 3.4269808743048806e+17
+    },
+    {
+        "step": 13850,
+        "train_loss": 3.104449152946472,
+        "val_loss": 3.152713358402252,
+        "lr": 0.02,
+        "tokens_per_sec": 687207.8565221422,
+        "mfu": 6.460426054626306,
+        "total_flops": 3.4393965719873126e+17
+    },
+    {
+        "step": 13900,
+        "train_loss": 3.1265454292297363,
+        "val_loss": 3.1454100966453553,
+        "lr": 0.02,
+        "tokens_per_sec": 769994.2727430232,
+        "mfu": 7.238699345955137,
+        "total_flops": 3.4518122696697446e+17
+    },
+    {
+        "step": 13950,
+        "train_loss": 3.035269320011139,
+        "val_loss": 3.1418202042579653,
+        "lr": 0.02,
+        "tokens_per_sec": 677319.992196272,
+        "mfu": 6.367470457991035,
+        "total_flops": 3.4642279673521766e+17
+    },
+    {
+        "step": 14000,
+        "train_loss": 3.102819263935089,
+        "val_loss": 3.14750554561615,
+        "lr": 0.02,
+        "tokens_per_sec": 705157.8708164443,
+        "mfu": 6.629173747085332,
+        "total_flops": 3.4766436650346086e+17
+    },
+    {
+        "step": 14050,
+        "train_loss": 3.034712076187134,
+        "val_loss": 3.1495706915855406,
+        "lr": 0.02,
+        "tokens_per_sec": 736450.2024026074,
+        "mfu": 6.923352273088169,
+        "total_flops": 3.4890593627170406e+17
+    },
+    {
+        "step": 14100,
+        "train_loss": 3.10324889421463,
+        "val_loss": 3.149154031276703,
+        "lr": 0.02,
+        "tokens_per_sec": 680789.2004978532,
+        "mfu": 6.400084409487294,
+        "total_flops": 3.5014750603994726e+17
+    },
+    {
+        "step": 14150,
+        "train_loss": 3.1322348713874817,
+        "val_loss": 3.1461058974266054,
+        "lr": 0.02,
+        "tokens_per_sec": 749168.8854789592,
+        "mfu": 7.042920334988449,
+        "total_flops": 3.5138907580819046e+17
+    },
+    {
+        "step": 14200,
+        "train_loss": 3.097643196582794,
+        "val_loss": 3.151403570175171,
+        "lr": 0.02,
+        "tokens_per_sec": 681189.7392139593,
+        "mfu": 6.40384986521202,
+        "total_flops": 3.5263064557643366e+17
+    },
+    {
+        "step": 14250,
+        "train_loss": 3.064367115497589,
+        "val_loss": 3.147836744785309,
+        "lr": 0.02,
+        "tokens_per_sec": 756080.3892174037,
+        "mfu": 7.107895230727371,
+        "total_flops": 3.5387221534467686e+17
+    },
+    {
+        "step": 14300,
+        "train_loss": 3.1396385431289673,
+        "val_loss": 3.1502734065055846,
+        "lr": 0.02,
+        "tokens_per_sec": 724403.9458191768,
+        "mfu": 6.810105677966042,
+        "total_flops": 3.5511378511292006e+17
+    },
+    {
+        "step": 14350,
+        "train_loss": 3.0647523403167725,
+        "val_loss": 3.149309492111206,
+        "lr": 0.02,
+        "tokens_per_sec": 722727.958460705,
+        "mfu": 6.794349757402658,
+        "total_flops": 3.5635535488116326e+17
+    },
+    {
+        "step": 14400,
+        "train_loss": 3.0957531929016113,
+        "val_loss": 3.152403235435486,
+        "lr": 0.02,
+        "tokens_per_sec": 766725.8902000515,
+        "mfu": 7.207973353030722,
+        "total_flops": 3.5759692464940646e+17
+    },
+    {
+        "step": 14450,
+        "train_loss": 3.0434666872024536,
+        "val_loss": 3.145326030254364,
+        "lr": 0.02,
+        "tokens_per_sec": 722017.6919229714,
+        "mfu": 6.787672557189428,
+        "total_flops": 3.5883849441764966e+17
+    },
+    {
+        "step": 14500,
+        "train_loss": 3.0627703070640564,
+        "val_loss": 3.1456138014793398,
+        "lr": 0.02,
+        "tokens_per_sec": 738478.2262390595,
+        "mfu": 6.94241768089466,
+        "total_flops": 3.6008006418589286e+17
+    },
+    {
+        "step": 14550,
+        "train_loss": 3.0802829265594482,
+        "val_loss": 3.1429310202598573,
+        "lr": 0.02,
+        "tokens_per_sec": 702269.6436384912,
+        "mfu": 6.602021586447123,
+        "total_flops": 3.6132163395413606e+17
+    },
+    {
+        "step": 14600,
+        "train_loss": 3.1189873218536377,
+        "val_loss": 3.1505560874938965,
+        "lr": 0.02,
+        "tokens_per_sec": 696383.4380146917,
+        "mfu": 6.546685495897557,
+        "total_flops": 3.6256320372237926e+17
+    },
+    {
+        "step": 14650,
+        "train_loss": 3.066819429397583,
+        "val_loss": 3.143692100048065,
+        "lr": 0.02,
+        "tokens_per_sec": 732030.1378021343,
+        "mfu": 6.881799342287081,
+        "total_flops": 3.6380477349062246e+17
+    },
+    {
+        "step": 14700,
+        "train_loss": 3.0843369364738464,
+        "val_loss": 3.145070564746857,
+        "lr": 0.02,
+        "tokens_per_sec": 719142.5045767998,
+        "mfu": 6.760642983725084,
+        "total_flops": 3.6504634325886566e+17
+    },
+    {
+        "step": 14750,
+        "train_loss": 3.039639413356781,
+        "val_loss": 3.1410356044769285,
+        "lr": 0.02,
+        "tokens_per_sec": 766536.2919992043,
+        "mfu": 7.206190944484257,
+        "total_flops": 3.6628791302710886e+17
+    },
+    {
+        "step": 14800,
+        "train_loss": 3.0864169001579285,
+        "val_loss": 3.1432830691337585,
+        "lr": 0.02,
+        "tokens_per_sec": 765598.1552002148,
+        "mfu": 7.197371540920293,
+        "total_flops": 3.6752948279535206e+17
+    },
+    {
+        "step": 14850,
+        "train_loss": 3.0957528352737427,
+        "val_loss": 3.149162566661835,
+        "lr": 0.02,
+        "tokens_per_sec": 776568.60407019,
+        "mfu": 7.300504491217478,
+        "total_flops": 3.6877105256359526e+17
+    },
+    {
+        "step": 14900,
+        "train_loss": 3.0988165736198425,
+        "val_loss": 3.1451326608657837,
+        "lr": 0.02,
+        "tokens_per_sec": 681868.4795820845,
+        "mfu": 6.410230688593106,
+        "total_flops": 3.7001262233183846e+17
+    },
+    {
+        "step": 14950,
+        "train_loss": 3.0074263215065002,
+        "val_loss": 3.139087998867035,
+        "lr": 0.02,
+        "tokens_per_sec": 678105.8915915034,
+        "mfu": 6.374858681046231,
+        "total_flops": 3.7125419210008166e+17
+    },
+    {
+        "step": 15000,
+        "train_loss": 3.054157257080078,
+        "val_loss": 3.150736141204834,
+        "lr": 0.02,
+        "tokens_per_sec": 719883.2601829135,
+        "mfu": 6.767606811004456,
+        "total_flops": 3.7249576186832486e+17
+    },
+    {
+        "step": 15050,
+        "train_loss": 3.0841880440711975,
+        "val_loss": 3.144093596935272,
+        "lr": 0.02,
+        "tokens_per_sec": 761035.6944468117,
+        "mfu": 7.154479946994598,
+        "total_flops": 3.7373733163656806e+17
+    },
+    {
+        "step": 15100,
+        "train_loss": 3.1251383423805237,
+        "val_loss": 3.148873770236969,
+        "lr": 0.02,
+        "tokens_per_sec": 746738.0365748014,
+        "mfu": 7.020067977515766,
+        "total_flops": 3.7497890140481126e+17
+    },
+    {
+        "step": 15150,
+        "train_loss": 3.043576240539551,
+        "val_loss": 3.136397635936737,
+        "lr": 0.02,
+        "tokens_per_sec": 638864.8231680673,
+        "mfu": 6.005954253589394,
+        "total_flops": 3.7622047117305446e+17
+    },
+    {
+        "step": 15200,
+        "train_loss": 3.1045437455177307,
+        "val_loss": 3.1389421463012694,
+        "lr": 0.02,
+        "tokens_per_sec": 733851.3261809672,
+        "mfu": 6.898920294472552,
+        "total_flops": 3.7746204094129766e+17
+    },
+    {
+        "step": 15250,
+        "train_loss": 3.0337718725204468,
+        "val_loss": 3.1406340479850767,
+        "lr": 0.02,
+        "tokens_per_sec": 673028.6294144924,
+        "mfu": 6.327127450177403,
+        "total_flops": 3.7870361070954086e+17
+    },
+    {
+        "step": 15300,
+        "train_loss": 3.143120229244232,
+        "val_loss": 3.1418453574180605,
+        "lr": 0.02,
+        "tokens_per_sec": 691795.0422515095,
+        "mfu": 6.503550087511195,
+        "total_flops": 3.7994518047778406e+17
+    },
+    {
+        "step": 15350,
+        "train_loss": 3.133650004863739,
+        "val_loss": 3.143596124649048,
+        "lr": 0.02,
+        "tokens_per_sec": 687503.2952059626,
+        "mfu": 6.463203467242276,
+        "total_flops": 3.8118675024602726e+17
+    },
+    {
+        "step": 15400,
+        "train_loss": 3.029381215572357,
+        "val_loss": 3.1452716112136843,
+        "lr": 0.02,
+        "tokens_per_sec": 641473.306409728,
+        "mfu": 6.030476547590464,
+        "total_flops": 3.8242832001427046e+17
+    },
+    {
+        "step": 15450,
+        "train_loss": 3.0394997596740723,
+        "val_loss": 3.139203870296478,
+        "lr": 0.02,
+        "tokens_per_sec": 701028.9046245916,
+        "mfu": 6.590357426067825,
+        "total_flops": 3.8366988978251366e+17
+    },
+    {
+        "step": 15500,
+        "train_loss": 3.0841792821884155,
+        "val_loss": 3.141300177574158,
+        "lr": 0.02,
+        "tokens_per_sec": 690827.9719891226,
+        "mfu": 6.494458681089567,
+        "total_flops": 3.8491145955075686e+17
+    },
+    {
+        "step": 15550,
+        "train_loss": 3.0710747838020325,
+        "val_loss": 3.1462770342826842,
+        "lr": 0.02,
+        "tokens_per_sec": 692112.1702584228,
+        "mfu": 6.506531400979995,
+        "total_flops": 3.8615302931900006e+17
+    },
+    {
+        "step": 15600,
+        "train_loss": 3.0950469970703125,
+        "val_loss": 3.138612949848175,
+        "lr": 0.02,
+        "tokens_per_sec": 770514.9559267653,
+        "mfu": 7.243594277196867,
+        "total_flops": 3.8739459908724326e+17
+    },
+    {
+        "step": 15650,
+        "train_loss": 3.068549394607544,
+        "val_loss": 3.1408026933670046,
+        "lr": 0.02,
+        "tokens_per_sec": 753367.3281070498,
+        "mfu": 7.082389802466077,
+        "total_flops": 3.8863616885548646e+17
+    },
+    {
+        "step": 15700,
+        "train_loss": 3.131138026714325,
+        "val_loss": 3.1432745099067687,
+        "lr": 0.02,
+        "tokens_per_sec": 768443.0158710635,
+        "mfu": 7.224116013972079,
+        "total_flops": 3.8987773862372966e+17
+    },
+    {
+        "step": 15750,
+        "train_loss": 3.085524320602417,
+        "val_loss": 3.143316113948822,
+        "lr": 0.02,
+        "tokens_per_sec": 724972.6384412758,
+        "mfu": 6.815451944889525,
+        "total_flops": 3.9111930839197286e+17
+    },
+    {
+        "step": 15800,
+        "train_loss": 3.0742884278297424,
+        "val_loss": 3.1385276079177857,
+        "lr": 0.02,
+        "tokens_per_sec": 696762.4148946902,
+        "mfu": 6.550248249271819,
+        "total_flops": 3.9236087816021606e+17
+    },
+    {
+        "step": 15850,
+        "train_loss": 3.0977269411087036,
+        "val_loss": 3.140676200389862,
+        "lr": 0.02,
+        "tokens_per_sec": 687534.4738464371,
+        "mfu": 6.46349657696062,
+        "total_flops": 3.9360244792845926e+17
+    },
+    {
+        "step": 15900,
+        "train_loss": 3.0782747864723206,
+        "val_loss": 3.140700113773346,
+        "lr": 0.02,
+        "tokens_per_sec": 715785.0217764467,
+        "mfu": 6.72907936122645,
+        "total_flops": 3.9484401769670246e+17
+    },
+    {
+        "step": 15950,
+        "train_loss": 3.100230097770691,
+        "val_loss": 3.134903037548065,
+        "lr": 0.02,
+        "tokens_per_sec": 705794.6056878131,
+        "mfu": 6.635159677708562,
+        "total_flops": 3.9608558746494566e+17
+    },
+    {
+        "step": 16000,
+        "train_loss": 3.068934679031372,
+        "val_loss": 3.1448294639587404,
+        "lr": 0.02,
+        "tokens_per_sec": 734678.9750939112,
+        "mfu": 6.906701003832202,
+        "total_flops": 3.9732715723318886e+17
+    },
+    {
+        "step": 16050,
+        "train_loss": 3.0025221705436707,
+        "val_loss": 3.1464810132980348,
+        "lr": 0.01943255903154212,
+        "tokens_per_sec": 699908.4735491821,
+        "mfu": 6.5798242779914595,
+        "total_flops": 3.9856872700143206e+17
+    },
+    {
+        "step": 16100,
+        "train_loss": 3.070408046245575,
+        "val_loss": 3.14631609916687,
+        "lr": 0.018881217525718468,
+        "tokens_per_sec": 643071.6331094927,
+        "mfu": 6.0455023819969895,
+        "total_flops": 3.9981029676967526e+17
+    },
+    {
+        "step": 16150,
+        "train_loss": 3.0675635933876038,
+        "val_loss": 3.1374461889266967,
+        "lr": 0.01834551870779559,
+        "tokens_per_sec": 617864.6048335439,
+        "mfu": 5.80853165954659,
+        "total_flops": 4.0105186653791846e+17
+    },
+    {
+        "step": 16200,
+        "train_loss": 3.045469880104065,
+        "val_loss": 3.1379233717918398,
+        "lr": 0.01782501876267491,
+        "tokens_per_sec": 747194.9668446358,
+        "mfu": 7.0243635690058355,
+        "total_flops": 4.0229343630616166e+17
+    },
+    {
+        "step": 16250,
+        "train_loss": 3.113093376159668,
+        "val_loss": 3.1413995742797853,
+        "lr": 0.01731928646720131,
+        "tokens_per_sec": 718972.9093617701,
+        "mfu": 6.759048622811544,
+        "total_flops": 4.0353500607440486e+17
+    },
+    {
+        "step": 16300,
+        "train_loss": 3.0531409978866577,
+        "val_loss": 3.1344855070114135,
+        "lr": 0.016827902832903904,
+        "tokens_per_sec": 564062.8449018005,
+        "mfu": 5.302742489139186,
+        "total_flops": 4.0477657584264806e+17
+    },
+    {
+        "step": 16350,
+        "train_loss": 3.0165595412254333,
+        "val_loss": 3.1350361824035646,
+        "lr": 0.016350460758873,
+        "tokens_per_sec": 708849.9905872872,
+        "mfu": 6.663883284437057,
+        "total_flops": 4.0601814561089126e+17
+    },
+    {
+        "step": 16400,
+        "train_loss": 3.020295739173889,
+        "val_loss": 3.1313610792160036,
+        "lr": 0.01588656469448563,
+        "tokens_per_sec": 717004.3346030491,
+        "mfu": 6.740542094486789,
+        "total_flops": 4.0725971537913446e+17
+    },
+    {
+        "step": 16450,
+        "train_loss": 3.117129921913147,
+        "val_loss": 3.1293462038040163,
+        "lr": 0.01543583031170025,
+        "tokens_per_sec": 519429.62306803424,
+        "mfu": 4.883146545204445,
+        "total_flops": 4.0850128514737766e+17
+    },
+    {
+        "step": 16500,
+        "train_loss": 3.0911515951156616,
+        "val_loss": 3.130696713924408,
+        "lr": 0.014997884186649118,
+        "tokens_per_sec": 712461.4443617964,
+        "mfu": 6.697834482518515,
+        "total_flops": 4.0974285491562086e+17
+    },
+    {
+        "step": 16550,
+        "train_loss": 3.041957437992096,
+        "val_loss": 3.1251978516578673,
+        "lr": 0.014572363490264556,
+        "tokens_per_sec": 733594.2115164933,
+        "mfu": 6.896503165125691,
+        "total_flops": 4.1098442468386406e+17
+    },
+    {
+        "step": 16600,
+        "train_loss": 3.0678828358650208,
+        "val_loss": 3.1350746273994448,
+        "lr": 0.014158915687682759,
+        "tokens_per_sec": 579977.1700169533,
+        "mfu": 5.452352712072391,
+        "total_flops": 4.1222599445210726e+17
+    },
+    {
+        "step": 16650,
+        "train_loss": 3.059101641178131,
+        "val_loss": 3.1194016695022584,
+        "lr": 0.013757198246176152,
+        "tokens_per_sec": 723278.225219649,
+        "mfu": 6.799522803188933,
+        "total_flops": 4.1346756422035046e+17
+    },
+    {
+        "step": 16700,
+        "train_loss": 3.0314047932624817,
+        "val_loss": 3.119983184337616,
+        "lr": 0.013366878351372293,
+        "tokens_per_sec": 706245.4855250673,
+        "mfu": 6.63939838921405,
+        "total_flops": 4.1470913398859366e+17
+    },
+    {
+        "step": 16750,
+        "train_loss": 3.0364831686019897,
+        "val_loss": 3.1191796541213987,
+        "lr": 0.012987632631524226,
+        "tokens_per_sec": 716420.5737692728,
+        "mfu": 6.735054171634326,
+        "total_flops": 4.1595070375683686e+17
+    },
+    {
+        "step": 16800,
+        "train_loss": 3.0998443961143494,
+        "val_loss": 3.1202172160148622,
+        "lr": 0.012619146889603866,
+        "tokens_per_sec": 714708.8932903456,
+        "mfu": 6.718962700824868,
+        "total_flops": 4.1719227352508006e+17
+    },
+    {
+        "step": 16850,
+        "train_loss": 3.0862051844596863,
+        "val_loss": 3.113366889953613,
+        "lr": 0.012261115842996415,
+        "tokens_per_sec": 692450.4445954027,
+        "mfu": 6.50971151063604,
+        "total_flops": 4.1843384329332326e+17
+    },
+    {
+        "step": 16900,
+        "train_loss": 3.0278157591819763,
+        "val_loss": 3.10579172372818,
+        "lr": 0.01191324287058021,
+        "tokens_per_sec": 729828.1945023012,
+        "mfu": 6.861098921402715,
+        "total_flops": 4.1967541306156646e+17
+    },
+    {
+        "step": 16950,
+        "train_loss": 3.1150811314582825,
+        "val_loss": 3.1045249462127686,
+        "lr": 0.011575239766982413,
+        "tokens_per_sec": 759203.8006162018,
+        "mfu": 7.137258353090717,
+        "total_flops": 4.2091698282980966e+17
+    },
+    {
+        "step": 17000,
+        "train_loss": 3.0266743302345276,
+        "val_loss": 3.1035353422164915,
+        "lr": 0.011246826503806982,
+        "tokens_per_sec": 756266.9043518367,
+        "mfu": 7.1096486554337925,
+        "total_flops": 4.2215855259805286e+17
+    },
+    {
+        "step": 17050,
+        "train_loss": 3.0746100544929504,
+        "val_loss": 3.1017999410629273,
+        "lr": 0.010927730997637084,
+        "tokens_per_sec": 694065.5937923228,
+        "mfu": 6.5248954929710195,
+        "total_flops": 4.2340012236629606e+17
+    },
+    {
+        "step": 17100,
+        "train_loss": 3.0298961400985718,
+        "val_loss": 3.101312255859375,
+        "lr": 0.010617688884619766,
+        "tokens_per_sec": 720785.0277298539,
+        "mfu": 6.776084308023996,
+        "total_flops": 4.2464169213453926e+17
+    },
+    {
+        "step": 17150,
+        "train_loss": 3.0325640439987183,
+        "val_loss": 3.101479244232178,
+        "lr": 0.010316443301446114,
+        "tokens_per_sec": 729438.6692286865,
+        "mfu": 6.857437002810383,
+        "total_flops": 4.2588326190278246e+17
+    },
+    {
+        "step": 17200,
+        "train_loss": 3.0670490860939026,
+        "val_loss": 3.1049660444259644,
+        "lr": 0.010023744672545448,
+        "tokens_per_sec": 740081.2137324706,
+        "mfu": 6.957487331320491,
+        "total_flops": 4.2712483167102566e+17
+    },
+    {
+        "step": 17250,
+        "train_loss": 2.9903318881988525,
+        "val_loss": 3.1029152631759644,
+        "lr": 0.009739350503317262,
+        "tokens_per_sec": 671767.1005807547,
+        "mfu": 6.3152678451557325,
+        "total_flops": 4.2836640143926886e+17
+    },
+    {
+        "step": 17300,
+        "train_loss": 3.0684115886688232,
+        "val_loss": 3.1053053736686707,
+        "lr": 0.00946302517922961,
+        "tokens_per_sec": 734440.326670526,
+        "mfu": 6.904457475214622,
+        "total_flops": 4.2960797120751206e+17
+    },
+    {
+        "step": 17350,
+        "train_loss": 3.0212934017181396,
+        "val_loss": 3.1042049407958983,
+        "lr": 0.009194539770617445,
+        "tokens_per_sec": 725588.0457060201,
+        "mfu": 6.821237375148546,
+        "total_flops": 4.3084954097575526e+17
+    },
+    {
+        "step": 17400,
+        "train_loss": 3.019513189792633,
+        "val_loss": 3.105720841884613,
+        "lr": 0.008933671843019263,
+        "tokens_per_sec": 712422.1938231154,
+        "mfu": 6.697465489061375,
+        "total_flops": 4.3209111074399846e+17
+    },
+    {
+        "step": 17450,
+        "train_loss": 3.0266613960266113,
+        "val_loss": 3.10250746011734,
+        "lr": 0.008680205272894878,
+        "tokens_per_sec": 695055.7171985689,
+        "mfu": 6.534203621494723,
+        "total_flops": 4.3333268051224166e+17
+    },
+    {
+        "step": 17500,
+        "train_loss": 3.087554156780243,
+        "val_loss": 3.101519727706909,
+        "lr": 0.008433930068571646,
+        "tokens_per_sec": 713302.2077206185,
+        "mfu": 6.705738480497522,
+        "total_flops": 4.3457425028048486e+17
+    },
+    {
+        "step": 17550,
+        "train_loss": 2.9932942390441895,
+        "val_loss": 3.0988452196121217,
+        "lr": 0.008194642196270831,
+        "tokens_per_sec": 782921.0291510923,
+        "mfu": 7.360223500703804,
+        "total_flops": 4.3581582004872806e+17
+    },
+    {
+        "step": 17600,
+        "train_loss": 3.0073635578155518,
+        "val_loss": 3.09357488155365,
+        "lr": 0.007962143411069945,
+        "tokens_per_sec": 736169.624928723,
+        "mfu": 6.920714570382327,
+        "total_flops": 4.3705738981697126e+17
+    },
+    {
+        "step": 17650,
+        "train_loss": 3.1045629382133484,
+        "val_loss": 3.0909008145332337,
+        "lr": 0.007736241092661044,
+        "tokens_per_sec": 679335.1463116886,
+        "mfu": 6.3864148778310685,
+        "total_flops": 4.3829895958521446e+17
+    },
+    {
+        "step": 17700,
+        "train_loss": 3.031808853149414,
+        "val_loss": 3.0918583154678343,
+        "lr": 0.007516748085768884,
+        "tokens_per_sec": 387732.62828782527,
+        "mfu": 3.6450659727559356,
+        "total_flops": 4.3954052935345766e+17
+    },
+    {
+        "step": 17750,
+        "train_loss": 3.082665503025055,
+        "val_loss": 3.0935746788978578,
+        "lr": 0.007303482545096754,
+        "tokens_per_sec": 728887.4120509602,
+        "mfu": 6.852254646118762,
+        "total_flops": 4.4078209912170086e+17
+    },
+    {
+        "step": 17800,
+        "train_loss": 3.0321786403656006,
+        "val_loss": 3.0965789198875426,
+        "lr": 0.00709626778467151,
+        "tokens_per_sec": 691484.3255468745,
+        "mfu": 6.50062904655513,
+        "total_flops": 4.4202366888994406e+17
+    },
+    {
+        "step": 17850,
+        "train_loss": 3.062150776386261,
+        "val_loss": 3.0917367458343508,
+        "lr": 0.0068949321314629875,
+        "tokens_per_sec": 699998.4053803874,
+        "mfu": 6.580669725173044,
+        "total_flops": 4.4326523865818726e+17
+    },
+    {
+        "step": 17900,
+        "train_loss": 2.999358355998993,
+        "val_loss": 3.0902578592300416,
+        "lr": 0.006699308783156554,
+        "tokens_per_sec": 694246.9714867986,
+        "mfu": 6.526600620716574,
+        "total_flops": 4.4450680842643046e+17
+    },
+    {
+        "step": 17950,
+        "train_loss": 3.023828446865082,
+        "val_loss": 3.0946322441101075,
+        "lr": 0.006509235669960918,
+        "tokens_per_sec": 715732.6380369696,
+        "mfu": 6.728586902835374,
+        "total_flops": 4.4574837819467366e+17
+    },
+    {
+        "step": 18000,
+        "train_loss": 3.040572166442871,
+        "val_loss": 3.089223766326904,
+        "lr": 0.006324555320336759,
+        "tokens_per_sec": 665364.2394382448,
+        "mfu": 6.255074687355458,
+        "total_flops": 4.4698994796291686e+17
+    },
+    {
+        "step": 18050,
+        "train_loss": 3.0870242714881897,
+        "val_loss": 3.090847408771515,
+        "lr": 0.006145114730534894,
+        "tokens_per_sec": 766372.5203509134,
+        "mfu": 7.204651330794475,
+        "total_flops": 4.4823151773116006e+17
+    },
+    {
+        "step": 18100,
+        "train_loss": 3.063161551952362,
+        "val_loss": 3.091911458969116,
+        "lr": 0.005970765237835919,
+        "tokens_per_sec": 688490.1831549242,
+        "mfu": 6.472481179302693,
+        "total_flops": 4.4947308749940326e+17
+    },
+    {
+        "step": 18150,
+        "train_loss": 3.0751683115959167,
+        "val_loss": 3.0886308670043947,
+        "lr": 0.005801362397386308,
+        "tokens_per_sec": 734212.2270090751,
+        "mfu": 6.902313115277664,
+        "total_flops": 4.5071465726764646e+17
+    },
+    {
+        "step": 18200,
+        "train_loss": 3.027254581451416,
+        "val_loss": 3.089142549037933,
+        "lr": 0.005636765862528908,
+        "tokens_per_sec": 688938.4922131193,
+        "mfu": 6.476695722970377,
+        "total_flops": 4.5195622703588966e+17
+    },
+    {
+        "step": 18250,
+        "train_loss": 3.0408639311790466,
+        "val_loss": 3.090257740020752,
+        "lr": 0.0054768392685287225,
+        "tokens_per_sec": 692402.7830808776,
+        "mfu": 6.509263445778634,
+        "total_flops": 4.5319779680413286e+17
+    },
+    {
+        "step": 18300,
+        "train_loss": 3.0395489931106567,
+        "val_loss": 3.0902520656585692,
+        "lr": 0.005321450119597621,
+        "tokens_per_sec": 719781.5263763262,
+        "mfu": 6.766650413709992,
+        "total_flops": 4.5443936657237606e+17
+    },
+    {
+        "step": 18350,
+        "train_loss": 3.0594890117645264,
+        "val_loss": 3.0901105999946594,
+        "lr": 0.005170469679124381,
+        "tokens_per_sec": 701543.6814319686,
+        "mfu": 6.595196831594308,
+        "total_flops": 4.5568093634061926e+17
+    },
+    {
+        "step": 18400,
+        "train_loss": 3.026129126548767,
+        "val_loss": 3.0864153623580934,
+        "lr": 0.005023772863019161,
+        "tokens_per_sec": 685278.1300899048,
+        "mfu": 6.442284738570608,
+        "total_flops": 4.5692250610886246e+17
+    },
+    {
+        "step": 18450,
+        "train_loss": 2.992745041847229,
+        "val_loss": 3.0890154004096986,
+        "lr": 0.0048812381360839605,
+        "tokens_per_sec": 728753.9542899799,
+        "mfu": 6.851000012621711,
+        "total_flops": 4.5816407587710566e+17
+    },
+    {
+        "step": 18500,
+        "train_loss": 3.0637913942337036,
+        "val_loss": 3.0905563712120054,
+        "lr": 0.004742747411323311,
+        "tokens_per_sec": 721667.1139879038,
+        "mfu": 6.784376781676397,
+        "total_flops": 4.5940564564534886e+17
+    },
+    {
+        "step": 18550,
+        "train_loss": 2.9693461060523987,
+        "val_loss": 3.09128623008728,
+        "lr": 0.004608185952111692,
+        "tokens_per_sec": 662050.8319155766,
+        "mfu": 6.223925415579997,
+        "total_flops": 4.6064721541359206e+17
+    },
+    {
+        "step": 18600,
+        "train_loss": 2.983288586139679,
+        "val_loss": 3.0884334087371825,
+        "lr": 0.004477442277136679,
+        "tokens_per_sec": 720536.761625188,
+        "mfu": 6.773750363794689,
+        "total_flops": 4.6188878518183526e+17
+    },
+    {
+        "step": 18650,
+        "train_loss": 2.9768336415290833,
+        "val_loss": 3.092844784259796,
+        "lr": 0.004350408068039046,
+        "tokens_per_sec": 729005.233665835,
+        "mfu": 6.853362284547675,
+        "total_flops": 4.6313035495007846e+17
+    },
+    {
+        "step": 18700,
+        "train_loss": 3.0071200132369995,
+        "val_loss": 3.0864765763282778,
+        "lr": 0.004226978079673293,
+        "tokens_per_sec": 719243.7284677255,
+        "mfu": 6.761594587313548,
+        "total_flops": 4.6437192471832166e+17
+    },
+    {
+        "step": 18750,
+        "train_loss": 2.9823778867721558,
+        "val_loss": 3.086198890209198,
+        "lr": 0.004107050052914292,
+        "tokens_per_sec": 715917.9550265683,
+        "mfu": 6.7303290638083695,
+        "total_flops": 4.6561349448656486e+17
+    },
+    {
+        "step": 18800,
+        "train_loss": 3.015293300151825,
+        "val_loss": 3.0859832167625427,
+        "lr": 0.0039905246299377604,
+        "tokens_per_sec": 700826.0866038192,
+        "mfu": 6.588450738282884,
+        "total_flops": 4.6685506425480806e+17
+    },
+    {
+        "step": 18850,
+        "train_loss": 3.012860596179962,
+        "val_loss": 3.0853739976882935,
+        "lr": 0.0038773052719044145,
+        "tokens_per_sec": 758256.9238838439,
+        "mfu": 7.128356785603984,
+        "total_flops": 4.6809663402305126e+17
+    },
+    {
+        "step": 18900,
+        "train_loss": 3.057746469974518,
+        "val_loss": 3.0853593587875365,
+        "lr": 0.003767298178979601,
+        "tokens_per_sec": 702816.8260277645,
+        "mfu": 6.607165636141467,
+        "total_flops": 4.6933820379129446e+17
+    },
+    {
+        "step": 18950,
+        "train_loss": 2.973491668701172,
+        "val_loss": 3.0860115766525267,
+        "lr": 0.003660412212622112,
+        "tokens_per_sec": 717484.8124776696,
+        "mfu": 6.745059056495312,
+        "total_flops": 4.7057977355953766e+17
+    },
+    {
+        "step": 19000,
+        "train_loss": 3.0762404203414917,
+        "val_loss": 3.0846158027648927,
+        "lr": 0.003556558820077846,
+        "tokens_per_sec": 684364.6308949208,
+        "mfu": 6.4336969525839605,
+        "total_flops": 4.7182134332778086e+17
+    },
+    {
+        "step": 19050,
+        "train_loss": 2.918803095817566,
+        "val_loss": 3.086686110496521,
+        "lr": 0.003455651961015727,
+        "tokens_per_sec": 408454.8850148041,
+        "mfu": 3.8398754557952426,
+        "total_flops": 4.7306291309602406e+17
+    },
+    {
+        "step": 19100,
+        "train_loss": 2.989955246448517,
+        "val_loss": 3.0845702648162843,
+        "lr": 0.0033576080362451204,
+        "tokens_per_sec": 685496.9734419179,
+        "mfu": 6.444342080144644,
+        "total_flops": 4.7430448286426726e+17
+    },
+    {
+        "step": 19150,
+        "train_loss": 3.026902973651886,
+        "val_loss": 3.083245599269867,
+        "lr": 0.003262345818455677,
+        "tokens_per_sec": 696411.1376441265,
+        "mfu": 6.546945899509078,
+        "total_flops": 4.7554605263251046e+17
+    },
+    {
+        "step": 19200,
+        "train_loss": 2.9645996689796448,
+        "val_loss": 3.0823713302612306,
+        "lr": 0.003169786384922227,
+        "tokens_per_sec": 708031.469734792,
+        "mfu": 6.6561883877743835,
+        "total_flops": 4.7678762240075366e+17
+    },
+    {
+        "step": 19250,
+        "train_loss": 3.022516131401062,
+        "val_loss": 3.0823891878128054,
+        "lr": 0.0030798530521189845,
+        "tokens_per_sec": 704160.6039027537,
+        "mfu": 6.619798462604113,
+        "total_flops": 4.7802919216899686e+17
+    },
+    {
+        "step": 19300,
+        "train_loss": 3.0703765749931335,
+        "val_loss": 3.0858085632324217,
+        "lr": 0.0029924713121888677,
+        "tokens_per_sec": 634319.5635431412,
+        "mfu": 5.963224367096942,
+        "total_flops": 4.7927076193724006e+17
+    },
+    {
+        "step": 19350,
+        "train_loss": 3.0681645274162292,
+        "val_loss": 3.0809340119361877,
+        "lr": 0.0029075687712153235,
+        "tokens_per_sec": 740793.2418918696,
+        "mfu": 6.964181092500533,
+        "total_flops": 4.8051233170548326e+17
+    },
+    {
+        "step": 19400,
+        "train_loss": 3.0543947219848633,
+        "val_loss": 3.0824201107025146,
+        "lr": 0.0028250750892455092,
+        "tokens_per_sec": 747138.0373387374,
+        "mfu": 7.023828375963887,
+        "total_flops": 4.8175390147372646e+17
+    },
+    {
+        "step": 19450,
+        "train_loss": 3.022664248943329,
+        "val_loss": 3.0830381870269776,
+        "lr": 0.002744921922015124,
+        "tokens_per_sec": 763554.831876657,
+        "mfu": 7.178162302969573,
+        "total_flops": 4.8299547124196966e+17
+    },
+    {
+        "step": 19500,
+        "train_loss": 3.0661675333976746,
+        "val_loss": 3.0806783080101012,
+        "lr": 0.002667042864326648,
+        "tokens_per_sec": 738727.4103861994,
+        "mfu": 6.94476025562122,
+        "total_flops": 4.8423704101021286e+17
+    },
+    {
+        "step": 19550,
+        "train_loss": 3.0482770204544067,
+        "val_loss": 3.0791345119476317,
+        "lr": 0.0025913733950340394,
+        "tokens_per_sec": 706756.9039727594,
+        "mfu": 6.644206222874462,
+        "total_flops": 4.8547861077845606e+17
+    },
+    {
+        "step": 19600,
+        "train_loss": 3.0468355417251587,
+        "val_loss": 3.079674780368805,
+        "lr": 0.0025178508235883345,
+        "tokens_per_sec": 786332.9609115301,
+        "mfu": 7.392298996687352,
+        "total_flops": 4.8672018054669926e+17
+    },
+    {
+        "step": 19650,
+        "train_loss": 2.9020655751228333,
+        "val_loss": 3.079472291469574,
+        "lr": 0.0024464142380998632,
+        "tokens_per_sec": 707581.8591161177,
+        "mfu": 6.6519616081650526,
+        "total_flops": 4.8796175031494246e+17
+    },
+    {
+        "step": 19700,
+        "train_loss": 3.0088971853256226,
+        "val_loss": 3.076182794570923,
+        "lr": 0.0023770044548740367,
+        "tokens_per_sec": 663726.0970363328,
+        "mfu": 6.2396745464023855,
+        "total_flops": 4.8920332008318566e+17
+    },
+    {
+        "step": 19750,
+        "train_loss": 3.035418212413788,
+        "val_loss": 3.0742117404937743,
+        "lr": 0.0023095639693789167,
+        "tokens_per_sec": 749332.4079574648,
+        "mfu": 7.04445760623852,
+        "total_flops": 4.9044488985142886e+17
+    },
+    {
+        "step": 19800,
+        "train_loss": 3.022461414337158,
+        "val_loss": 3.075487720966339,
+        "lr": 0.0022440369086039274,
+        "tokens_per_sec": 753592.443440084,
+        "mfu": 7.084506106796748,
+        "total_flops": 4.9168645961967206e+17
+    },
+    {
+        "step": 19850,
+        "train_loss": 3.0326911211013794,
+        "val_loss": 3.0735740065574646,
+        "lr": 0.0021803689847702556,
+        "tokens_per_sec": 610836.9087012644,
+        "mfu": 5.742464441650178,
+        "total_flops": 4.9292802938791526e+17
+    },
+    {
+        "step": 19900,
+        "train_loss": 2.9602994322776794,
+        "val_loss": 3.07362140417099,
+        "lr": 0.0021185074503545782,
+        "tokens_per_sec": 707356.9151921803,
+        "mfu": 6.649846915247556,
+        "total_flops": 4.9416959915615846e+17
+    },
+    {
+        "step": 19950,
+        "train_loss": 2.967044711112976,
+        "val_loss": 3.0712722659111025,
+        "lr": 0.0020584010543888563,
+        "tokens_per_sec": 721988.8068136637,
+        "mfu": 6.787401008907505,
+        "total_flops": 4.9541116892440166e+17
+    }
+]
\ No newline at end of file