step 18800

checkpoints/step_18800/metadata.json

@@ -0,0 +1,3143 @@
+{
+  "step": 18800,
+  "tokens_consumed": 9856614400,
+  "sequences_consumed": 4812800,
+  "data_state": {
+    "stream_state": {
+      "rng_state": [
+        "MT19937",
+        [
+          2976965755,
+          160918995,
+          3496288601,
+          683057393,
+          98615017,
+          2294044812,
+          4288459450,
+          1888055787,
+          1835270608,
+          4228592700,
+          1379011328,
+          2823835502,
+          3454996493,
+          2814847853,
+          4231272384,
+          2143850863,
+          3250157159,
+          4003190325,
+          1981133118,
+          2058027955,
+          1434552940,
+          1027330435,
+          4033443357,
+          4230381641,
+          1303574656,
+          2996560263,
+          2271349905,
+          2821481654,
+          331496648,
+          2237540016,
+          59616114,
+          2707001790,
+          3051425575,
+          834537866,
+          1655968509,
+          3851005356,
+          1100642436,
+          2550834166,
+          2448207816,
+          2435585717,
+          3475925536,
+          3039476644,
+          3228982982,
+          972714228,
+          2139524242,
+          53989883,
+          547211637,
+          1435472174,
+          911408010,
+          1232312292,
+          1231786084,
+          3436810153,
+          4110877395,
+          1895700769,
+          2123527281,
+          3201217149,
+          3131955054,
+          2429648645,
+          2762668308,
+          2538957489,
+          2630866466,
+          292476535,
+          510705195,
+          2901821125,
+          3430909014,
+          4066240866,
+          931729172,
+          3693132426,
+          1748017451,
+          3344100819,
+          2219495564,
+          3512803265,
+          3685489527,
+          3120924161,
+          2056529160,
+          1514782935,
+          3414245287,
+          376546829,
+          3346898374,
+          1056835573,
+          379852822,
+          1864783080,
+          970497680,
+          2926292904,
+          2230965553,
+          4178756951,
+          3744682058,
+          3455256893,
+          3503893044,
+          1022200053,
+          4272409295,
+          3539111098,
+          367424546,
+          826390730,
+          4173945158,
+          1024932676,
+          1434917002,
+          4089556480,
+          880690144,
+          2533853482,
+          2110306272,
+          1198071246,
+          582564355,
+          4016293613,
+          1928016032,
+          1548477123,
+          3715565107,
+          4178715864,
+          899954533,
+          1044535005,
+          1506565394,
+          2139642294,
+          1774142565,
+          1673181131,
+          1006203316,
+          3618224614,
+          2346302567,
+          2697950981,
+          1999963573,
+          3611212296,
+          3083854036,
+          3544576937,
+          108919857,
+          2193616199,
+          638230217,
+          4035200925,
+          1673442315,
+          2599728932,
+          1914175300,
+          549896719,
+          4068003207,
+          765280665,
+          4100284624,
+          2837205053,
+          2686590004,
+          2961942161,
+          516978845,
+          2407665570,
+          1018898486,
+          1564794769,
+          2967322605,
+          2853829389,
+          2547108619,
+          2763276579,
+          667004428,
+          3298050090,
+          2259374231,
+          2740748463,
+          347248850,
+          1795922786,
+          1843366315,
+          3319840210,
+          2127762995,
+          598362603,
+          3485881137,
+          137738567,
+          3245146605,
+          2498600231,
+          3739246486,
+          2840768582,
+          1164642920,
+          3040218595,
+          2436890646,
+          1805895149,
+          2903480146,
+          909924889,
+          1489374398,
+          2769386946,
+          306103360,
+          3841331391,
+          1439047070,
+          2896889133,
+          4106191001,
+          2730656974,
+          4131731976,
+          573133908,
+          4052116916,
+          1095566601,
+          3678088503,
+          2790099602,
+          2474102191,
+          2879178813,
+          3212000588,
+          2494858499,
+          298266702,
+          182957861,
+          4228065866,
+          2446210865,
+          460531016,
+          2089358891,
+          808088897,
+          2075120199,
+          2441491428,
+          1993881596,
+          2698986868,
+          2569293183,
+          4082443718,
+          4180390948,
+          2517534507,
+          3784684373,
+          4294424290,
+          4150900019,
+          3242617258,
+          2436897158,
+          1029995275,
+          2549099659,
+          3851344061,
+          1195898658,
+          1421511989,
+          2327754743,
+          464289692,
+          3167749636,
+          1733337320,
+          2872543240,
+          229185327,
+          2005898764,
+          3721530610,
+          557071211,
+          3756663703,
+          1280918132,
+          2275642377,
+          961575160,
+          2744158274,
+          286147525,
+          3157342871,
+          1802278172,
+          1749879855,
+          1830909549,
+          2345637998,
+          2193940710,
+          2921661232,
+          2600527719,
+          3080659191,
+          203137951,
+          3035575856,
+          2688776664,
+          447410011,
+          1408827930,
+          4158627622,
+          3179795203,
+          1084237616,
+          4151708564,
+          3883134588,
+          1471572725,
+          834054760,
+          4216256691,
+          1277613636,
+          884010808,
+          2230542452,
+          1582920724,
+          157086651,
+          2272628211,
+          2275110759,
+          3416996172,
+          336050467,
+          3946832383,
+          2660675040,
+          2999700524,
+          1817985684,
+          269946174,
+          1030996180,
+          1112106850,
+          490241273,
+          2101975466,
+          3040055116,
+          1815292318,
+          2409838949,
+          3511097281,
+          1519618150,
+          1309976577,
+          2466620544,
+          614103771,
+          2758186895,
+          2342180208,
+          15518774,
+          788111258,
+          2704713169,
+          2454698615,
+          2099953576,
+          1410402477,
+          3005662432,
+          1890494321,
+          2977708728,
+          4119000734,
+          2917335066,
+          593200399,
+          61658623,
+          3718894914,
+          1475491122,
+          848177358,
+          264814075,
+          520622439,
+          1623486363,
+          2575871393,
+          829495173,
+          2472743096,
+          649633223,
+          3905011976,
+          117144358,
+          2447564825,
+          1624402552,
+          2397653487,
+          3993001938,
+          3366626970,
+          2283817753,
+          1545816891,
+          3904355158,
+          749861592,
+          3355981459,
+          1999696000,
+          1784187456,
+          53964824,
+          54216812,
+          1135609682,
+          1618082223,
+          3278663369,
+          2340257985,
+          334606808,
+          227634724,
+          1399205135,
+          3115282167,
+          4124731432,
+          1510495927,
+          526820528,
+          6236475,
+          3433598929,
+          586581510,
+          907342968,
+          1372036936,
+          891215198,
+          312273447,
+          1094099823,
+          3794684653,
+          2317875273,
+          2340289224,
+          869466785,
+          1808322779,
+          179423970,
+          2586647931,
+          496443987,
+          3759200546,
+          3608512017,
+          627163819,
+          4203921956,
+          3079977776,
+          798694057,
+          1733650080,
+          2815519093,
+          3848828487,
+          4197328378,
+          3633435584,
+          1114981087,
+          2488547964,
+          2489114879,
+          4274377559,
+          3352754835,
+          1382205571,
+          2612357647,
+          247357132,
+          3356898900,
+          3706414496,
+          1793862832,
+          2153804609,
+          1406504614,
+          2076823670,
+          2067880244,
+          642768111,
+          1037932593,
+          1893919033,
+          326034971,
+          1162028669,
+          3931950506,
+          589234045,
+          3486461256,
+          3938806987,
+          4043330152,
+          1537418689,
+          2401465035,
+          3246024269,
+          357070832,
+          1603682943,
+          564245231,
+          1754654585,
+          4242559940,
+          1405662352,
+          304337439,
+          954374797,
+          2171425718,
+          1490685293,
+          4173134361,
+          1995784092,
+          289628106,
+          4152164384,
+          3843338908,
+          237479743,
+          2441542114,
+          1020515019,
+          3291417701,
+          986416526,
+          1356008802,
+          3484613178,
+          113108980,
+          318453004,
+          1391963122,
+          1109687376,
+          1115778726,
+          230643981,
+          2859243985,
+          1936439260,
+          118238506,
+          1803800220,
+          906563004,
+          154835669,
+          346756003,
+          3360729070,
+          3099550103,
+          3101784492,
+          776170906,
+          1297281926,
+          380753325,
+          3758151639,
+          646035826,
+          909690966,
+          2379017172,
+          444537579,
+          382442146,
+          455800383,
+          909492369,
+          1746445085,
+          2422594107,
+          1178105959,
+          3136714449,
+          517714868,
+          1884374483,
+          494017371,
+          20483656,
+          554441382,
+          3157181873,
+          2769127460,
+          3766793682,
+          1195272478,
+          4268145063,
+          117918643,
+          585904462,
+          2698540751,
+          3593682985,
+          2256758653,
+          4134754565,
+          2008633636,
+          2785094040,
+          3537192527,
+          3335680292,
+          3546859426,
+          3431150853,
+          2347328657,
+          1607160710,
+          969816097,
+          880669845,
+          738436752,
+          3820897665,
+          1305366819,
+          3355293238,
+          793247514,
+          1171122913,
+          2393007827,
+          4036658587,
+          3069377098,
+          118671747,
+          2936823412,
+          2971712946,
+          3753541213,
+          3502016946,
+          2202454204,
+          1765844041,
+          1952360365,
+          354629885,
+          3216371157,
+          2830633394,
+          2520544302,
+          1555810795,
+          732793226,
+          1239312774,
+          1125426851,
+          737387215,
+          3518808549,
+          1525287841,
+          1408314279,
+          81258382,
+          1308391152,
+          2644657614,
+          1272357305,
+          3454601683,
+          257913207,
+          1149254,
+          508780028,
+          3758313929,
+          2997741298,
+          1693091323,
+          2261639261,
+          851241597,
+          3028283967,
+          2510375103,
+          2759802543,
+          3697277582,
+          1410320161,
+          3177175921,
+          282042218,
+          3047933295,
+          740797795,
+          2956179584,
+          4217417214,
+          1250846495,
+          1843921063,
+          977296386,
+          1098971825,
+          975561946,
+          2538843837,
+          1057029641,
+          1432664248,
+          963916206,
+          3001780574,
+          1528201062,
+          3427061470,
+          4161258636,
+          3293572051,
+          1280475438,
+          1154333027,
+          2351022552,
+          2460256320,
+          3765627127,
+          4084025787,
+          49345815,
+          2495925625,
+          192204618,
+          1308900253,
+          324537907,
+          1277369739,
+          2344827154,
+          1242583116,
+          2283476546,
+          2133315572,
+          4111275624,
+          257212101,
+          1110112940,
+          3165624041,
+          291676699,
+          3807518229,
+          3607905791,
+          4065527413,
+          198558585,
+          108918172,
+          4104537885,
+          3872943291,
+          2276176828,
+          2270335955,
+          571664811,
+          92921650,
+          3784463072,
+          2924229426,
+          2456600604,
+          2543495007,
+          2610702095,
+          892609138,
+          135529307,
+          362332904,
+          1768647448,
+          1395364994,
+          2053359072,
+          2473968761,
+          1137695485,
+          393180218,
+          2939312008,
+          1841571272,
+          4209450181,
+          3902014467,
+          1001627943,
+          801790897,
+          68821046,
+          179433736,
+          1753197870,
+          4141692472,
+          3370571722,
+          2231995440,
+          4247944013,
+          4012404212,
+          353227806,
+          54371384,
+          3397819443,
+          1495359527,
+          845805439,
+          677891988,
+          2158343693,
+          1994654247,
+          1188127151,
+          1767692673,
+          2006360143,
+          3631424607,
+          3675570420,
+          2733291861,
+          577080264,
+          2616507875,
+          2211283331,
+          1326669121,
+          3846030707,
+          2740812548,
+          1473200649,
+          2389709127,
+          52636113,
+          3522089805,
+          3478607524,
+          2010576051,
+          4184507682,
+          2203425184,
+          3454474791,
+          328100385,
+          2491101292,
+          3818517334,
+          986513433,
+          2254570280,
+          2689101291,
+          3619772603,
+          1032979283,
+          2907793772,
+          3177386025
+        ],
+        70,
+        0,
+        0.0
+      ],
+      "doc_counts": {
+        "fineweb": 1172738,
+        "code": 488213,
+        "math": 292516
+      },
+      "code_lang_idx": 42
+    },
+    "token_state": {
+      "buffer_tokens": [
+        528,
+        3777,
+        105293,
+        7025,
+        848,
+        1393,
+        90389,
+        9970,
+        528,
+        1823,
+        52098,
+        5567,
+        7104,
+        201,
+        528,
+        2280,
+        16,
+        52098,
+        5567,
+        438,
+        44917,
+        59872,
+        15130,
+        5567,
+        14724,
+        484,
+        425,
+        17507,
+        14,
+        56113,
+        79216,
+        682,
+        9970,
+        528,
+        1823,
+        82470,
+        79,
+        7104,
+        201,
+        528,
+        2280,
+        3314,
+        318,
+        79,
+        438,
+        44917,
+        6547,
+        44806,
+        10,
+        52098,
+        5567,
+        79216,
+        14,
+        539,
+        36662,
+        14414,
+        79216,
+        14,
+        539,
+        36662,
+        893,
+        7755,
+        40087,
+        36259,
+        7755,
+        40087,
+        14,
+        539,
+        36662,
+        116908,
+        31,
+        18689,
+        58580,
+        14,
+        539,
+        36662,
+        84603,
+        31,
+        26420,
+        606,
+        989,
+        36662,
+        26359,
+        74371,
+        19552,
+        682,
+        9970,
+        528,
+        1823,
+        70,
+        2187,
+        7104,
+        201,
+        528,
+        2280,
+        4658,
+        69,
+        438,
+        44917,
+        6547,
+        47883,
+        19177,
+        9239,
+        79216,
+        982,
+        223,
+        20,
+        14,
+        5238,
+        79216,
+        682,
+        9970,
+        528,
+        1823,
+        49414,
+        2019,
+        201,
+        528,
+        2280,
+        16,
+        651,
+        438,
+        44917,
+        2396,
+        24360,
+        1797,
+        3390,
+        9970,
+        361,
+        1351,
+        6058,
+        6666,
+        14,
+        3051,
+        14,
+        3051,
+        29079,
+        85,
+        3395,
+        9970,
+        528,
+        1823,
+        2067,
+        438,
+        764,
+        81143,
+        3701,
+        26909,
+        309,
+        29079,
+        2296,
+        528,
+        23944,
+        438,
+        2280,
+        16,
+        52098,
+        5567,
+        40157,
+        682,
+        528,
+        1823,
+        119525,
+        438,
+        764,
+        81143,
+        3701,
+        14,
+        3667,
+        45834,
+        14,
+        3887,
+        5662,
+        2296,
+        40637,
+        528,
+        1823,
+        19653,
+        284,
+        8205,
+        201,
+        528,
+        26396,
+        65,
+        119525,
+        438,
+        44917,
+        61868,
+        21378,
+        20813,
+        3225,
+        714,
+        5224,
+        47058,
+        65,
+        40922,
+        10,
+        119525,
+        14,
+        3051,
+        29079,
+        85,
+        16181,
+        2471,
+        74371,
+        19552,
+        682,
+        9970,
+        528,
+        26396,
+        18499,
+        14,
+        343,
+        50024,
+        14,
+        2253,
+        11,
+        438,
+        2280,
+        3314,
+        318,
+        79,
+        7093,
+        38483,
+        65,
+        119525,
+        682,
+        528,
+        1823,
+        50024,
+        438,
+        764,
+        81143,
+        3701,
+        14,
+        893,
+        14177,
+        982,
+        893,
+        16623,
+        14,
+        43695,
+        5662,
+        2296,
+        528,
+        1823,
+        11908,
+        438,
+        764,
+        81143,
+        3701,
+        14,
+        893,
+        14177,
+        982,
+        893,
+        16623,
+        14,
+        43695,
+        5662,
+        2296,
+        9970,
+        528,
+        1823,
+        109478,
+        270,
+        4087,
+        6058,
+        305,
+        34555,
+        14414,
+        2501,
+        201,
+        528,
+        14414,
+        438,
+        34536,
+        4001,
+        268,
+        6980,
+        50024,
+        26913,
+        20,
+        62155,
+        62155,
+        3031,
+        14414,
+        26913,
+        19,
+        62155,
+        62155,
+        35495,
+        5662,
+        438,
+        223,
+        19,
+        682,
+        41419,
+        528,
+        1823,
+        50024,
+        438,
+        764,
+        81143,
+        3701,
+        14,
+        25342,
+        5662,
+        982,
+        893,
+        16623,
+        2296,
+        528,
+        27091,
+        18499,
+        85,
+        65369,
+        4658,
+        69,
+        19177,
+        9239,
+        868,
+        528,
+        1823,
+        37557,
+        16277,
+        2019,
+        201,
+        528,
+        25798,
+        65369,
+        16,
+        651,
+        7164,
+        2187,
+        18499,
+        85,
+        682,
+        9970,
+        528,
+        1354,
+        25798,
+        6328,
+        5,
+        940,
+        764,
+        7747,
+        7919,
+        63,
+        4099,
+        1281,
+        67,
+        47,
+        23,
+        71,
+        4739,
+        49,
+        14235,
+        79,
+        5188,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        2067,
+        2701,
+        5,
+        26662,
+        62042,
+        10249,
+        270,
+        2645,
+        271,
+        5,
+        940,
+        4099,
+        1281,
+        70,
+        29020,
+        26,
+        80683,
+        4710,
+        43,
+        1173,
+        44,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        75685,
+        14707,
+        65,
+        16931,
+        3362,
+        582,
+        5395,
+        2272,
+        38017,
+        28,
+        22073,
+        18,
+        17,
+        1760,
+        582,
+        14888,
+        3362,
+        223,
+        2012,
+        95,
+        5238,
+        3372,
+        1281,
+        21,
+        68,
+        1173,
+        3871,
+        72,
+        15,
+        22,
+        67,
+        5844,
+        15,
+        2372,
+        20702,
+        15,
+        27168,
+        19,
+        9855,
+        24,
+        68,
+        15004,
+        72,
+        1892,
+        3814,
+        21,
+        2701,
+        3098,
+        23555,
+        14181,
+        24481,
+        16826,
+        484,
+        425,
+        15531,
+        5,
+        940,
+        4099,
+        1281,
+        25913,
+        83,
+        20,
+        319,
+        46473,
+        7755,
+        16333,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        75685,
+        14707,
+        65,
+        16931,
+        3362,
+        582,
+        5395,
+        2272,
+        38017,
+        28,
+        22073,
+        18,
+        17,
+        1760,
+        582,
+        14888,
+        3362,
+        223,
+        9127,
+        95,
+        5238,
+        3372,
+        1281,
+        31208,
+        26,
+        6088,
+        27,
+        72,
+        2417,
+        26675,
+        24,
+        15,
+        23152,
+        19,
+        15,
+        2248,
+        2122,
+        15,
+        28892,
+        25,
+        70,
+        13636,
+        3663,
+        70,
+        22,
+        2701,
+        13305,
+        12139,
+        46172,
+        201,
+        11920,
+        19704,
+        19525,
+        484,
+        425,
+        438,
+        9080,
+        14181,
+        24481,
+        16826,
+        484,
+        425,
+        682,
+        52098,
+        5567,
+        79216,
+        438,
+        223,
+        1457,
+        201,
+        10064,
+        17462,
+        9239,
+        76737,
+        438,
+        223,
+        2111,
+        201,
+        10064,
+        18499,
+        76737,
+        438,
+        223,
+        19,
+        201,
+        10064,
+        7755,
+        40087,
+        438,
+        223,
+        20,
+        201,
+        18689,
+        14040,
+        438,
+        11485,
+        201,
+        26420,
+        606,
+        438,
+        223,
+        18,
+        16,
+        20,
+        271,
+        5,
+        116284,
+        10249,
+        270,
+        2645,
+        201,
+        18278,
+        438,
+        59934,
+        36035,
+        19704,
+        19525,
+        484,
+        425,
+        14,
+        56113,
+        79216,
+        14,
+        893,
+        17462,
+        9239,
+        76737,
+        117405,
+        18499,
+        76737,
+        14,
+        893,
+        7755,
+        40087,
+        14,
+        539,
+        23259,
+        116908,
+        438,
+        11485,
+        14,
+        84603,
+        438,
+        84603,
+        682,
+        3098,
+        46250,
+        14532,
+        5,
+        940,
+        4099,
+        1281,
+        59,
+        80247,
+        76,
+        72,
+        54,
+        25,
+        2417,
+        100177,
+        24,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        75685,
+        14707,
+        65,
+        16931,
+        3362,
+        582,
+        5395,
+        2272,
+        38017,
+        28,
+        22073,
+        18,
+        17,
+        1760,
+        582,
+        14888,
+        3362,
+        223,
+        4287,
+        95,
+        5238,
+        3372,
+        1281,
+        2616,
+        68,
+        20,
+        67,
+        553,
+        72,
+        15,
+        27772,
+        21,
+        15,
+        24865,
+        20,
+        1761,
+        68,
+        4414,
+        2846,
+        26,
+        68,
+        19,
+        71,
+        3600,
+        14124,
+        22,
+        67,
+        2701,
+        3465,
+        1895,
+        65,
+        46172,
+        46250,
+        3395,
+        361,
+        1354,
+        2595,
+        7093,
+        57251,
+        317,
+        1393,
+        362,
+        280,
+        295,
+        2645,
+        16,
+        46172,
+        1393,
+        855,
+        280,
+        16,
+        101936,
+        15810,
+        2673,
+        682,
+        5191,
+        3098,
+        5123,
+        125191,
+        2645,
+        769,
+        680,
+        10499,
+        65,
+        46172,
+        46250,
+        2605,
+        14,
+        95,
+        9251,
+        679,
+        8252,
+        21114,
+        5,
+        97458,
+        270,
+        6652,
+        17021,
+        56113,
+        201,
+        6452,
+        17021,
+        65,
+        3365,
+        42576,
+        1045,
+        438,
+        66427,
+        16826,
+        484,
+        425,
+        16,
+        77060,
+        201,
+        18278,
+        16,
+        52098,
+        5567,
+        120299,
+        14514,
+        69151,
+        24625,
+        6452,
+        17021,
+        65,
+        3365,
+        42576,
+        1045,
+        868,
+        3098,
+        10,
+        6452,
+        17021,
+        65,
+        3365,
+        42576,
+        1045,
+        43816,
+        868,
+        5,
+        940,
+        764,
+        7747,
+        7919,
+        63,
+        4099,
+        1281,
+        74,
+        22,
+        89,
+        18,
+        15387,
+        25104,
+        11340,
+        83,
+        39,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        2067,
+        2701,
+        5,
+        26662,
+        39285,
+        6177,
+        396,
+        9251,
+        270,
+        2645,
+        271,
+        5,
+        940,
+        4099,
+        1281,
+        22,
+        83,
+        81,
+        25,
+        47645,
+        18,
+        30009,
+        25,
+        118455,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        5364,
+        1778,
+        1897,
+        34536,
+        16,
+        49461,
+        412,
+        7944,
+        271,
+        13305,
+        109946,
+        305,
+        4721,
+        201,
+        49461,
+        9160,
+        438,
+        7944,
+        16,
+        50590,
+        46250,
+        16,
+        46172,
+        12258,
+        69,
+        91415,
+        438,
+        44917,
+        16,
+        6588,
+        4659,
+        3726,
+        10414,
+        13305,
+        17588,
+        201,
+        3465,
+        11680,
+        65,
+        70336,
+        7093,
+        1157,
+        85,
+        14,
+        383,
+        3395,
+        361,
+        1823,
+        1373,
+        26145,
+        304,
+        270,
+        22510,
+        9132,
+        201,
+        361,
+        22912,
+        58537,
+        85,
+        438,
+        34536,
+        113252,
+        7093,
+        1157,
+        85,
+        682,
+        5191,
+        361,
+        4813,
+        438,
+        343,
+        63206,
+        58537,
+        85,
+        2606,
+        383,
+        797,
+        15891,
+        1393,
+        539,
+        361,
+        1086,
+        438,
+        4813,
+        65172,
+        1393,
+        1492,
+        9080,
+        6434,
+        43818,
+        682,
+        361,
+        1354,
+        1086,
+        29406,
+        5,
+        38312,
+        304,
+        274,
+        16502,
+        855,
+        3510,
+        201,
+        18278,
+        438,
+        2645,
+        14373,
+        10,
+        32351,
+        682,
+        69,
+        91415,
+        438,
+        32866,
+        14373,
+        10,
+        32351,
+        14532,
+        5,
+        940,
+        4099,
+        1281,
+        54,
+        86,
+        511,
+        52488,
+        38,
+        73,
+        126500,
+        47,
+        72,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        5364,
+        1778,
+        3465,
+        9251,
+        46250,
+        14,
+        65540,
+        14,
+        109946,
+        14,
+        32866,
+        3395,
+        5191,
+        361,
+        1823,
+        100716,
+        1750,
+        49704,
+        539,
+        361,
+        49704,
+        73090,
+        438,
+        223,
+        18,
+        201,
+        361,
+        49704,
+        122794,
+        438,
+        223,
+        18,
+        29406,
+        361,
+        1823,
+        3284,
+        270,
+        2645,
+        295,
+        5026,
+        7110,
+        201,
+        361,
+        2645,
+        123044,
+        1393,
+        2143,
+        5191,
+        361,
+        362,
+        26359,
+        295,
+        65540,
+        1137,
+        9970,
+        528,
+        1823,
+        443,
+        1697,
+        270,
+        59773,
+        1561,
+        1750,
+        26359,
+        201,
+        528,
+        109946,
+        16,
+        20692,
+        15810,
+        2673,
+        1393,
+        14320,
+        9970,
+        528,
+        1823,
+        2101,
+        59395,
+        3051,
+        305,
+        1119,
+        16,
+        294,
+        3345,
+        201,
+        528,
+        3051,
+        14,
+        3051,
+        29079,
+        85,
+        438,
+        26359,
+        20607,
+        14320,
+        9970,
+        528,
+        1823,
+        54276,
+        304,
+        223,
+        19,
+        38,
+        40492,
+        201,
+        528,
+        26145,
+        438,
+        2645,
+        40157,
+        14,
+        3051,
+        29079,
+        85,
+        797,
+        85,
+        91998,
+        2317,
+        1393,
+        2143,
+        9970,
+        528,
+        1823,
+        119461,
+        270,
+        4721,
+        201,
+        528,
+        4721,
+        438,
+        32866,
+        7093,
+        18349,
+        685,
+        14,
+        26359,
+        45310,
+        11,
+        97247,
+        528,
+        1823,
+        119461,
+        270,
+        11680,
+        12075,
+        201,
+        528,
+        1086,
+        438,
+        11680,
+        65,
+        70336,
+        7093,
+        18349,
+        685,
+        14,
+        26359,
+        45310,
+        11,
+        14320,
+        9970,
+        528,
+        1823,
+        3555,
+        1297,
+        14101,
+        270,
+        4721,
+        305,
+        23917,
+        270,
+        59773,
+        201,
+        528,
+        4721,
+        62835,
+        1625,
+        1393,
+        528,
+        42130,
+        528,
+        1823,
+        16051,
+        270,
+        23997,
+        201,
+        528,
+        109946,
+        7192,
+        755,
+        1393,
+        40637,
+        9970,
+        528,
+        1823,
+        32511,
+        305,
+        12075,
+        201,
+        528,
+        49704,
+        73090,
+        7004,
+        4721,
+        59666,
+        1393,
+        2143,
+        528,
+        49704,
+        122794,
+        7004,
+        1086,
+        59666,
+        1393,
+        5191,
+        9970,
+        361,
+        1354,
+        49704,
+        73090,
+        1492,
+        9080,
+        10,
+        43131,
+        754,
+        49704,
+        122794,
+        1492,
+        9080,
+        10,
+        43131,
+        14532,
+        5,
+        940,
+        4099,
+        1281,
+        110197,
+        25,
+        76,
+        9417,
+        13902,
+        37,
+        44,
+        16077,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        5364,
+        1778,
+        3465,
+        12582,
+        46250,
+        14,
+        65540,
+        14,
+        32866,
+        3395,
+        5191,
+        361,
+        1823,
+        100716,
+        1750,
+        49704,
+        201,
+        361,
+        49704,
+        73090,
+        438,
+        223,
+        18,
+        201,
+        361,
+        49704,
+        122794,
+        438,
+        223,
+        18,
+        271,
+        361,
+        1823,
+        928,
+        651,
+        21100,
+        84603,
+        14177,
+        201,
+        361,
+        2645,
+        4987,
+        1879,
+        3390,
+        5191,
+        361,
+        1823,
+        928,
+        52254,
+        1074,
+        2667,
+        121452,
+        201,
+        361,
+        418,
+        34536,
+        58787,
+        15810,
+        2673,
+        15536,
+        5191,
+        528,
+        362,
+        26359,
+        295,
+        65540,
+        1137,
+        9970,
+        775,
+        1823,
+        2101,
+        59395,
+        3051,
+        305,
+        1119,
+        16,
+        294,
+        3345,
+        201,
+        775,
+        3051,
+        14,
+        3051,
+        29079,
+        85,
+        438,
+        26359,
+        20607,
+        87346,
+        775,
+        1823,
+        54276,
+        304,
+        223,
+        19,
+        70,
+        40492,
+        201,
+        775,
+        26145,
+        438,
+        2645,
+        40157,
+        14,
+        3051,
+        29079,
+        85,
+        797,
+        85,
+        91998,
+        2317,
+        3390,
+        20256,
+        775,
+        1823,
+        119461,
+        4721,
+        305,
+        12075,
+        201,
+        775,
+        4721,
+        438,
+        32866,
+        7093,
+        18349,
+        685,
+        14,
+        26359,
+        45310,
+        682,
+        775,
+        1086,
+        438,
+        11680,
+        65,
+        70336,
+        7093,
+        18349,
+        685,
+        14,
+        26359,
+        45310,
+        682,
+        20256,
+        775,
+        1823,
+        38140,
+        6781,
+        294,
+        4721,
+        305,
+        12075,
+        201,
+        775,
+        49704,
+        73090,
+        7004,
+        4721,
+        59666,
+        3390,
+        775,
+        49704,
+        122794,
+        7004,
+        1086,
+        59666,
+        3390,
+        9970,
+        361,
+        1354,
+        49704,
+        73090,
+        1492,
+        9080,
+        10,
+        43131,
+        754,
+        49704,
+        122794,
+        1492,
+        9080,
+        10,
+        43131,
+        14532,
+        5,
+        940,
+        764,
+        7747,
+        7919,
+        63,
+        4099,
+        1281,
+        44,
+        11853,
+        856,
+        7079,
+        24,
+        5438,
+        20907,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        2067,
+        2701,
+        5,
+        26662,
+        33477,
+        270,
+        2645,
+        201,
+        5,
+        9909,
+        17621,
+        513,
+        566,
+        4366,
+        28,
+        5042,
+        2272,
+        4771,
+        16,
+        11739,
+        77621,
+        28817,
+        78829,
+        2193,
+        78150,
+        17,
+        939,
+        18,
+        17,
+        2616,
+        11686,
+        1061,
+        33338,
+        19116,
+        2886,
+        4767,
+        3095,
+        3472,
+        73507,
+        9805,
+        5,
+        940,
+        4099,
+        1281,
+        17659,
+        4362,
+        59,
+        91,
+        1328,
+        10525,
+        21,
+        87994,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        75685,
+        14707,
+        65,
+        16931,
+        3362,
+        582,
+        5395,
+        2272,
+        38017,
+        28,
+        22073,
+        18,
+        17,
+        1760,
+        582,
+        14888,
+        3362,
+        223,
+        20530,
+        95,
+        5238,
+        3372,
+        1281,
+        5628,
+        18,
+        69,
+        3565,
+        69,
+        24,
+        15,
+        2111,
+        71,
+        22,
+        15,
+        22,
+        928,
+        19,
+        2846,
+        29573,
+        15,
+        1002,
+        72,
+        28175,
+        70,
+        3318,
+        5920,
+        25,
+        2701,
+        48,
+        65,
+        7989,
+        49,
+        5087,
+        53,
+        438,
+        223,
+        553,
+        201,
+        21910,
+        63581,
+        73090,
+        438,
+        12249,
+        2292,
+        16156,
+        21114,
+        2251,
+        49704,
+        295,
+        3291,
+        14615,
+        65,
+        7989,
+        49,
+        5087,
+        53,
+        3395,
+        51235,
+        361,
+        1823,
+        56050,
+        270,
+        2645,
+        201,
+        361,
+        9251,
+        73090,
+        14,
+        9251,
+        122794,
+        438,
+        9251,
+        46250,
+        14,
+        9251,
+        96557,
+        14,
+        109946,
+        14,
+        32866,
+        682,
+        5191,
+        361,
+        1823,
+        71,
+        71990,
+        270,
+        2645,
+        201,
+        361,
+        6405,
+        73090,
+        14,
+        6405,
+        122794,
+        438,
+        12582,
+        46250,
+        14,
+        6405,
+        96557,
+        14,
+        32866,
+        682,
+        5191,
+        361,
+        1823,
+        39357,
+        270,
+        2455,
+        2645,
+        201,
+        361,
+        855,
+        6405,
+        73090,
+        818,
+        2455,
+        63581,
+        73090,
+        1137,
+        528,
+        2455,
+        63581,
+        73090,
+        438,
+        6405,
+        73090,
+        201,
+        528,
+        34536,
+        37875,
+        46250,
+        33148,
+        30426,
+        10474,
+        905,
+        85,
+        14240,
+        65,
+        40300,
+        16,
+        529,
+        6374,
+        5191,
+        361,
+        2777,
+        5123,
+        58456,
+        86,
+        75939,
+        31662,
+        28,
+        680,
+        56050,
+        73090,
+        47657,
+        21,
+        72,
+        95,
+        369,
+        33477,
+        5306,
+        28,
+        680,
+        56050,
+        122794,
+        12,
+        1457,
+        47657,
+        20,
+        72,
+        95,
+        7,
+        6374,
+        361,
+        2777,
+        5123,
+        58456,
+        86,
+        4752,
+        16,
+        31662,
+        28,
+        680,
+        11356,
+        73090,
+        47657,
+        21,
+        72,
+        95,
+        369,
+        223,
+        4752,
+        16,
+        5306,
+        28,
+        680,
+        11356,
+        122794,
+        12,
+        1457,
+        47657,
+        20,
+        72,
+        95,
+        7,
+        21114,
+        5,
+        940,
+        764,
+        7747,
+        7919,
+        63,
+        4099,
+        1281,
+        68,
+        48,
+        86206,
+        109344,
+        61376,
+        19793,
+        6446,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        2067,
+        2701,
+        5,
+        1823,
+        6575,
+        270,
+        2645,
+        734,
+        26145,
+        201,
+        25358,
+        5,
+        1350,
+        1928,
+        611,
+        260,
+        17024,
+        2645,
+        16141,
+        295,
+        16141,
+        65,
+        40300,
+        16,
+        529,
+        16,
+        1350,
+        759,
+        7263,
+        436,
+        304,
+        270,
+        1950,
+        1341,
+        16,
+        20534,
+        14,
+        579,
+        477,
+        554,
+        22925,
+        943,
+        12075,
+        295,
+        270,
+        7075,
+        4880,
+        16,
+        1884,
+        1132,
+        10342,
+        579,
+        1438,
+        1970,
+        304,
+        1635,
+        3645,
+        566,
+        2645,
+        344,
+        49495,
+        2765,
+        29750,
+        15,
+        69945,
+        3345,
+        339,
+        5,
+        940,
+        4099,
+        1281,
+        6754,
+        25,
+        89,
+        49,
+        27,
+        56,
+        86,
+        601,
+        38,
+        82,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        5364,
+        1778,
+        5,
+        3932,
+        23997,
+        201,
+        9860,
+        6471,
+        85,
+        14240,
+        65,
+        40300,
+        16,
+        529,
+        5832,
+        18278,
+        35162,
+        32915,
+        30426,
+        4665,
+        73507,
+        35162,
+        34333,
+        7021,
+        18278,
+        4987,
+        1879,
+        8517,
+        5,
+        261,
+        2838,
+        539,
+        1897,
+        878,
+        3926,
+        201,
+        58596,
+        82,
+        438,
+        878,
+        3926,
+        35162,
+        2292,
+        267,
+        21114,
+        3465,
+        6235,
+        46250,
+        14,
+        10175,
+        3395,
+        361,
+        17840,
+        1766,
+        438,
+        764,
+        77734,
+        20607,
+        362,
+        28828,
+        295,
+        313,
+        65541,
+        99885,
+        9160,
+        3982,
+        51015,
+        14245,
+        223,
+        1823,
+        33912,
+        1387,
+        270,
+        10175,
+        539,
+        361,
+        65647,
+        438,
+        764,
+        54,
+        24481,
+        16826,
+        484,
+        425,
+        7192,
+        8408,
+        45515,
+        63,
+        362,
+        259,
+        295,
+        17840,
+        1766,
+        63,
+        3392,
+        1823,
+        54276,
+        304,
+        9132,
+        8205,
+        201,
+        361,
+        4892,
+        438,
+        764,
+        6321,
+        38827,
+        284,
+        14245,
+        15970,
+        1823,
+        119461,
+        1119,
+        16,
+        294,
+        3345,
+        201,
+        361,
+        40492,
+        438,
+        34536,
+        6547,
+        700,
+        95975,
+        38827,
+        284,
+        797,
+        1495,
+        10,
+        32351,
+        11,
+        6118,
+        1823,
+        54276,
+        304,
+        40492,
+        201,
+        361,
+        40492,
+        438,
+        40492,
+        16,
+        9921,
+        91998,
+        2317,
+        10,
+        19,
+        797,
+        54,
+        33901,
+        1823,
+        93269,
+        295,
+        1179,
+        294,
+        26359,
+        14,
+        3567,
+        16,
+        294,
+        3345,
+        201,
+        361,
+        4892,
+        3682,
+        14772,
+        438,
+        34536,
+        6547,
+        700,
+        95975,
+        82380,
+        11,
+        23259,
+        1823,
+        54276,
+        304,
+        40492,
+        201,
+        361,
+        18800,
+        438,
+        2645,
+        4665,
+        14772,
+        14,
+        4892,
+        3682,
+        14772,
+        11,
+        11228,
+        1823,
+        49654,
+        4131,
+        539,
+        361,
+        1354,
+        18800,
+        61,
+        18,
+        2655,
+        6813,
+        59931,
+        5,
+        940,
+        4099,
+        1281,
+        20,
+        84,
+        31451,
+        24,
+        12465,
+        26,
+        78,
+        32207,
+        26,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        5364,
+        1778,
+        26758,
+        3682,
+        92928,
+        438,
+        1950,
+        102972,
+        2349,
+        5404,
+        10,
+        4015,
+        72137,
+        19,
+        797,
+        64770,
+        20798,
+        7164,
+        2690,
+        19552,
+        682,
+        26758,
+        3682,
+        92928,
+        10931,
+        59724,
+        4131,
+        13747,
+        438,
+        764,
+        92980,
+        46250,
+        14,
+        26758,
+        3682,
+        92928,
+        5497,
+        266,
+        87394,
+        7233,
+        3166,
+        8123,
+        362,
+        1008,
+        295,
+        3291,
+        23555,
+        3982,
+        51198,
+        3682,
+        92928,
+        2542,
+        1839,
+        5,
+        940,
+        4099,
+        1281,
+        2047,
+        4687,
+        78,
+        3255,
+        59,
+        78,
+        24,
+        98533,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        75685,
+        14707,
+        65,
+        16931,
+        3362,
+        582,
+        5395,
+        2272,
+        38017,
+        28,
+        22073,
+        18,
+        17,
+        1760,
+        582,
+        14888,
+        3362,
+        223,
+        9489,
+        95,
+        5238,
+        3372,
+        1281,
+        27437,
+        353,
+        23103,
+        15,
+        2597,
+        69,
+        23,
+        15,
+        23559,
+        25,
+        1761,
+        20512,
+        15,
+        26,
+        68,
+        24,
+        70,
+        2738,
+        70,
+        1883,
+        9787,
+        26,
+        2701,
+        26758,
+        3682,
+        92928,
+        33697,
+        10414,
+        5,
+        940,
+        4099,
+        1281,
+        80,
+        47,
+        114965,
+        44,
+        74812,
+        24873,
+        2417,
+        51,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        8308,
+        4,
+        1278,
+        425,
+        75685,
+        14707,
+        65,
+        16931,
+        3362,
+        582,
+        5395,
+        2272,
+        38017,
+        28,
+        22073,
+        18,
+        17,
+        1760,
+        582,
+        14888,
+        3362,
+        223,
+        18528,
+        95,
+        5238,
+        3372,
+        1281,
+        71,
+        1671,
+        70,
+        31353,
+        69,
+        15,
+        3315,
+        26675,
+        15,
+        22,
+        439,
+        18,
+        15,
+        26,
+        71,
+        2973,
+        1761,
+        3661,
+        68,
+        5769,
+        15004,
+        70,
+        20,
+        2701,
+        26758,
+        3682,
+        92928,
+        61,
+        26758,
+        3682,
+        92928,
+        16,
+        29140,
+        1955,
+        2395,
+        3682,
+        92928,
+        4584,
+        35,
+        2655,
+        59724,
+        4131,
+        5196,
+        435,
+        79153,
+        6471,
+        27602,
+        1780,
+        8339,
+        72137,
+        21,
+        682,
+        26758,
+        3682,
+        92928,
+        61,
+        26758,
+        3682,
+        92928,
+        4584,
+        35,
+        1955,
+        2395,
+        3682,
+        92928,
+        16,
+        29140,
+        2655,
+        59724,
+        4131,
+        5196,
+        435,
+        79153,
+        6471,
+        83471,
+        1780,
+        8339,
+        72137,
+        21,
+        682,
+        10338,
+        1154,
+        438,
+        764,
+        93225,
+        6980,
+        18,
+        14,
+        18,
+        754,
+        19,
+        14,
+        19,
+        14,
+        10323,
+        38676,
+        14,
+        1366,
+        1281,
+        77,
+        1760,
+        8339,
+        72137,
+        20,
+        11,
+        362,
+        274,
+        295,
+        13771,
+        27602,
+        7759,
+        83471,
+        6713,
+        2296,
+        90854,
+        31,
+        20584,
+        6203,
+        9472,
+        13896,
+        10177,
+        6203,
+        29750,
+        13896,
+        35043,
+        26624,
+        30010,
+        1698,
+        36763,
+        13999,
+        62330,
+        5385,
+        14614,
+        1884,
+        330,
+        92928,
+        418,
+        32016,
+        12040,
+        763,
+        57812,
+        3410,
+        5225,
+        26624,
+        16,
+        66090,
+        10,
+        10338,
+        1154,
+        14,
+        22377,
+        682,
+        26624,
+        116610,
+        1698,
+        36763,
+        13999,
+        5225,
+        26624,
+        36282,
+        10414,
+        5,
+        940,
+        764,
+        7747,
+        7919,
+        63,
+        4099,
+        1281,
+        18,
+        68,
+        40761,
+        7398,
+        17992,
+        17174,
+        10644,
+        4,
+        1278,
+        425,
+        15136,
+        1281,
+        2067,
+        2701,
+        5,
+        1350,
+        611,
+        270,
+        1529,
+        579,
+        5829,
+        201,
+        1
+      ],
+      "seq_len": 2048
+    }
+  },
+  "config": {
+    "vocab_size": 129280,
+    "dim": 1536,
+    "n_layers": 16,
+    "n_heads": 12,
+    "head_dim": 128,
+    "rope_head_dim": 32,
+    "q_lora_rank": 384,
+    "o_groups": 4,
+    "o_lora_rank": 256,
+    "norm_eps": 1e-06,
+    "n_routed_experts": 12,
+    "n_shared_experts": 1,
+    "n_activated_experts": 2,
+    "moe_inter_dim": 1408,
+    "score_func": "softmax",
+    "route_scale": 1.0,
+    "swiglu_limit": 15.0,
+    "n_dense_layers": 2,
+    "load_balance_alpha": 0.003,
+    "hc_mult": 3,
+    "hc_sinkhorn_iters": 8,
+    "hc_eps": 1e-06,
+    "window_size": 128,
+    "compress_ratio": 4,
+    "n_mtp_layers": 1,
+    "mtp_loss_weight": 0.02,
+    "rope_theta": 10000.0,
+    "rope_factor": 1.0,
+    "original_seq_len": 0,
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "ctx_phase_boundaries": [
+      60000,
+      80000,
+      100000
+    ],
+    "ctx_phase_lengths": [
+      2048,
+      4096,
+      8192
+    ],
+    "yarn_scale": 40.0,
+    "yarn_alpha": 1.0,
+    "yarn_beta": 32.0,
+    "yarn_original_seq_len": 4096,
+    "micro_batch_size": 1,
+    "gradient_accumulation_steps": 32,
+    "max_lr": 0.00015,
+    "min_lr": 1.5e-05,
+    "warmup_steps": 2000,
+    "total_steps": 200000,
+    "decay_steps": 180000,
+    "weight_decay": 0.1,
+    "muon_beta": 0.95,
+    "muon_ns_steps": 7,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.98,
+    "grad_clip_norm": 5.0,
+    "init_std": 0.006,
+    "dtype": "bfloat16",
+    "fineweb_config": "default",
+    "approx_tokens_per_fineweb_doc": 800,
+    "approx_tokens_per_code_doc": 1200,
+    "approx_tokens_per_math_doc": 1500,
+    "data_mix_probs": [
+      0.6,
+      0.25,
+      0.15
+    ],
+    "save_every_steps": 400,
+    "log_every_steps": 10,
+    "push_every_steps": 400,
+    "checkpoint_dir": "/kaggle/working/zenyxv3checkpoints"
+  }
+}

checkpoints/step_18800/opt_state.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ef12ff684c0ffd78876e67f08a2f184aa969b95da14ca8a5b9f554c14fd71e9
+size 3154940698

checkpoints/step_18800/params.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74202a289da428f01b06482fb1ffe2aaf644ef73192e8fa69e1c4fd40d63283f
+size 3154768094