step 22800

checkpoints/step_22800/metadata.json

@@ -0,0 +1,3090 @@
+{
+  "step": 22800,
+  "tokens_consumed": 11953766400,
+  "sequences_consumed": 5836800,
+  "data_state": {
+    "stream_state": {
+      "rng_state": [
+        "MT19937",
+        [
+          2174665208,
+          1685337710,
+          3897210000,
+          1430225562,
+          1078670980,
+          442154163,
+          1522673139,
+          2191761991,
+          1026304802,
+          3454828,
+          1326029782,
+          2911147431,
+          477047128,
+          2494279014,
+          945190775,
+          4138676206,
+          3846061560,
+          2447097797,
+          3201596734,
+          86781320,
+          3967658519,
+          4154716925,
+          4138203041,
+          3957033364,
+          782912307,
+          1975513767,
+          139812365,
+          759270200,
+          195343042,
+          36948854,
+          1994740553,
+          4112984638,
+          839630459,
+          2241415643,
+          2054257405,
+          1115147949,
+          388312073,
+          4105464317,
+          2224519055,
+          983693630,
+          1671589638,
+          3081423815,
+          610541539,
+          478928868,
+          816718980,
+          2194857003,
+          3190497537,
+          1006602145,
+          2443050906,
+          439343078,
+          3188263187,
+          2817030351,
+          3043890967,
+          2455355250,
+          2582538627,
+          765388789,
+          449786934,
+          2929891150,
+          1840158683,
+          2189947573,
+          3092102428,
+          3654127033,
+          355361267,
+          3632195780,
+          1073347004,
+          4030500059,
+          994420685,
+          1484137878,
+          4257197669,
+          174570493,
+          2680472913,
+          1914628899,
+          2369588451,
+          3879049272,
+          4196073421,
+          678755113,
+          4096615214,
+          2248107362,
+          3186324151,
+          68322489,
+          3593942367,
+          944270376,
+          3635597445,
+          4254989992,
+          682701881,
+          2534322821,
+          981793962,
+          3577803080,
+          1023462497,
+          3102661008,
+          3438038117,
+          1359415275,
+          1474290227,
+          1046341291,
+          1565461925,
+          3157872524,
+          4111275780,
+          3174626279,
+          169611168,
+          2002127173,
+          3589873487,
+          4285119476,
+          3433142243,
+          50450429,
+          2131558782,
+          1113612423,
+          3718003634,
+          3551992592,
+          2845433669,
+          2251695397,
+          553497568,
+          452992892,
+          435450655,
+          606915774,
+          659819309,
+          69340338,
+          1705707092,
+          2517964165,
+          1789468097,
+          2764869498,
+          1549318808,
+          3971647506,
+          1304270320,
+          2604850972,
+          2723324075,
+          332953824,
+          2592897299,
+          251147492,
+          352013217,
+          1976068517,
+          770872534,
+          826896142,
+          3226194350,
+          2253973625,
+          2406573910,
+          3061038311,
+          3225339076,
+          1341949969,
+          2916816774,
+          2376768124,
+          281376406,
+          3266882392,
+          1703686407,
+          413437204,
+          1385647890,
+          1572777422,
+          2168414684,
+          596413994,
+          870177135,
+          3936450466,
+          1276326564,
+          3804282209,
+          998070537,
+          3204665044,
+          2715654501,
+          2200410634,
+          1449684605,
+          3268120328,
+          1126985003,
+          1939757115,
+          3486483693,
+          3835449114,
+          2781768940,
+          339017284,
+          2787659299,
+          2129999746,
+          297661081,
+          720550516,
+          585834575,
+          2555311114,
+          3668276260,
+          3971420274,
+          4190483645,
+          3772576521,
+          1849816302,
+          2773487027,
+          103121364,
+          3339104156,
+          4128071663,
+          161096551,
+          1068141518,
+          1293283550,
+          415218316,
+          1557216627,
+          3266222167,
+          4201739507,
+          4257869694,
+          2773643593,
+          133900371,
+          3399508679,
+          4093907186,
+          3197993197,
+          1227920485,
+          679626337,
+          3467071603,
+          581395748,
+          706733843,
+          3610028040,
+          2673122547,
+          3002836336,
+          3101414347,
+          1618766589,
+          1731871403,
+          67300286,
+          4105203008,
+          3699350553,
+          2503565454,
+          4289740174,
+          3002269321,
+          3559052178,
+          3834289506,
+          3751042028,
+          3497212816,
+          1236880617,
+          3073028452,
+          2247213686,
+          3362094520,
+          869006299,
+          4060290068,
+          1493609756,
+          983741027,
+          2751630843,
+          631431097,
+          2686633894,
+          2645329779,
+          3470362342,
+          3892567852,
+          3802858832,
+          921651499,
+          2005983687,
+          3503031753,
+          1928594895,
+          3803495608,
+          702457811,
+          2712667346,
+          3485408325,
+          2681718574,
+          1131178268,
+          256078945,
+          3434277238,
+          2805210404,
+          642442786,
+          3109022081,
+          3832698601,
+          3158284598,
+          3037713332,
+          1335533573,
+          912194645,
+          2089094294,
+          3997037320,
+          3052543780,
+          766363640,
+          179729759,
+          3303248507,
+          1378975232,
+          839342394,
+          2155817248,
+          1602083849,
+          3888507790,
+          1918432453,
+          4068167215,
+          3055128607,
+          3516953632,
+          405919675,
+          2340739197,
+          2136006932,
+          2089942315,
+          14075319,
+          2554421905,
+          1253872791,
+          3800219680,
+          1346087824,
+          1184878290,
+          2530172369,
+          2596532055,
+          908160838,
+          621869457,
+          136129246,
+          2015378687,
+          458984907,
+          1486582657,
+          1253481873,
+          2936341865,
+          3298870905,
+          4143273582,
+          3599249924,
+          3140638446,
+          3290077581,
+          2408876710,
+          792919155,
+          785974567,
+          1316754245,
+          1693549520,
+          2290854577,
+          186615801,
+          1308096488,
+          453962618,
+          1426400823,
+          1277083805,
+          2317201161,
+          2581378817,
+          141159061,
+          3856901207,
+          3232573587,
+          1303019916,
+          1020538568,
+          2891091591,
+          1434442583,
+          4129813088,
+          2699366154,
+          1197448051,
+          3072225153,
+          578677225,
+          3348047590,
+          973553329,
+          2510726141,
+          2604662623,
+          1647413243,
+          4014771310,
+          3586851785,
+          2341516257,
+          2584654995,
+          3142548244,
+          3271078686,
+          1593626492,
+          2243008091,
+          6980476,
+          1226563915,
+          3106377217,
+          2754295836,
+          3222056624,
+          4098496996,
+          1765846765,
+          2044257728,
+          2135889124,
+          1306063651,
+          3760175518,
+          1588777045,
+          4150203955,
+          3243949370,
+          3494445499,
+          3102197490,
+          168057893,
+          4048377031,
+          4236214132,
+          1067505003,
+          3482915777,
+          688637525,
+          3015052956,
+          543210964,
+          1315192938,
+          2704276911,
+          1853538758,
+          2066665783,
+          4199815794,
+          1538627898,
+          1012784129,
+          3925705622,
+          1823462903,
+          2147575997,
+          1871415296,
+          466698915,
+          2673289792,
+          1865367306,
+          2342571293,
+          2942901671,
+          641557064,
+          1291855950,
+          92102452,
+          1964722778,
+          4131376732,
+          3559698554,
+          2804493083,
+          2549275195,
+          1437529784,
+          516737748,
+          2649023047,
+          553491511,
+          832461959,
+          864603183,
+          844462990,
+          1635634555,
+          1357628727,
+          1210165311,
+          3251620924,
+          4227922295,
+          1725136972,
+          1237449414,
+          1292270623,
+          3975881530,
+          4147327584,
+          616587807,
+          2624159025,
+          283461359,
+          1081566315,
+          429980623,
+          2582393108,
+          3108909292,
+          2338547893,
+          3960116262,
+          2087653253,
+          3115662225,
+          4205448810,
+          2210162458,
+          1267322956,
+          2062761051,
+          1238219475,
+          4036784002,
+          2023166525,
+          2551827647,
+          2579482472,
+          3906541486,
+          1879107651,
+          676397511,
+          3626003293,
+          1321966558,
+          231693291,
+          2829398065,
+          4050340932,
+          1612130677,
+          415692594,
+          797408035,
+          1235856336,
+          3100966616,
+          4077406297,
+          2617887485,
+          3827482205,
+          2104487301,
+          2093674897,
+          3370845544,
+          409095372,
+          782953668,
+          2832231227,
+          666938300,
+          3548440344,
+          271703271,
+          2109474601,
+          524582947,
+          2666993687,
+          1458397622,
+          799298016,
+          1961305557,
+          2776826597,
+          1731744463,
+          4093719714,
+          673960135,
+          2064280785,
+          2272633983,
+          3741164656,
+          1505497098,
+          803184090,
+          529242729,
+          1823545207,
+          2957653399,
+          1726130029,
+          4101738724,
+          2929471648,
+          2471019252,
+          2438629622,
+          2772387195,
+          990747776,
+          1081195198,
+          3293032027,
+          2492249567,
+          2516987883,
+          2935675168,
+          2413399567,
+          1481819482,
+          914816221,
+          980257386,
+          1060994675,
+          209058741,
+          1402430607,
+          3976877449,
+          3184570776,
+          139402967,
+          4180309301,
+          666126761,
+          4282582868,
+          95754307,
+          2084192042,
+          821655510,
+          475395060,
+          2142394854,
+          4187427956,
+          376385462,
+          3783840824,
+          446803304,
+          1293539522,
+          2137089092,
+          686070669,
+          3321409644,
+          4026351005,
+          4188432497,
+          3467968641,
+          480579859,
+          896781175,
+          2992152139,
+          2219425703,
+          530594484,
+          163845696,
+          531954315,
+          3481047363,
+          447698738,
+          1107712767,
+          2065831686,
+          4163708996,
+          1893744795,
+          2740558825,
+          713217428,
+          2182962144,
+          363259781,
+          1457685389,
+          3755427957,
+          3395749698,
+          3429870028,
+          4004850478,
+          3628910489,
+          894503830,
+          3176646683,
+          3724541256,
+          494896043,
+          605546758,
+          3620532608,
+          2125506268,
+          950701959,
+          3144632692,
+          273739871,
+          2636169006,
+          4270617145,
+          2987515943,
+          2200982612,
+          2113096521,
+          3566190675,
+          3108829879,
+          3381626803,
+          968684829,
+          919582919,
+          1881238209,
+          3074962065,
+          1768564599,
+          283559447,
+          1034154266,
+          999402606,
+          2601727235,
+          3562122343,
+          242521030,
+          315243547,
+          3835239207,
+          2866101454,
+          960566325,
+          363767639,
+          4161422539,
+          3191964873,
+          4028957613,
+          1440092992,
+          2207833714,
+          32888329,
+          2194250342,
+          4278062099,
+          1422676812,
+          2067696311,
+          4094165419,
+          865308009,
+          2541457376,
+          544300695,
+          3983321919,
+          2005544457,
+          3183721136,
+          1091538885,
+          3625069589,
+          232890366,
+          3639340604,
+          1793731839,
+          1903508866,
+          972866613,
+          3066349290,
+          2456800913,
+          1090656178,
+          2119020182,
+          3811874550,
+          2282276788,
+          2286082266,
+          1419705541,
+          968504097,
+          3468594649,
+          3594448219,
+          1814518094,
+          2420593817,
+          4111601960,
+          1651021339,
+          1881312719,
+          1270088854,
+          3729115616,
+          1909918259,
+          2265133423,
+          297838051,
+          2963250500,
+          3345793307,
+          2685179578,
+          3627173724,
+          3753656241,
+          438636984,
+          2792436946,
+          4080401924,
+          599455564,
+          1310662230,
+          3537502458,
+          2493145721,
+          4029880786,
+          622575995,
+          526696787,
+          343214292,
+          379858399,
+          1993909678,
+          267797854,
+          1462036856,
+          315148079,
+          865800102,
+          3466205427,
+          2323370381,
+          4109756874,
+          1291985248,
+          2569228776
+        ],
+        162,
+        0,
+        0.0
+      ],
+      "doc_counts": {
+        "fineweb": 2181211,
+        "code": 908522,
+        "math": 545148
+      },
+      "code_lang_idx": 8
+    },
+    "token_state": {
+      "buffer_tokens": [
+        554,
+        366,
+        4737,
+        513,
+        7918,
+        418,
+        260,
+        3980,
+        294,
+        4443,
+        122407,
+        603,
+        56,
+        409,
+        1534,
+        55434,
+        3820,
+        6332,
+        12324,
+        40741,
+        14,
+        30522,
+        14,
+        469,
+        117562,
+        588,
+        366,
+        6803,
+        25603,
+        304,
+        45501,
+        57918,
+        116784,
+        603,
+        87965,
+        305,
+        77706,
+        6255,
+        11663,
+        201,
+        10701,
+        8447,
+        294,
+        83452,
+        14,
+        4861,
+        6025,
+        79588,
+        14,
+        1801,
+        12218,
+        1082,
+        3843,
+        2188,
+        38711,
+        14,
+        3490,
+        436,
+        21185,
+        304,
+        4575,
+        855,
+        260,
+        4135,
+        344,
+        3946,
+        16,
+        5183,
+        10977,
+        2118,
+        915,
+        21018,
+        295,
+        32096,
+        477,
+        8368,
+        362,
+        11135,
+        8447,
+        603,
+        64430,
+        990,
+        12406,
+        1664,
+        3645,
+        3920,
+        344,
+        30331,
+        724,
+        791,
+        36762,
+        1114,
+        4138,
+        3851,
+        294,
+        12007,
+        5939,
+        1561,
+        83452,
+        14,
+        1438,
+        412,
+        61380,
+        55434,
+        1918,
+        16,
+        4895,
+        4570,
+        477,
+        4844,
+        16,
+        660,
+        270,
+        45721,
+        14,
+        436,
+        1142,
+        366,
+        2993,
+        304,
+        3394,
+        32096,
+        13075,
+        295,
+        782,
+        8611,
+        6240,
+        294,
+        5177,
+        36762,
+        16,
+        28110,
+        304,
+        782,
+        10817,
+        304,
+        1762,
+        778,
+        344,
+        2455,
+        362,
+        440,
+        16,
+        330,
+        2154,
+        87,
+        14,
+        32096,
+        33554,
+        14,
+        2059,
+        559,
+        74,
+        14,
+        305,
+        32096,
+        12053,
+        477,
+        1894,
+        7236,
+        294,
+        32096,
+        603,
+        2991,
+        418,
+        32096,
+        14,
+        10977,
+        12406,
+        1664,
+        3645,
+        3920,
+        98458,
+        52659,
+        36762,
+        1114,
+        990,
+        4138,
+        3851,
+        294,
+        12007,
+        5939,
+        1561,
+        83452,
+        14,
+        1438,
+        412,
+        61380,
+        55434,
+        1918,
+        16,
+        4895,
+        4570,
+        477,
+        4844,
+        16,
+        660,
+        270,
+        45721,
+        14,
+        436,
+        1142,
+        366,
+        2993,
+        304,
+        3394,
+        98458,
+        52659,
+        295,
+        782,
+        8611,
+        6240,
+        294,
+        5177,
+        36762,
+        16,
+        28110,
+        304,
+        782,
+        10817,
+        304,
+        1762,
+        778,
+        344,
+        2455,
+        362,
+        440,
+        16,
+        3718,
+        1325,
+        52659,
+        990,
+        7272,
+        21138,
+        2800,
+        261,
+        21795,
+        297,
+        6301,
+        14,
+        778,
+        270,
+        3197,
+        44931,
+        1055,
+        54235,
+        15,
+        21,
+        25578,
+        16733,
+        16,
+        3109,
+        25578,
+        16733,
+        477,
+        1894,
+        362,
+        4082,
+        2195,
+        603,
+        2107,
+        588,
+        366,
+        3375,
+        304,
+        1178,
+        4128,
+        19993,
+        1407,
+        782,
+        8611,
+        14,
+        832,
+        440,
+        1142,
+        1309,
+        304,
+        2188,
+        260,
+        17896,
+        16,
+        983,
+        344,
+        2239,
+        304,
+        1733,
+        270,
+        8715,
+        304,
+        1762,
+        1192,
+        1949,
+        68835,
+        19993,
+        260,
+        17896,
+        7272,
+        343,
+        9336,
+        734,
+        1192,
+        1949,
+        19993,
+        782,
+        3197,
+        588,
+        5300,
+        1347,
+        797,
+        2454,
+        477,
+        3586,
+        14021,
+        294,
+        19993,
+        36762,
+        16,
+        68356,
+        119541,
+        6179,
+        304,
+        366,
+        850,
+        7186,
+        31713,
+        513,
+        270,
+        3197,
+        14,
+        790,
+        436,
+        769,
+        2477,
+        68835,
+        19993,
+        1099,
+        19993,
+        48217,
+        16,
+        68356,
+        48217,
+        14,
+        4499,
+        14,
+        4086,
+        411,
+        6301,
+        3431,
+        304,
+        366,
+        31713,
+        14,
+        832,
+        344,
+        2455,
+        4737,
+        418,
+        260,
+        9220,
+        294,
+        21943,
+        22808,
+        603,
+        4484,
+        5036,
+        778,
+        1179,
+        294,
+        19993,
+        440,
+        2188,
+        14,
+        436,
+        734,
+        2993,
+        304,
+        13633,
+        782,
+        27905,
+        6443,
+        270,
+        2173,
+        832,
+        396,
+        440,
+        477,
+        554,
+        5177,
+        850,
+        1099,
+        223,
+        3712,
+        7797,
+        509,
+        260,
+        1014,
+        16,
+        2042,
+        1142,
+        611,
+        304,
+        2188,
+        850,
+        1099,
+        834,
+        14799,
+        412,
+        260,
+        1529,
+        16,
+        20534,
+        396,
+        440,
+        1142,
+        990,
+        1178,
+        1093,
+        19993,
+        1407,
+        782,
+        8611,
+        14,
+        305,
+        1895,
+        396,
+        1055,
+        782,
+        3452,
+        362,
+        270,
+        2173,
+        603,
+        671,
+        1347,
+        294,
+        38515,
+        344,
+        260,
+        1014,
+        47871,
+        4090,
+        304,
+        38200,
+        270,
+        3197,
+        305,
+        23476,
+        4745,
+        16,
+        5334,
+        2382,
+        14,
+        4499,
+        14,
+        588,
+        14930,
+        3317,
+        4680,
+        305,
+        588,
+        12982,
+        418,
+        915,
+        38515,
+        14,
+        36762,
+        14,
+        469,
+        21461,
+        16,
+        1884,
+        1305,
+        8218,
+        14,
+        440,
+        1531,
+        2188,
+        38515,
+        418,
+        2312,
+        14,
+        1202,
+        270,
+        30062,
+        294,
+        260,
+        2195,
+        2312,
+        47532,
+        16,
+        107531,
+        1505,
+        304,
+        45501,
+        83452,
+        8447,
+        9757,
+        295,
+        786,
+        15385,
+        538,
+        6626,
+        304,
+        6626,
+        16,
+        1924,
+        418,
+        32503,
+        18836,
+        4737,
+        304,
+        45501,
+        83452,
+        8447,
+        14,
+        1093,
+        3843,
+        1142,
+        1783,
+        14254,
+        418,
+        28141,
+        33458,
+        1901,
+        3628,
+        1142,
+        554,
+        603,
+        671,
+        2502,
+        38515,
+        477,
+        990,
+        7097,
+        10957,
+        304,
+        45501,
+        8447,
+        1345,
+        412,
+        6025,
+        79588,
+        14,
+        57918,
+        116784,
+        14,
+        305,
+        17628,
+        68399,
+        14,
+        7254,
+        4933,
+        344,
+        11168,
+        469,
+        29719,
+        16,
+        12554,
+        32096,
+        14,
+        851,
+        1142,
+        6242,
+        4514,
+        2951,
+        1420,
+        95478,
+        343,
+        25663,
+        1495,
+        416,
+        95478,
+        11,
+        396,
+        1494,
+        1214,
+        1277,
+        55434,
+        295,
+        270,
+        3197,
+        305,
+        14016,
+        11493,
+        270,
+        3851,
+        294,
+        12007,
+        5939,
+        16,
+        28110,
+        304,
+        782,
+        10817,
+        1936,
+        5177,
+        1305,
+        38515,
+        1137,
+        1124,
+        270,
+        3669,
+        14,
+        8235,
+        383,
+        356,
+        343,
+        38,
+        4091,
+        23982,
+        67,
+        9043,
+        9141,
+        11,
+        769,
+        7097,
+        1047,
+        9475,
+        412,
+        260,
+        4135,
+        362,
+        2420,
+        73775,
+        8447,
+        14,
+        1754,
+        34095,
+        2622,
+        92157,
+        881,
+        35080,
+        538,
+        8235,
+        383,
+        356,
+        295,
+        270,
+        223,
+        2722,
+        18,
+        85,
+        16,
+        3211,
+        14,
+        4570,
+        1801,
+        396,
+        270,
+        3197,
+        344,
+        554,
+        3495,
+        304,
+        4321,
+        8235,
+        383,
+        356,
+        1055,
+        92157,
+        14,
+        832,
+        436,
+        344,
+        554,
+        4888,
+        304,
+        1694,
+        45501,
+        1117,
+        8447,
+        294,
+        83452,
+        603,
+        36898,
+        4570,
+        1801,
+        88483,
+        1142,
+        1694,
+        1623,
+        3843,
+        418,
+        8447,
+        294,
+        83452,
+        14,
+        6440,
+        418,
+        6025,
+        79588,
+        305,
+        17628,
+        4123,
+        603,
+        15068,
+        2767,
+        4570,
+        611,
+        16058,
+        270,
+        15385,
+        294,
+        3549,
+        2680,
+        73616,
+        33458,
+        14,
+        6687,
+        2680,
+        25078,
+        85,
+        1142,
+        2255,
+        270,
+        2502,
+        44013,
+        362,
+        270,
+        4135,
+        294,
+        83452,
+        2951,
+        377,
+        786,
+        4134,
+        305,
+        3217,
+        16,
+        14875,
+        82075,
+        260,
+        38239,
+        14,
+        2680,
+        25078,
+        85,
+        2188,
+        1055,
+        3513,
+        260,
+        1856,
+        734,
+        26225,
+        2613,
+        16,
+        334,
+        26225,
+        2613,
+        344,
+        6428,
+        412,
+        260,
+        1856,
+        734,
+        5131,
+        14,
+        10613,
+        14,
+        305,
+        16345,
+        42824,
+        16,
+        1417,
+        10552,
+        2680,
+        25078,
+        86961,
+        710,
+        294,
+        1305,
+        3687,
+        1082,
+        14741,
+        270,
+        1473,
+        6798,
+        4135,
+        362,
+        1660,
+        2811,
+        603,
+        10701,
+        4570,
+        3797,
+        396,
+        3607,
+        304,
+        12460,
+        270,
+        3197,
+        343,
+        16176,
+        6470,
+        14,
+        5212,
+        20235,
+        11,
+        1142,
+        6252,
+        270,
+        14899,
+        294,
+        6025,
+        79588,
+        603,
+        2497,
+        6237,
+        4805,
+        305,
+        108082,
+        201,
+        2991,
+        55434,
+        4563,
+        15861,
+        2184,
+        83452,
+        14,
+        260,
+        6626,
+        734,
+        3851,
+        294,
+        8357,
+        270,
+        2502,
+        4132,
+        1142,
+        4138,
+        1137,
+        37682,
+        59434,
+        344,
+        892,
+        294,
+        270,
+        4095,
+        25255,
+        1699,
+        362,
+        710,
+        3843,
+        16,
+        2454,
+        477,
+        1623,
+        33458,
+        3510,
+        14,
+        4499,
+        14,
+        304,
+        1694,
+        18091,
+        270,
+        8447,
+        305,
+        6252,
+        2195,
+        11027,
+        4515,
+        418,
+        83452,
+        16,
+        99396,
+        270,
+        1748,
+        4135,
+        362,
+        440,
+        588,
+        21896,
+        32215,
+        305,
+        29800,
+        270,
+        7765,
+        362,
+        260,
+        11797,
+        14,
+        8112,
+        14,
+        32105,
+        1988,
+        2184,
+        305,
+        1561,
+        83452,
+        603,
+        3509,
+        6526,
+        14026,
+        37026,
+        447,
+        14,
+        7320,
+        85225,
+        327,
+        16,
+        58067,
+        5805,
+        4123,
+        2184,
+        83452,
+        16,
+        373,
+        4616,
+        11873,
+        16,
+        223,
+        792,
+        27,
+        29,
+        4774,
+        10,
+        20,
+        2605,
+        9107,
+        15,
+        25,
+        603,
+        36,
+        2238,
+        448,
+        14,
+        10134,
+        803,
+        73394,
+        11547,
+        265,
+        1653,
+        755,
+        38557,
+        59907,
+        14,
+        14616,
+        327,
+        14,
+        55561,
+        327,
+        14,
+        20665,
+        552,
+        14,
+        17902,
+        1653,
+        14,
+        1231,
+        423,
+        16,
+        95544,
+        305,
+        13622,
+        2178,
+        294,
+        260,
+        46486,
+        2499,
+        10027,
+        411,
+        344,
+        61695,
+        276,
+        6815,
+        5159,
+        1218,
+        74,
+        15708,
+        10532,
+        295,
+        7891,
+        3843,
+        418,
+        2420,
+        73775,
+        8447,
+        28,
+        260,
+        29963,
+        14,
+        6042,
+        18054,
+        14,
+        11550,
+        43449,
+        2286,
+        14842,
+        53395,
+        98767,
+        16,
+        373,
+        4616,
+        11873,
+        16,
+        223,
+        792,
+        25,
+        14722,
+        223,
+        397,
+        29,
+        3175,
+        10,
+        19,
+        2605,
+        2181,
+        15,
+        3286,
+        603,
+        36,
+        2939,
+        71,
+        721,
+        14,
+        774,
+        4422,
+        59020,
+        628,
+        14,
+        23846,
+        71,
+        373,
+        14,
+        10134,
+        803,
+        73394,
+        11547,
+        265,
+        1653,
+        755,
+        38557,
+        59907,
+        16,
+        7479,
+        1218,
+        74,
+        15708,
+        418,
+        469,
+        2503,
+        774,
+        16,
+        3880,
+        734,
+        123571,
+        362,
+        37175,
+        16224,
+        18181,
+        2182,
+        297,
+        4135,
+        3820,
+        3083,
+        294,
+        260,
+        3226,
+        16057,
+        14,
+        12092,
+        14,
+        50323,
+        2286,
+        16,
+        373,
+        4616,
+        11873,
+        16,
+        223,
+        792,
+        25,
+        4639,
+        223,
+        397,
+        29,
+        3351,
+        10,
+        22,
+        2605,
+        19839,
+        15,
+        929,
+        603,
+        2425,
+        458,
+        1431,
+        288,
+        721,
+        14,
+        47078,
+        463,
+        16370,
+        373,
+        16,
+        95544,
+        10870,
+        294,
+        380,
+        8988,
+        9224,
+        13611,
+        41814,
+        343,
+        9381,
+        11,
+        2765,
+        96863,
+        1630,
+        560,
+        55434,
+        343,
+        37,
+        6729,
+        11,
+        418,
+        17,
+        38889,
+        1685,
+        307,
+        9897,
+        1297,
+        75689,
+        52991,
+        343,
+        6806,
+        35,
+        11,
+        295,
+        270,
+        4135,
+        294,
+        18181,
+        2182,
+        297,
+        8447,
+        295,
+        716,
+        19975,
+        73775,
+        3843,
+        28,
+        7110,
+        9698,
+        2286,
+        16,
+        552,
+        3238,
+        78144,
+        40569,
+        16,
+        223,
+        792,
+        25,
+        14722,
+        29,
+        2225,
+        10,
+        27,
+        2605,
+        10925,
+        18,
+        15,
+        24,
+        603,
+        25228,
+        552,
+        14,
+        9974,
+        5440,
+        334,
+        14,
+        660,
+        1816,
+        552,
+        14,
+        14510,
+        4989,
+        327,
+        14,
+        87530,
+        433,
+        16,
+        19955,
+        294,
+        2420,
+        73775,
+        8447,
+        513,
+        20189,
+        53506,
+        30520,
+        28,
+        260,
+        26775,
+        14,
+        29963,
+        12092,
+        8307,
+        16,
+        64755,
+        9689,
+        16,
+        223,
+        792,
+        25,
+        4466,
+        29,
+        1173,
+        10,
+        23,
+        2605,
+        24496,
+        15,
+        6048,
+        603,
+        42,
+        23117,
+        2710,
+        17691,
+        14,
+        345,
+        1462,
+        337,
+        5016,
+        17169,
+        14,
+        6784,
+        6065,
+        81,
+        471,
+        14,
+        1231,
+        423,
+        16,
+        455,
+        1788,
+        294,
+        4332,
+        667,
+        3063,
+        344,
+        30331,
+        724,
+        3408,
+        377,
+        2420,
+        73775,
+        8447,
+        14,
+        57853,
+        305,
+        57918,
+        14672,
+        1677,
+        295,
+        2420,
+        73775,
+        3843,
+        28,
+        334,
+        29963,
+        14,
+        6042,
+        78900,
+        14,
+        38711,
+        43449,
+        2286,
+        16,
+        94285,
+        83273,
+        16,
+        223,
+        792,
+        23,
+        29,
+        1602,
+        28,
+        17153,
+        15,
+        16326,
+        603,
+        42,
+        48687,
+        33691,
+        14,
+        118826,
+        894,
+        14,
+        13857,
+        406,
+        14,
+        1231,
+        423,
+        16,
+        334,
+        29963,
+        12092,
+        20081,
+        2286,
+        294,
+        88483,
+        362,
+        2411,
+        107959,
+        6025,
+        79588,
+        28,
+        1788,
+        377,
+        92182,
+        6025,
+        79588,
+        305,
+        7053,
+        4474,
+        16,
+        77298,
+        30220,
+        321,
+        16,
+        223,
+        792,
+        24,
+        29,
+        5926,
+        28,
+        8996,
+        15,
+        23888,
+        603,
+        45,
+        266,
+        10637,
+        2030,
+        12496,
+        11170,
+        387,
+        327,
+        14,
+        28496,
+        462,
+        14,
+        19460,
+        4580,
+        2248,
+        11617,
+        14,
+        1231,
+        423,
+        16,
+        19096,
+        294,
+        32096,
+        6886,
+        10027,
+        344,
+        30331,
+        724,
+        3408,
+        377,
+        13707,
+        2019,
+        14,
+        12188,
+        19504,
+        9677,
+        14,
+        305,
+        16715,
+        57853,
+        295,
+        2411,
+        107959,
+        3843,
+        16,
+        84810,
+        16,
+        223,
+        792,
+        22,
+        29,
+        18307,
+        28,
+        2738,
+        15,
+        6048,
+        603,
+        36784,
+        25774,
+        330,
+        16,
+        12651,
+        59434,
+        28,
+        260,
+        3862,
+        294,
+        96722,
+        25209,
+        36762,
+        16,
+        455,
+        3707,
+        5726,
+        294,
+        13369,
+        16,
+        223,
+        792,
+        23,
+        29,
+        8610,
+        10,
+        736,
+        11,
+        90822,
+        223,
+        20,
+        603,
+        46,
+        48353,
+        373,
+        14,
+        3210,
+        38557,
+        489,
+        14,
+        9979,
+        6358,
+        345,
+        14,
+        1231,
+        423,
+        16,
+        21424,
+        294,
+        65671,
+        5457,
+        17204,
+        630,
+        13684,
+        22742,
+        297,
+        6301,
+        54235,
+        15,
+        21,
+        25578,
+        6301,
+        61416,
+        377,
+        6025,
+        79588,
+        305,
+        4474,
+        294,
+        1988,
+        3611,
+        7484,
+        59994,
+        3843,
+        28,
+        260,
+        6042,
+        78900,
+        14,
+        38711,
+        43449,
+        14,
+        29963,
+        7452,
+        8307,
+        16,
+        12651,
+        59434,
+        16,
+        223,
+        792,
+        27,
+        29,
+        926,
+        28,
+        20530,
+        15,
+        2574,
+        603,
+        46,
+        1111,
+        657,
+        16,
+        12651,
+        59434,
+        305,
+        270,
+        12651,
+        73775,
+        43088,
+        16,
+        11338,
+        15727,
+        1219,
+        294,
+        5264,
+        5992,
+        16,
+        223,
+        792,
+        26,
+        29,
+        6152,
+        10,
+        23,
+        5772,
+        49,
+        11352,
+        387,
+        433,
+        14,
+        23846,
+        71,
+        373,
+        14,
+        36364,
+        2438,
+        446,
+        14,
+        1231,
+        423,
+        16,
+        95544,
+        305,
+        6555,
+        294,
+        344,
+        61695,
+        276,
+        6815,
+        5159,
+        1218,
+        74,
+        15708,
+        10532,
+        362,
+        18181,
+        2182,
+        297,
+        8447,
+        16,
+        93843,
+        94285,
+        16,
+        223,
+        792,
+        23,
+        29,
+        6992,
+        28,
+        7792,
+        22,
+        15,
+        6924,
+        21,
+        603,
+        50,
+        1355,
+        1587,
+        35652,
+        14,
+        96054,
+        103594,
+        14,
+        462,
+        476,
+        15535,
+        6030,
+        15449,
+        14,
+        1231,
+        423,
+        16,
+        25280,
+        9698,
+        23913,
+        108185,
+        655,
+        14,
+        98214,
+        14,
+        15330,
+        3541,
+        15,
+        3901,
+        15669,
+        345,
+        60191,
+        34726,
+        294,
+        7479,
+        80087,
+        15708,
+        295,
+        270,
+        7967,
+        294,
+        14746,
+        3718,
+        25284,
+        28,
+        18604,
+        107907,
+        34726,
+        471,
+        2616,
+        5564,
+        19,
+        16,
+        552,
+        15727,
+        50142,
+        16,
+        223,
+        792,
+        24,
+        29,
+        1173,
+        28,
+        17872,
+        24,
+        15,
+        18460,
+        19,
+        603,
+        50,
+        5703,
+        9750,
+        35911,
+        14,
+        373,
+        1446,
+        276,
+        63691,
+        14,
+        76815,
+        2599,
+        23184,
+        16,
+        9266,
+        62284,
+        7329,
+        117219,
+        25578,
+        16733,
+        305,
+        270,
+        14152,
+        294,
+        12188,
+        22113,
+        16,
+        8699,
+        18142,
+        3238,
+        343,
+        12553,
+        8471,
+        797,
+        223,
+        792,
+        25,
+        4276,
+        29,
+        14466,
+        10,
+        553,
+        2605,
+        8870,
+        23,
+        15,
+        2851,
+        16,
+        6820,
+        603,
+        6689,
+        1665,
+        75,
+        327,
+        14,
+        27182,
+        24881,
+        14,
+        4276,
+        347,
+        2928,
+        63691,
+        14,
+        77206,
+        42621,
+        14,
+        47885,
+        69525,
+        1285,
+        17691,
+        14,
+        23461,
+        8762,
+        14,
+        1231,
+        423,
+        16,
+        63036,
+        10520,
+        294,
+        98458,
+        52659,
+        362,
+        270,
+        4227,
+        294,
+        6025,
+        79588,
+        16,
+        552,
+        15582,
+        14362,
+        84,
+        50142,
+        16,
+        223,
+        792,
+        25,
+        22771,
+        29,
+        23,
+        10,
+        21,
+        2605,
+        7336,
+        15,
+        736,
+        603,
+        52,
+        6498,
+        327,
+        16,
+        455,
+        6004,
+        43597,
+        14635,
+        294,
+        83452,
+        16,
+        7711,
+        552,
+        22562,
+        16,
+        223,
+        792,
+        27,
+        29,
+        864,
+        10,
+        24,
+        2605,
+        17565,
+        15,
+        21,
+        603,
+        35936,
+        1594,
+        657,
+        14,
+        25746,
+        3272,
+        95990,
+        14,
+        92792,
+        43335,
+        420,
+        16,
+        36992,
+        420,
+        19759,
+        305,
+        12007,
+        5939,
+        3851,
+        295,
+        2411,
+        107959,
+        3843,
+        28,
+        270,
+        29576,
+        12151,
+        734,
+        4842,
+        10096,
+        16,
+        14658,
+        45945,
+        11155,
+        16,
+        223,
+        792,
+        25,
+        14722,
+        29,
+        864,
+        10,
+        25,
+        2605,
+        28409,
+        15,
+        6078,
+        603,
+        4374,
+        87745,
+        489,
+        14,
+        899,
+        500,
+        11540,
+        44414,
+        14,
+        3167,
+        9253,
+        75,
+        334,
+        14,
+        1231,
+        423,
+        16,
+        75997,
+        344,
+        30331,
+        724,
+        3408,
+        305,
+        113695,
+        362,
+        270,
+        14254,
+        294,
+        18181,
+        2182,
+        297,
+        8447,
+        28,
+        260,
+        107205,
+        14,
+        6042,
+        78900,
+        14,
+        29963,
+        2286,
+        16,
+        373,
+        4616,
+        11873,
+        16,
+        223,
+        792,
+        22,
+        29,
+        2491,
+        28,
+        779,
+        15,
+        397,
+        603,
+        98855,
+        20032,
+        420,
+        14,
+        16589,
+        967,
+        446,
+        14,
+        721,
+        10237,
+        552,
+        14,
+        1231,
+        423,
+        16,
+        6332,
+        12324,
+        11727,
+        4577,
+        294,
+        6610,
+        272,
+        722,
+        4798,
+        47077,
+        28,
+        1940,
+        425,
+        2379,
+        261,
+        305,
+        1940,
+        425,
+        32426,
+        14,
+        295,
+        26862,
+        19638,
+        295,
+        23222,
+        305,
+        295,
+        24505,
+        16,
+        552,
+        30220,
+        1797,
+        45657,
+        16023,
+        18142,
+        16,
+        223,
+        792,
+        22,
+        29,
+        6207,
+        28,
+        10594,
+        15,
+        10081,
+        603,
+        9223,
+        555,
+        3272,
+        462,
+        14,
+        26037,
+        55489,
+        14,
+        19460,
+        463,
+        1007,
+        36302,
+        14,
+        41763,
+        16578,
+        14,
+        105885,
+        9703,
+        657,
+        14,
+        4176,
+        102259,
+        1423,
+        48004,
+        16,
+        455,
+        4680,
+        294,
+        5159,
+        1218,
+        74,
+        15708,
+        33458,
+        377,
+        57853,
+        14,
+        91552,
+        5624,
+        14,
+        19571,
+        305,
+        22777,
+        16,
+        373,
+        4616,
+        11873,
+        16,
+        223,
+        792,
+        25,
+        10933,
+        223,
+        397,
+        29,
+        3351,
+        10,
+        20,
+        2605,
+        3298,
+        15,
+        11154,
+        603,
+        55,
+        95816,
+        74,
+        714,
+        433,
+        14,
+        2751,
+        2020,
+        1336,
+        116318,
+        14,
+        5570,
+        31642,
+        395,
+        98231,
+        14,
+        1231,
+        423,
+        16,
+        7479,
+        1218,
+        74,
+        15708,
+        305,
+        774,
+        16,
+        3880,
+        734,
+        123571,
+        362,
+        18181,
+        2182,
+        297,
+        30443,
+        28,
+        260,
+        29963,
+        8307,
+        16,
+        93843,
+        94285,
+        16,
+        223,
+        792,
+        24,
+        29,
+        7792,
+        28,
+        15971,
+        15,
+        9489,
+        603,
+        57,
+        12953,
+        2438,
+        448,
+        14,
+        552,
+        19879,
+        437,
+        14,
+        1583,
+        109705,
+        16085,
+        12040,
+        12953,
+        2438,
+        420,
+        16,
+        69346,
+        1668,
+        724,
+        3408,
+        3820,
+        7681,
+        3344,
+        61953,
+        469,
+        14681,
+        11368,
+        33,
+        15326,
+        288,
+        2127,
+        7619,
+        16,
+        223,
+        792,
+        25,
+        4639,
+        29,
+        24,
+        10,
+        20,
+        2605,
+        4980,
+        15,
+        2851,
+        603,
+        81441,
+        54156,
+        14,
+        462,
+        23144,
+        44410,
+        14,
+        43135,
+        27432,
+        14,
+        1231,
+        423,
+        16,
+        334,
+        80648,
+        14,
+        6042,
+        78900,
+        14,
+        38711,
+        43449,
+        8307,
+        377,
+        270,
+        1788,
+        294,
+        19950,
+        16333,
+        5624,
+        331,
+        377,
+        270,
+        18181,
+        2182,
+        297,
+        14968,
+        295,
+        42485,
+        100606,
+        73775,
+        3843,
+        16,
+        34348,
+        93843,
+        94285,
+        99856,
+        16,
+        223,
+        792,
+        25,
+        29,
+        5926,
+        28,
+        9203,
+        15,
+        5361,
+        603,
+        60,
+        3264,
+        71440,
+        446,
+        14,
+        22431,
+        16633,
+        975,
+        9750,
+        552,
+        14,
+        16417,
+        6870,
+        327,
+        14,
+        1231,
+        423,
+        16,
+        21424,
+        294,
+        88483,
+        14,
+        6803,
+        29756,
+        14,
+        1420,
+        95478,
+        14,
+        305,
+        38711,
+        377,
+        6025,
+        1208,
+        29188,
+        295,
+        2411,
+        107959,
+        3843,
+        28,
+        411,
+        3499,
+        294,
+        1234,
+        26775,
+        14,
+        11550,
+        14,
+        29963,
+        4570,
+        16,
+        84086,
+        2182,
+        297,
+        16,
+        223,
+        792,
+        25,
+        29,
+        553,
+        28,
+        2080,
+        15,
+        1883,
+        603,
+        19417,
+        7222,
+        28,
+        223,
+        553,
+        17,
+        1449,
+        17,
+        643,
+        19,
+        201,
+        52731,
+        3715,
+        28,
+        34753,
+        420,
+        16,
+        105885,
+        9703,
+        14,
+        471,
+        16255,
+        14,
+        22069,
+        5108,
+        65073,
+        14,
+        260,
+        3729,
+        4646,
+        82535,
+        295,
+        28141,
+        305,
+        9235,
+        13677,
+        14,
+        41168,
+        14,
+        48004,
+        16,
+        6820,
+        4496,
+        513,
+        5948,
+        75,
+        5904,
+        32946,
+        12282,
+        603,
+        671,
+        1951,
+        4496,
+        23512,
+        1531,
+        554,
+        366,
+        1505,
+        2184,
+        1117,
+        5931,
+        13331,
+        469,
+        362,
+        270,
+        12463,
+        469,
+        4135,
+        294,
+        1117,
+        5931,
+        5664,
+        16,
+        334,
+        28931,
+        5931,
+        6687,
+        1531,
+        366,
+        63970,
+        362,
+        12463,
+        305,
+        4135,
+        294,
+        1117,
+        305,
+        710,
+        5931,
+        4132,
+        16,
+        14983,
+        223,
+        24465,
+        362,
+        710,
+        5931,
+        57697,
+        16,
+        35539,
+        304,
+        915,
+        8926,
+        477,
+        4496,
+        362,
+        1951,
+        1353,
+        3820,
+        851,
+        696,
+        554,
+        26640,
+        33546,
+        3860,
+        294,
+        1948,
+        915,
+        8926,
+        16,
+        13420,
+        223,
+        1357,
+        25,
+        15,
+        334,
+        5249,
+        4584,
+        4705,
+        1042,
+        7081,
+        16,
+        10649,
+        84280,
+        469,
+        6380,
+        294,
+        270,
+        1951,
+        14541,
+        23512,
+        344,
+        29851,
+        38340,
+        16,
+        1
+      ],
+      "seq_len": 2048
+    }
+  },
+  "config": {
+    "vocab_size": 129280,
+    "dim": 1536,
+    "n_layers": 16,
+    "n_heads": 12,
+    "head_dim": 128,
+    "rope_head_dim": 32,
+    "q_lora_rank": 384,
+    "o_groups": 4,
+    "o_lora_rank": 256,
+    "norm_eps": 1e-06,
+    "n_routed_experts": 12,
+    "n_shared_experts": 1,
+    "n_activated_experts": 2,
+    "moe_inter_dim": 1408,
+    "score_func": "softmax",
+    "route_scale": 1.0,
+    "swiglu_limit": 15.0,
+    "n_dense_layers": 2,
+    "load_balance_alpha": 0.0005,
+    "hc_mult": 3,
+    "hc_sinkhorn_iters": 8,
+    "hc_eps": 1e-06,
+    "window_size": 128,
+    "compress_ratio": 4,
+    "n_mtp_layers": 1,
+    "mtp_loss_weight": 0.02,
+    "rope_theta": 10000.0,
+    "rope_factor": 1.0,
+    "original_seq_len": 0,
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "ctx_phase_boundaries": [
+      60000,
+      80000,
+      100000
+    ],
+    "ctx_phase_lengths": [
+      2048,
+      4096,
+      8192
+    ],
+    "yarn_scale": 40.0,
+    "yarn_alpha": 1.0,
+    "yarn_beta": 32.0,
+    "yarn_original_seq_len": 4096,
+    "micro_batch_size": 1,
+    "gradient_accumulation_steps": 32,
+    "max_lr": 0.00015,
+    "min_lr": 1.5e-05,
+    "warmup_steps": 2000,
+    "total_steps": 200000,
+    "decay_steps": 180000,
+    "weight_decay": 0.1,
+    "muon_beta": 0.95,
+    "muon_ns_steps": 7,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.98,
+    "grad_clip_norm": 5.0,
+    "init_std": 0.006,
+    "dtype": "bfloat16",
+    "fineweb_config": "default",
+    "approx_tokens_per_fineweb_doc": 800,
+    "approx_tokens_per_code_doc": 1200,
+    "approx_tokens_per_math_doc": 1500,
+    "data_mix_probs": [
+      0.6,
+      0.25,
+      0.15
+    ],
+    "save_every_steps": 400,
+    "log_every_steps": 10,
+    "push_every_steps": 400,
+    "checkpoint_dir": "/kaggle/working/zenyxv3checkpoints"
+  }
+}

checkpoints/step_22800/opt_state.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e286d637039d755d3cf16c76e2d54375774f5587e71b2c3abb0a282c37c9ef5c
+size 3154940698

checkpoints/step_22800/params.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af76049f58a4b0179664f79209f2871b30b4d47b454d677ea50ba2b389df991f
+size 3154768094