step 22400

checkpoints/step_22400/metadata.json

@@ -0,0 +1,4516 @@
+{
+  "step": 22400,
+  "tokens_consumed": 11744051200,
+  "sequences_consumed": 5734400,
+  "data_state": {
+    "stream_state": {
+      "rng_state": [
+        "MT19937",
+        [
+          3978216887,
+          1397878111,
+          1460362796,
+          973466703,
+          3851306789,
+          1968878021,
+          1543771552,
+          4054142582,
+          2590528940,
+          3700292295,
+          1055494947,
+          664712602,
+          514972534,
+          936371035,
+          2679385723,
+          341604897,
+          284564401,
+          3962927679,
+          2683260112,
+          3323627625,
+          2752706515,
+          151025079,
+          1749605091,
+          1220675538,
+          1758363906,
+          3405281460,
+          1232490591,
+          3389437953,
+          2547971335,
+          3034692099,
+          328981006,
+          120644013,
+          3906003552,
+          4216779995,
+          4147841953,
+          1049553749,
+          1699128718,
+          1382738855,
+          3038989172,
+          447048588,
+          2940958001,
+          3146618256,
+          2085308502,
+          3590614269,
+          2449989409,
+          2230819550,
+          4083701693,
+          4261072520,
+          2220319447,
+          1205958683,
+          917949787,
+          2097994315,
+          2080476808,
+          3705525683,
+          2462924631,
+          114628381,
+          1163868602,
+          3373414345,
+          307007561,
+          1309390989,
+          2700566170,
+          2776472855,
+          4147535286,
+          1212036604,
+          337287218,
+          4045911034,
+          917370201,
+          1759915562,
+          1121295883,
+          687171868,
+          1538029245,
+          3569892322,
+          589931077,
+          383480531,
+          2381653118,
+          1497769296,
+          4127417275,
+          1677746729,
+          4090250399,
+          2323063300,
+          392522039,
+          1613500335,
+          602952925,
+          4048462897,
+          633883105,
+          454033469,
+          1592749038,
+          1862471449,
+          160078959,
+          3525968475,
+          2181338309,
+          3829239889,
+          367284838,
+          3694300058,
+          2245098434,
+          2261599235,
+          2868002981,
+          2999785037,
+          2776635218,
+          911602117,
+          3537098783,
+          3640493345,
+          1319968466,
+          2438743586,
+          1846415959,
+          3118471739,
+          2603437905,
+          1886108512,
+          3557406607,
+          1632755445,
+          2269926877,
+          3823495730,
+          841395053,
+          730133654,
+          2854796357,
+          566077381,
+          1673907539,
+          124488808,
+          1381828511,
+          1390199257,
+          735523254,
+          2314477594,
+          4028902603,
+          1717938009,
+          626589270,
+          1248754146,
+          139525435,
+          1356855695,
+          49813525,
+          371417431,
+          4074983267,
+          4190233219,
+          1216286022,
+          1534719901,
+          3247424327,
+          1454527139,
+          1703302787,
+          2316162319,
+          1162185491,
+          2924111311,
+          2247246622,
+          2561362844,
+          4232776366,
+          4086809925,
+          3651589078,
+          2886132140,
+          2883319888,
+          3194410694,
+          3899633864,
+          507078465,
+          1296981964,
+          153307639,
+          25730897,
+          2581162512,
+          2219956308,
+          712324283,
+          704912231,
+          752440219,
+          2634671164,
+          203895177,
+          2348471336,
+          1690708533,
+          2954307060,
+          3209097827,
+          2066005043,
+          4134124678,
+          2310922012,
+          443693542,
+          3241932632,
+          685035347,
+          2477887534,
+          2461751504,
+          55326755,
+          542120417,
+          2197952548,
+          3345294728,
+          3355435956,
+          2555751241,
+          2685754195,
+          1312199726,
+          962820368,
+          289123341,
+          2868721794,
+          2291230528,
+          2509416043,
+          1050342667,
+          4007799606,
+          709286743,
+          4239687458,
+          3528051760,
+          4047440242,
+          915909825,
+          4269504045,
+          3443237312,
+          490903982,
+          2550991453,
+          545507869,
+          4064714051,
+          603123340,
+          1664153009,
+          1716890097,
+          3450068621,
+          3665467635,
+          634426232,
+          3238166391,
+          3774448910,
+          3387623197,
+          1480341141,
+          1305809969,
+          398270139,
+          4198219918,
+          1282727855,
+          2051158130,
+          617577140,
+          1104968501,
+          3146609650,
+          3487453296,
+          13935315,
+          4238957575,
+          2224211577,
+          3242927681,
+          3981894154,
+          3984177368,
+          3717650350,
+          3149268931,
+          2675435461,
+          3315175713,
+          3411824871,
+          1186771208,
+          1630690289,
+          4042551786,
+          824934250,
+          1648881106,
+          2813979000,
+          3078691007,
+          3811183099,
+          163098485,
+          649660259,
+          3554336252,
+          3295490077,
+          2092988999,
+          3817833236,
+          816138906,
+          1219811140,
+          1284359021,
+          1667299267,
+          3761758082,
+          1456823382,
+          945423141,
+          2269038130,
+          2980169601,
+          858365037,
+          2432818176,
+          2865094763,
+          234619259,
+          1121054194,
+          3088137271,
+          3490785469,
+          3845023112,
+          799396885,
+          795425983,
+          2593110316,
+          3020841575,
+          3552064016,
+          1395849845,
+          2670292171,
+          2086028404,
+          2501117284,
+          1703406468,
+          1215269041,
+          1875165786,
+          3290529985,
+          1014275253,
+          1662031722,
+          3631078713,
+          1209718260,
+          2019957989,
+          1795251317,
+          1053158003,
+          2648940914,
+          2882165354,
+          2259167322,
+          3075761001,
+          1391841466,
+          2584731690,
+          2095042304,
+          3001297454,
+          2098681358,
+          2352189257,
+          3363797004,
+          3962731119,
+          1682138315,
+          1159522063,
+          1613184180,
+          530717148,
+          2404149959,
+          380494983,
+          113330722,
+          3671980810,
+          3604016024,
+          787313482,
+          2355674821,
+          3370463724,
+          3641961187,
+          36844194,
+          1841410076,
+          2452382152,
+          1069632663,
+          3743249367,
+          685622471,
+          721961766,
+          1317397007,
+          3347888877,
+          1911678920,
+          4169107445,
+          3284658704,
+          4234183960,
+          481510393,
+          4117302670,
+          963493678,
+          2672369412,
+          754174526,
+          2941204014,
+          3930140309,
+          1521637472,
+          726603790,
+          3631199637,
+          933809905,
+          2758056097,
+          3921648764,
+          1953249625,
+          2881546134,
+          802755409,
+          3086740452,
+          1871086908,
+          458763192,
+          3927181178,
+          695942424,
+          1106998472,
+          3995711383,
+          3433143164,
+          2089073811,
+          4275829319,
+          3059222341,
+          3131825905,
+          397116409,
+          2996520061,
+          2107416515,
+          3346766436,
+          2245037848,
+          3132637703,
+          3154620415,
+          3389238946,
+          3205046765,
+          19042416,
+          1155709737,
+          3982630084,
+          4268252583,
+          402961114,
+          309247755,
+          3508148482,
+          2897471840,
+          3660938396,
+          1406530543,
+          323018942,
+          1402019771,
+          2880282844,
+          1884482870,
+          2659640541,
+          2260678153,
+          3770506130,
+          1147124700,
+          2044675438,
+          2539152334,
+          1185102254,
+          70053905,
+          3842655585,
+          1908215750,
+          3125999552,
+          2284180463,
+          1924697208,
+          281753758,
+          3285244181,
+          4277333824,
+          2371196263,
+          4141841580,
+          1625374190,
+          1708268640,
+          3077773050,
+          3580059629,
+          3934865361,
+          1180613964,
+          3569558449,
+          1865666381,
+          607256160,
+          2734566057,
+          3778472394,
+          3498631575,
+          2358455705,
+          1832500264,
+          372462489,
+          438191097,
+          3054566488,
+          2425558098,
+          1576661113,
+          1107784798,
+          3957783704,
+          3861615401,
+          698015136,
+          4285801974,
+          1425897430,
+          3381066462,
+          331220947,
+          119276391,
+          579516463,
+          3762906814,
+          1590085948,
+          1949464466,
+          3539435041,
+          3405523589,
+          2368808881,
+          3847874273,
+          2234682152,
+          3184691880,
+          402088024,
+          432486514,
+          842561595,
+          2330672329,
+          157834787,
+          1081634723,
+          3353364851,
+          209016181,
+          1796228590,
+          3031566608,
+          2118307472,
+          1757659740,
+          1163695986,
+          3907212953,
+          2604051998,
+          1611609324,
+          3764225886,
+          3897009631,
+          3869925358,
+          737944116,
+          1857094551,
+          2511783245,
+          642846868,
+          3317600961,
+          279694912,
+          3489792826,
+          114317187,
+          1501206725,
+          3838563401,
+          2675242666,
+          1664852085,
+          1365314145,
+          3942550244,
+          2821019131,
+          2084747462,
+          3413361775,
+          1510332598,
+          3439181879,
+          1002150907,
+          2625211903,
+          128675513,
+          3069907690,
+          1869980202,
+          3701842934,
+          1152511715,
+          532452544,
+          1028616851,
+          460730730,
+          12751177,
+          3512818995,
+          4072957018,
+          1878988890,
+          2534596309,
+          1178741068,
+          1070201172,
+          3680020761,
+          257451188,
+          4209931042,
+          1199025549,
+          757134808,
+          4228917974,
+          2969130469,
+          1463379872,
+          1546886221,
+          3786536333,
+          99366509,
+          1392983640,
+          2785085578,
+          278488293,
+          3473606374,
+          2240274167,
+          3163502644,
+          1650880606,
+          2218861306,
+          2554907496,
+          3891170724,
+          58274141,
+          3640040285,
+          826802925,
+          1999483535,
+          293947812,
+          3595803533,
+          1407273755,
+          2394335190,
+          3238824878,
+          1716324628,
+          3873757136,
+          910868571,
+          2487786063,
+          4056580920,
+          231332933,
+          1420020724,
+          765160699,
+          3665607100,
+          3169733835,
+          478150110,
+          182621365,
+          1535462126,
+          1037897480,
+          1852739295,
+          1217998859,
+          932627940,
+          89642239,
+          499159713,
+          3930681761,
+          4013330101,
+          1220723349,
+          3578309197,
+          3340304603,
+          4201460072,
+          3607606660,
+          410440008,
+          1730546070,
+          2342994703,
+          2241980166,
+          2707089548,
+          3040444778,
+          1710502257,
+          1064257134,
+          421893150,
+          969947785,
+          1888869581,
+          2933943875,
+          2084804890,
+          3669524071,
+          2663465249,
+          3936007279,
+          2921091036,
+          1877060856,
+          1260264563,
+          2167306192,
+          3011983907,
+          1539479525,
+          3952427787,
+          2805204997,
+          1815518053,
+          4157429396,
+          3563664105,
+          3214946689,
+          3415256045,
+          3533222943,
+          1183431251,
+          1937265138,
+          627599663,
+          2734372449,
+          2831408958,
+          3882544721,
+          2284051334,
+          3386699222,
+          2656463329,
+          1452674353,
+          977640942,
+          1461064137,
+          68687819,
+          2748952900,
+          3407247644,
+          3425445549,
+          1651030750,
+          1781335438,
+          1406224795,
+          1622915664,
+          4252971459,
+          72343413,
+          2915102911,
+          1197498061,
+          292109520,
+          2283301193,
+          3361394660,
+          2379432724,
+          2973240440,
+          2974958801,
+          382148477,
+          1121844709,
+          2513086010,
+          2979181404,
+          3521345101,
+          919237737,
+          4157731403,
+          345803634,
+          101825703,
+          2663945239,
+          23925876,
+          3615793455,
+          3645850848,
+          3516235467,
+          1758040308,
+          1700632067,
+          1196864592,
+          4010895741,
+          1900399293,
+          2641044795,
+          3521342724,
+          3902835257,
+          207397483,
+          3641098167,
+          2788271526,
+          2442470062,
+          1996003992,
+          3619125041,
+          1320238030
+        ],
+        508,
+        0,
+        0.0
+      ],
+      "doc_counts": {
+        "fineweb": 2080040,
+        "code": 866362,
+        "math": 519860
+      },
+      "code_lang_idx": 70
+    },
+    "token_state": {
+      "buffer_tokens": [
+        5292,
+        32541,
+        38800,
+        759,
+        2188,
+        440,
+        362,
+        3080,
+        565,
+        790,
+        436,
+        734,
+        2413,
+        7001,
+        304,
+        3047,
+        6409,
+        4323,
+        14356,
+        412,
+        1585,
+        16,
+        9611,
+        1482,
+        477,
+        21602,
+        29,
+        440,
+        1142,
+        366,
+        13915,
+        890,
+        295,
+        260,
+        462,
+        2463,
+        469,
+        260,
+        33541,
+        349,
+        16,
+        343,
+        6203,
+        5292,
+        270,
+        7017,
+        21960,
+        671,
+        14139,
+        115884,
+        74,
+        22812,
+        39185,
+        343,
+        23041,
+        1505,
+        304,
+        84949,
+        4494,
+        278,
+        305,
+        28596,
+        5341,
+        333,
+        65098,
+        11,
+        344,
+        304,
+        4219,
+        377,
+        6224,
+        11843,
+        305,
+        1752,
+        509,
+        270,
+        6337,
+        418,
+        260,
+        8550,
+        1748,
+        7931,
+        5607,
+        3543,
+        538,
+        270,
+        3197,
+        16,
+        18668,
+        14,
+        362,
+        31905,
+        14,
+        440,
+        1142,
+        1258,
+        260,
+        37470,
+        75570,
+        8984,
+        294,
+        270,
+        2526,
+        1748,
+        1893,
+        603,
+        83829,
+        344,
+        270,
+        6805,
+        4063,
+        16,
+        21793,
+        270,
+        65608,
+        5407,
+        14,
+        11232,
+        14,
+        73399,
+        14,
+        31895,
+        14,
+        305,
+        56736,
+        477,
+        19441,
+        16,
+        11232,
+        344,
+        260,
+        5010,
+        8321,
+        294,
+        40258,
+        305,
+        344,
+        1473,
+        2915,
+        270,
+        4063,
+        294,
+        6749,
+        295,
+        270,
+        10828,
+        305,
+        16115,
+        294,
+        20717,
+        16,
+        983,
+        344,
+        7681,
+        304,
+        12803,
+        396,
+        25739,
+        1117,
+        40258,
+        759,
+        2572,
+        11232,
+        14,
+        4499,
+        14,
+        366,
+        2884,
+        396,
+        295,
+        270,
+        17739,
+        5949,
+        1482,
+        1142,
+        366,
+        49354,
+        304,
+        1694,
+        440,
+        855,
+        440,
+        7065,
+        11232,
+        14,
+        4204,
+        412,
+        260,
+        8479,
+        264,
+        105671,
+        3697,
+        759,
+        366,
+        850,
+        109914,
+        1099,
+        304,
+        11232,
+        17051,
+        16,
+        2483,
+        270,
+        915,
+        1893,
+        14,
+        295,
+        22646,
+        5949,
+        305,
+        4861,
+        119496,
+        14,
+        11232,
+        344,
+        270,
+        1473,
+        10908,
+        19441,
+        4063,
+        16,
+        660,
+        6672,
+        305,
+        22646,
+        892,
+        294,
+        270,
+        3924,
+        440,
+        1142,
+        990,
+        1783,
+        1482,
+        1812,
+        1305,
+        1234,
+        10555,
+        19649,
+        343,
+        821,
+        3252,
+        1432,
+        92,
+        11530,
+        77,
+        965,
+        115184,
+        294,
+        10555,
+        797,
+        983,
+        344,
+        990,
+        2953,
+        362,
+        1482,
+        304,
+        4449,
+        304,
+        3628,
+        295,
+        786,
+        12818,
+        4063,
+        14,
+        63922,
+        294,
+        270,
+        106937,
+        12913,
+        442,
+        85,
+        834,
+        14,
+        832,
+        260,
+        39428,
+        12987,
+        11232,
+        1142,
+        366,
+        23676,
+        304,
+        295,
+        40258,
+        305,
+        20664,
+        74999,
+        67,
+        16,
+        106714,
+        14,
+        270,
+        6102,
+        14,
+        25635,
+        1952,
+        10555,
+        14,
+        790,
+        11232,
+        344,
+        850,
+        10908,
+        1505,
+        16,
+        3016,
+        14,
+        40258,
+        344,
+        850,
+        11477,
+        1811,
+        295,
+        10252,
+        305,
+        10734,
+        20717,
+        14,
+        11232,
+        565,
+        295,
+        17834,
+        305,
+        16198,
+        5949,
+        294,
+        270,
+        3924,
+        603,
+        46874,
+        1482,
+        477,
+        850,
+        4888,
+        304,
+        7065,
+        260,
+        2961,
+        3947,
+        14,
+        412,
+        436,
+        344,
+        270,
+        1473,
+        11737,
+        13833,
+        8479,
+        4063,
+        295,
+        2662,
+        603,
+        3575,
+        440,
+        477,
+        24185,
+        304,
+        20717,
+        14,
+        3281,
+        4338,
+        6569,
+        40258,
+        469,
+        6569,
+        11232,
+        1936,
+        1893,
+        343,
+        75,
+        4987,
+        16,
+        1153,
+        782,
+        17132,
+        2339,
+        1585,
+        11,
+        305,
+        7959,
+        611,
+        1093,
+        3189,
+        294,
+        3278,
+        304,
+        260,
+        3242,
+        15,
+        2761,
+        814,
+        20422,
+        565,
+        786,
+        9924,
+        5252,
+        574,
+        3625,
+        26568,
+        1167,
+        343,
+        37684,
+        791,
+        769,
+        260,
+        9924,
+        8697,
+        11,
+        588,
+        366,
+        260,
+        26910,
+        523,
+        16,
+        69069,
+        2119,
+        29780,
+        295,
+        1117,
+        6805,
+        3609,
+        343,
+        75939,
+        98250,
+        14,
+        19356,
+        14,
+        23939,
+        18899,
+        14,
+        7667,
+        3553,
+        813,
+        14,
+        5655,
+        5136,
+        759,
+        366,
+        3495,
+        304,
+        7065,
+        1117,
+        4063,
+        915,
+        1099,
+        40258,
+        469,
+        11232,
+        16,
+        2052,
+        440,
+        4006,
+        1153,
+        2658,
+        111994,
+        4063,
+        440,
+        759,
+        366,
+        3495,
+        304,
+        16237,
+        27864,
+        603,
+        2107,
+        344,
+        260,
+        1894,
+        5090,
+        304,
+        108443,
+        7405,
+        418,
+        270,
+        78834,
+        475,
+        297,
+        20045,
+        29,
+        566,
+        759,
+        8461,
+        440,
+        260,
+        3929,
+        294,
+        1014,
+        305,
+        15985,
+        603,
+        3054,
+        15391,
+        440,
+        759,
+        1473,
+        10381,
+        1309,
+        3277,
+        16886,
+        343,
+        74,
+        744,
+        88,
+        13346,
+        797,
+        3076,
+        44181,
+        14,
+        3651,
+        14,
+        7218,
+        45634,
+        305,
+        915,
+        16886,
+        9587,
+        4365,
+        477,
+        1855,
+        2953,
+        295,
+        11234,
+        14,
+        305,
+        270,
+        9587,
+        3711,
+        344,
+        5292,
+        1855,
+        7370,
+        343,
+        32812,
+        295,
+        106714,
+        14,
+        1479,
+        270,
+        9587,
+        3711,
+        344,
+        4077,
+        5724,
+        304,
+        915,
+        11234,
+        797,
+        3211,
+        14,
+        7097,
+        305,
+        295,
+        1093,
+        14664,
+        1031,
+        477,
+        4454,
+        418,
+        10524,
+        27250,
+        343,
+        272,
+        396,
+        344,
+        270,
+        6805,
+        6013,
+        754,
+        832,
+        696,
+        554,
+        9587,
+        2727,
+        1623,
+        13459,
+        11482,
+        440,
+        4750,
+        24185,
+        304,
+        270,
+        850,
+        38299,
+        4501,
+        16,
+        3245,
+        4843,
+        1856,
+        6005,
+        30416,
+        16339,
+        440,
+        2786,
+        366,
+        3495,
+        304,
+        3353,
+        295,
+        3076,
+        13459,
+        469,
+        125322,
+        14,
+        412,
+        1948,
+        477,
+        11737,
+        12836,
+        14,
+        305,
+        440,
+        2786,
+        295,
+        1577,
+        1178,
+        2993,
+        6671,
+        1099,
+        295,
+        6805,
+        9587,
+        4365,
+        16,
+        3211,
+        14,
+        366,
+        16372,
+        14,
+        1754,
+        436,
+        734,
+        554,
+        6537,
+        304,
+        1635,
+        16339,
+        418,
+        8479,
+        16886,
+        603,
+        3575,
+        440,
+        1970,
+        304,
+        6097,
+        1117,
+        3576,
+        294,
+        38848,
+        343,
+        82,
+        5065,
+        1045,
+        14,
+        2898,
+        402,
+        15876,
+        11,
+        295,
+        106714,
+        14,
+        270,
+        2445,
+        304,
+        5711,
+        344,
+        1890,
+        460,
+        340,
+        21307,
+        628,
+        60060,
+        571,
+        343,
+        7626,
+        35037,
+        452,
+        4978,
+        40266,
+        733,
+        8923,
+        39435,
+        60070,
+        66711,
+        1091,
+        295,
+        40258,
+        14,
+        105088,
+        1398,
+        16138,
+        16980,
+        8994,
+        649,
+        1932,
+        295,
+        11232,
+        5772,
+        21819,
+        51498,
+        24025,
+        43269,
+        8606,
+        4756,
+        342,
+        3699,
+        8655,
+        2570,
+        10290,
+        41124,
+        13969,
+        36169,
+        6010,
+        6367,
+        102098,
+        75334,
+        35610,
+        8606,
+        64000,
+        6010,
+        64515,
+        2508,
+        25458,
+        116851,
+        27789,
+        2622,
+        120311,
+        14,
+        75784,
+        14,
+        58698,
+        14,
+        9600,
+        30649,
+        14,
+        5655,
+        1613,
+        1162,
+        2950,
+        344,
+        29851,
+        53206,
+        509,
+        710,
+        16897,
+        4365,
+        294,
+        270,
+        3924,
+        305,
+        834,
+        11027,
+        9754,
+        48134,
+        305,
+        7959,
+        49513,
+        603,
+        15,
+        1924,
+        294,
+        5077,
+        223,
+        792,
+        27,
+        201,
+        19,
+        628,
+        30009,
+        438,
+        223,
+        18,
+        16,
+        7207,
+        22191,
+        343,
+        19,
+        22191,
+        438,
+        223,
+        25,
+        16,
+        1977,
+        628,
+        30009,
+        682,
+        19,
+        628,
+        30009,
+        438,
+        223,
+        18,
+        16,
+        25349,
+        43128,
+        343,
+        19,
+        43128,
+        438,
+        223,
+        553,
+        16,
+        5926,
+        628,
+        30009,
+        682,
+        19,
+        628,
+        30009,
+        31,
+        223,
+        18,
+        16,
+        35213,
+        343,
+        19,
+        65259,
+        438,
+        223,
+        779,
+        16,
+        3175,
+        628,
+        30009,
+        682,
+        83829,
+        41506,
+        344,
+        5686,
+        54562,
+        14,
+        790,
+        1438,
+        412,
+        915,
+        6087,
+        278,
+        1656,
+        295,
+        270,
+        5181,
+        6623,
+        260,
+        3929,
+        294,
+        7762,
+        15833,
+        14,
+        4861,
+        295,
+        270,
+        72996,
+        24575,
+        16,
+        37345,
+        3277,
+        3344,
+        5260,
+        582,
+        85,
+        17198,
+        4,
+        343,
+        85,
+        70046,
+        314,
+        593,
+        11,
+        305,
+        102423,
+        1277,
+        582,
+        68,
+        808,
+        74,
+        348,
+        4,
+        343,
+        24317,
+        2418,
+        295,
+        14153,
+        84,
+        5136,
+        1960,
+        294,
+        4332,
+        366,
+        1697,
+        469,
+        582,
+        67794,
+        976,
+        3332,
+        4,
+        343,
+        444,
+        24794,
+        419,
+        2525,
+        295,
+        14153,
+        84,
+        5136,
+        778,
+        344,
+        260,
+        21151,
+        30927,
+        28574,
+        16,
+        455,
+        1257,
+        14,
+        3010,
+        81,
+        14,
+        344,
+        8808,
+        3061,
+        440,
+        2786,
+        554,
+        1635,
+        7405,
+        3047,
+        565,
+        4499,
+        344,
+        260,
+        21151,
+        3317,
+        17089,
+        14,
+        412,
+        362,
+        270,
+        102423,
+        1902,
+        260,
+        2231,
+        109422,
+        17089,
+        603,
+        3575,
+        440,
+        477,
+        5854,
+        260,
+        3734,
+        4593,
+        469,
+        295,
+        11005,
+        943,
+        3344,
+        14,
+        8509,
+        29450,
+        305,
+        2953,
+        4880,
+        943,
+        1479,
+        440,
+        6097,
+        3344,
+        16,
+        18192,
+        304,
+        6097,
+        87785,
+        1353,
+        295,
+        105975,
+        469,
+        3226,
+        39323,
+        16765,
+        14,
+        3165,
+        4085,
+        270,
+        58782,
+        5045,
+        14,
+        305,
+        3214,
+        6097,
+        14110,
+        469,
+        32023,
+        5039,
+        377,
+        270,
+        12505,
+        343,
+        12829,
+        588,
+        6097,
+        1066,
+        377,
+        270,
+        3194,
+        14,
+        790,
+        554,
+        4851,
+        270,
+        3194,
+        5772,
+        4823,
+        18449,
+        260,
+        18282,
+        509,
+        778,
+        304,
+        8605,
+        14,
+        440,
+        759,
+        1783,
+        834,
+        396,
+        440,
+        1277,
+        2951,
+        377,
+        270,
+        15451,
+        17593,
+        513,
+        270,
+        23254,
+        294,
+        1750,
+        21048,
+        16,
+        1162,
+        1142,
+        5706,
+        17759,
+        14,
+        790,
+        295,
+        1473,
+        23721,
+        295,
+        20717,
+        1031,
+        477,
+        1093,
+        1855,
+        1894,
+        25453,
+        14,
+        7097,
+        1749,
+        65269,
+        6684,
+        14,
+        305,
+        1305,
+        25453,
+        696,
+        8463,
+        10883,
+        1960,
+        3344,
+        16,
+        2052,
+        440,
+        1277,
+        24185,
+        304,
+        850,
+        13629,
+        5949,
+        294,
+        270,
+        3924,
+        305,
+        477,
+        295,
+        11005,
+        943,
+        1205,
+        304,
+        8605,
+        14,
+        6360,
+        396,
+        18638,
+        477,
+        3165,
+        260,
+        7681,
+        6749,
+        603,
+        63964,
+        270,
+        1722,
+        440,
+        1142,
+        1783,
+        10722,
+        7892,
+        304,
+        8605,
+        554,
+        513,
+        2502,
+        270,
+        9497,
+        11064,
+        14,
+        790,
+        1438,
+        513,
+        37428,
+        270,
+        12709,
+        362,
+        270,
+        20928,
+        294,
+        7075,
+        8092,
+        33138,
+        16,
+        3109,
+        477,
+        2915,
+        7892,
+        1479,
+        851,
+        8463,
+        7075,
+        40258,
+        3344,
+        14,
+        2622,
+        1855,
+        54562,
+        568,
+        1844,
+        367,
+        5701,
+        343,
+        11696,
+        73898,
+        3403,
+        295,
+        14153,
+        84,
+        14361,
+        14676,
+        6433,
+        4797,
+        2093,
+        477,
+        1855,
+        17608,
+        14,
+        305,
+        850,
+        2915,
+        1099,
+        554,
+        440,
+        759,
+        366,
+        834,
+        294,
+        786,
+        1257,
+        8479,
+        17051,
+        16,
+        14493,
+        304,
+        270,
+        582,
+        68,
+        808,
+        74,
+        348,
+        4,
+        440,
+        2786,
+        990,
+        2997,
+        362,
+        582,
+        20292,
+        2928,
+        5701,
+        4,
+        343,
+        52270,
+        1770,
+        3403,
+        295,
+        14153,
+        84,
+        16,
+        565,
+        70179,
+        65675,
+        11300,
+        418,
+        14110,
+        469,
+        18638,
+        11,
+        469,
+        582,
+        1514,
+        106209,
+        4,
+        343,
+        22761,
+        649,
+        1384,
+        565,
+        37112,
+        90246,
+        797,
+        2042,
+        611,
+        304,
+        3047,
+        374,
+        591,
+        2928,
+        5701,
+        418,
+        30020,
+        305,
+        56318,
+        18298,
+        295,
+        260,
+        122674,
+        284,
+        34265,
+        305,
+        25967,
+        111027,
+        19816,
+        565,
+        436,
+        734,
+        260,
+        24403,
+        17089,
+        16,
+        3109,
+        477,
+        1438,
+        67477,
+        14,
+        790,
+        6684,
+        396,
+        2786,
+        10203,
+        440,
+        890,
+        6889,
+        603,
+        671,
+        40258,
+        45617,
+        344,
+        5415,
+        321,
+        3332,
+        343,
+        1805,
+        3277,
+        2329,
+        362,
+        127752,
+        11,
+        418,
+        24197,
+        16,
+        6241,
+        14021,
+        294,
+        127752,
+        477,
+        990,
+        5686,
+        5885,
+        565,
+        314,
+        35915,
+        343,
+        13433,
+        601,
+        754,
+        25480,
+        14,
+        126889,
+        14,
+        25339,
+        16,
+        65488,
+        3291,
+        538,
+        957,
+        19,
+        304,
+        957,
+        553,
+        343,
+        19,
+        15,
+        25,
+        19659,
+        8930,
+        18,
+        14,
+        23,
+        314,
+        16,
+        43760,
+        39626,
+        35967,
+        477,
+        3510,
+        362,
+        4077,
+        9572,
+        343,
+        22241,
+        35967,
+        7561,
+        54034,
+        294,
+        957,
+        1328,
+        343,
+        1810,
+        19659,
+        8930,
+        18,
+        16,
+        23,
+        314,
+        797,
+        2454,
+        344,
+        260,
+        2405,
+        6749,
+        294,
+        14394,
+        14,
+        1952,
+        12541,
+        305,
+        37386,
+        16,
+        455,
+        12541,
+        45290,
+        11755,
+        67964,
+        295,
+        270,
+        10828,
+        14,
+        295,
+        270,
+        34629,
+        276,
+        5181,
+        565,
+        3459,
+        362,
+        14394,
+        3490,
+        26025,
+        1559,
+        304,
+        3808,
+        12920,
+        19968,
+        1060,
+        223,
+        20,
+        14,
+        1320,
+        1737,
+        5084,
+        14,
+        7254,
+        45290,
+        538,
+        270,
+        4837,
+        9860,
+        976,
+        5181,
+        294,
+        628,
+        17684,
+        272,
+        401,
+        477,
+        990,
+        5686,
+        54562,
+        16,
+        65488,
+        362,
+        3277,
+        14394,
+        3291,
+        1469,
+        957,
+        20,
+        304,
+        957,
+        1328,
+        343,
+        20,
+        15,
+        1810,
+        19659,
+        11,
+        716,
+        21973,
+        294,
+        223,
+        18,
+        14,
+        2402,
+        314,
+        343,
+        56406,
+        270,
+        85520,
+        45290,
+        14,
+        957,
+        19,
+        469,
+        2477,
+        14,
+        412,
+        1305,
+        477,
+        7097,
+        96183,
+        412,
+        4200,
+        45290,
+        790,
+        7340,
+        412,
+        3277,
+        374,
+        650,
+        1572,
+        754,
+        4499,
+        14,
+        834,
+        588,
+        1783,
+        24954,
+        13890,
+        14,
+        7213,
+        14,
+        14030,
+        45290,
+        538,
+        957,
+        1328,
+        716,
+        21973,
+        305,
+        850,
+        295,
+        3734,
+        105975,
+        305,
+        1473,
+        25453,
+        16,
+        455,
+        5220,
+        294,
+        37386,
+        45290,
+        16405,
+        7572,
+        1060,
+        270,
+        2336,
+        1167,
+        294,
+        1737,
+        305,
+        14116,
+        10788,
+        3780,
+        44639,
+        295,
+        5220,
+        603,
+        4246,
+        477,
+        260,
+        3929,
+        294,
+        49505,
+        343,
+        32973,
+        60504,
+        305,
+        2408,
+        15217,
+        43678,
+        797,
+        40258,
+        22753,
+        344,
+        294,
+        1855,
+        1894,
+        4474,
+        16,
+        62960,
+        538,
+        55672,
+        469,
+        50094,
+        85,
+        343,
+        6439,
+        2953,
+        295,
+        107801,
+        11,
+        344,
+        2915,
+        106900,
+        1910,
+        16,
+        345,
+        9834,
+        22753,
+        344,
+        554,
+        1855,
+        2953,
+        295,
+        20717,
+        305,
+        7097,
+        554,
+        294,
+        270,
+        1975,
+        4474,
+        412,
+        270,
+        1975,
+        6890,
+        7340,
+        295,
+        35967,
+        16,
+        455,
+        2455,
+        80802,
+        477,
+        2375,
+        31843,
+        513,
+        462,
+        60697,
+        85,
+        2438,
+        14,
+        541,
+        4552,
+        265,
+        305,
+        32907,
+        36,
+        343,
+        43726,
+        4059,
+        19170,
+        268,
+        2720,
+        7570,
+        88,
+        1937,
+        601,
+        797,
+        17725,
+        284,
+        80802,
+        477,
+        990,
+        11737,
+        3510,
+        790,
+        850,
+        14738,
+        1256,
+        362,
+        7409,
+        14,
+        260,
+        21973,
+        294,
+        55315,
+        4560,
+        317,
+        971,
+        1151,
+        588,
+        2753,
+        54034,
+        294,
+        957,
+        20,
+        37258,
+        1901,
+        5004,
+        5220,
+        294,
+        40258,
+        22753,
+        344,
+        223,
+        1328,
+        9888,
+        16,
+        2899,
+        4962,
+        14,
+        40258,
+        80802,
+        477,
+        1855,
+        54562,
+        305,
+        30069,
+        23881,
+        21345,
+        295,
+        4174,
+        603,
+        6932,
+        2408,
+        15217,
+        43678,
+        49505,
+        834,
+        1531,
+        3047,
+        473,
+        12265,
+        1256,
+        260,
+        8600,
+        1891,
+        724,
+        297,
+        13312,
+        1960,
+        294,
+        125508,
+        469,
+        25339,
+        16,
+        10272,
+        270,
+        8012,
+        834,
+        588,
+        7186,
+        6097,
+        436,
+        538,
+        21962,
+        12505,
+        38498,
+        16,
+        983,
+        442,
+        85,
+        2993,
+        304,
+        6097,
+        436,
+        295,
+        35967,
+        3504,
+        294,
+        13026,
+        2400,
+        1592,
+        449,
+        294,
+        270,
+        28326,
+        16,
+        72386,
+        26737,
+        14,
+        294,
+        710,
+        29074,
+        14,
+        477,
+        990,
+        3510,
+        14,
+        7254,
+        11755,
+        295,
+        105975,
+        16,
+        62987,
+        1154,
+        294,
+        19504,
+        2237,
+        477,
+        3510,
+        21602,
+        14,
+        412,
+        1585,
+        412,
+        30112,
+        4403,
+        14,
+        22753,
+        14,
+        305,
+        3828,
+        26737,
+        16,
+        3245,
+        11800,
+        304,
+        6097,
+        96183,
+        2237,
+        1635,
+        3645,
+        304,
+        2997,
+        362,
+        582,
+        88,
+        13078,
+        21011,
+        15675,
+        87,
+        4,
+        343,
+        9372,
+        2503,
+        5606,
+        11,
+        9694,
+        440,
+        477,
+        4888,
+        304,
+        366,
+        27592,
+        270,
+        7569,
+        638,
+        13312,
+        603,
+        57,
+        55392,
+        16,
+        16696,
+        6097,
+        127752,
+        469,
+        10837,
+        439,
+        343,
+        1805,
+        3277,
+        2329,
+        362,
+        8156,
+        91,
+        11,
+        5854,
+        105975,
+        469,
+        58073,
+        16765,
+        14,
+        362,
+        1031,
+        477,
+        260,
+        3929,
+        294,
+        285,
+        2357,
+        16,
+        9198,
+        1141,
+        260,
+        2767,
+        2464,
+        412,
+        260,
+        1529,
+        294,
+        26457,
+        49974,
+        565,
+        260,
+        13176,
+        1505,
+        304,
+        1635,
+        32640,
+        61042,
+        65098,
+        603,
+        39981,
+        1713,
+        2786,
+        366,
+        260,
+        43261,
+        3217,
+        362,
+        260,
+        17739,
+        264,
+        18159,
+        5854,
+        106714,
+        16,
+        455,
+        36059,
+        270,
+        16161,
+        14,
+        270,
+        7294,
+        270,
+        8369,
+        294,
+        1093,
+        5686,
+        44912,
+        61108,
+        14,
+        4861,
+        362,
+        1948,
+        554,
+        10881,
+        418,
+        270,
+        17292,
+        26073,
+        2192,
+        294,
+        4008,
+        778,
+        2413,
+        7926,
+        295,
+        1623,
+        7892,
+        603,
+        4246,
+        477,
+        1623,
+        6431,
+        15,
+        3291,
+        343,
+        39,
+        223,
+        1069,
+        15,
+        1883,
+        11,
+        6536,
+        5854,
+        106714,
+        16,
+        1884,
+        7409,
+        295,
+        52384,
+        4728,
+        44341,
+        340,
+        4056,
+        343,
+        84037,
+        270,
+        4837,
+        9860,
+        3697,
+        754,
+        270,
+        2887,
+        3711,
+        344,
+        9648,
+        223,
+        1810,
+        16403,
+        362,
+        260,
+        27899,
+        343,
+        15130,
+        5638,
+        305,
+        13507,
+        4772,
+        11,
+        261,
+        25198,
+        33926,
+        601,
+        16,
+        8647,
+        36895,
+        611,
+        270,
+        6749,
+        1469,
+        116054,
+        18380,
+        2283,
+        87,
+        3489,
+        12951,
+        30415,
+        5392,
+        5925,
+        305,
+        554,
+        116054,
+        18380,
+        343,
+        71,
+        12481,
+        22503,
+        276,
+        5392,
+        797,
+        455,
+        2336,
+        6749,
+        344,
+        850,
+        1099,
+        223,
+        1328,
+        7,
+        36059,
+        305,
+        7047,
+        440,
+        260,
+        68972,
+        3072,
+        91349,
+        223,
+        20,
+        4772,
+        27899,
+        14,
+        6569,
+        790,
+        6219,
+        8567,
+        4246,
+        477,
+        1353,
+        2038,
+        223,
+        23,
+        48465,
+        36895,
+        28,
+        295,
+        106714,
+        3252,
+        270,
+        31773,
+        37622,
+        305,
+        46535,
+        14,
+        305,
+        2658,
+        834,
+        295,
+        61694,
+        999,
+        3252,
+        7740,
+        68,
+        624,
+        37622,
+        14,
+        790,
+        851,
+        477,
+        1855,
+        14738,
+        16,
+        8819,
+        17739,
+        36895,
+        8028,
+        10725,
+        14,
+        9683,
+        33123,
+        60994,
+        305,
+        11697,
+        2313,
+        16,
+        2359,
+        477,
+        554,
+        377,
+        270,
+        21589,
+        3317,
+        4338,
+        305,
+        477,
+        5292,
+        3530,
+        14,
+        832,
+        1635,
+        58606,
+        295,
+        10055,
+        603,
+        13718,
+        7284,
+        344,
+        304,
+        11917,
+        411,
+        23594,
+        377,
+        270,
+        9898,
+        1936,
+        440,
+        6559,
+        782,
+        3924,
+        16,
+        2454,
+        734,
+        1623,
+        304,
+        7238,
+        538,
+        295,
+        106714,
+        305,
+        19481,
+        19034,
+        16,
+        42542,
+        28,
+        5453,
+        106714,
+        295,
+        782,
+        24623,
+        377,
+        9898,
+        8567,
+        3085,
+        1623,
+        1482,
+        538,
+        472,
+        1715,
+        86899,
+        5407,
+        696,
+        344,
+        304,
+        807,
+        304,
+        270,
+        34229,
+        9466,
+        14,
+        1479,
+        851,
+        3047,
+        304,
+        1783,
+        1482,
+        995,
+        477,
+        11401,
+        304,
+        11917,
+        260,
+        4772,
+        16,
+        65488,
+        477,
+        5292,
+        1949,
+        36059,
+        305,
+        855,
+        1031,
+        477,
+        4128,
+        1482,
+        11523,
+        270,
+        4772,
+        440,
+        588,
+        1635,
+        2405,
+        20833,
+        343,
+        261,
+        894,
+        79380,
+        1482,
+        477,
+        4975,
+        16041,
+        304,
+        366,
+        3495,
+        304,
+        4449,
+        304,
+        440,
+        5772,
+        10137,
+        20833,
+        477,
+        5292,
+        554,
+        6537,
+        305,
+        851,
+        759,
+        2188,
+        440,
+        304,
+        260,
+        12560,
+        1936,
+        56862,
+        16,
+        12025,
+        3645,
+        851,
+        611,
+        7722,
+        2237,
+        14,
+        305,
+        2090,
+        1664,
+        366,
+        17601,
+        304,
+        2507,
+        436,
+        734,
+        554,
+        1205,
+        440,
+        5604,
+        1082,
+        11545,
+        270,
+        4772,
+        603,
+        4246,
+        477,
+        260,
+        3929,
+        294,
+        8479,
+        2604,
+        295,
+        40258,
+        21894,
+        16,
+        406,
+        114737,
+        344,
+        10140,
+        14,
+        440,
+        588,
+        4486,
+        260,
+        66329,
+        2155,
+        3638,
+        20704,
+        1438,
+        13796,
+        343,
+        1805,
+        1257,
+        305,
+        2336,
+        3137,
+        294,
+        2286,
+        11,
+        855,
+        440,
+        611,
+        4601,
+        16,
+        3841,
+        734,
+        260,
+        12139,
+        106132,
+        14,
+        294,
+        3584,
+        14,
+        790,
+        270,
+        2041,
+        1988,
+        344,
+        554,
+        1949,
+        1688,
+        16,
+        5673,
+        3584,
+        855,
+        834,
+        10059,
+        304,
+        4486,
+        1894,
+        4134,
+        851,
+        759,
+        14,
+        790,
+        19583,
+        295,
+        1345,
+        260,
+        6299,
+        344,
+        3157,
+        603,
+        14737,
+        1482,
+        759,
+        4575,
+        440,
+        396,
+        440,
+        588,
+        2188,
+        260,
+        7610,
+        294,
+        782,
+        36223,
+        418,
+        440,
+        16,
+        85618,
+        14,
+        1093,
+        1482,
+        3217,
+        14805,
+        1060,
+        566,
+        16,
+        983,
+        734,
+        3165,
+        2993,
+        304,
+        7457,
+        782,
+        50090,
+        418,
+        440,
+        16,
+        334,
+        52266,
+        3738,
+        588,
+        366,
+        21054,
+        412,
+        12745,
+        294,
+        9639,
+        16,
+        334,
+        8697,
+        1926,
+        304,
+        260,
+        3277,
+        396,
+        588,
+        1694,
+        440,
+        759,
+        2915,
+        12827,
+        1855,
+        5379,
+        304,
+        1694,
+        440,
+        603,
+        6287,
+        270,
+        6744,
+        294,
+        782,
+        3277,
+        80771,
+        305,
+        7959,
+        1120,
+        21471,
+        295,
+        10055,
+        305,
+        7155,
+        786,
+        13331,
+        3737,
+        603,
+        3575,
+        440,
+        588,
+        436,
+        344,
+        6923,
+        304,
+        611,
+        260,
+        3242,
+        15,
+        2761,
+        814,
+        82691,
+        995,
+        588,
+        366,
+        3252,
+        295,
+        411,
+        13331,
+        469,
+        855,
+        440,
+        13442,
+        18330,
+        16,
+        2052,
+        21936,
+        362,
+        1117,
+        4892,
+        294,
+        1014,
+        436,
+        344,
+        52538,
+        304,
+        1178,
+        260,
+        3277,
+        40151,
+        5474,
+        362,
+        782,
+        9924,
+        362,
+        57697,
+        14,
+        305,
+        362,
+        36059,
+        3277,
+        10699,
+        125377,
+        85,
+        16,
+        3109,
+        477,
+        11737,
+        3510,
+        14,
+        21589,
+        305,
+        5418,
+        304,
+        905,
+        16126,
+        7550,
+        5832,
+        2991,
+        295,
+        1117,
+        915,
+        3924,
+        14,
+        1812,
+        2953,
+        4880,
+        1082,
+        24185,
+        295,
+        20717,
+        759,
+        21896,
+        1117,
+        20029,
+        294,
+        1902,
+        11575,
+        294,
+        79311,
+        13403,
+        305,
+        38015,
+        16,
+        18192,
+        554,
+        304,
+        1528,
+        1387,
+        270,
+        1577,
+        396,
+        440,
+        4750,
+        260,
+        8479,
+        264,
+        469,
+        1208,
+        67,
+        3992,
+        782,
+        12520,
+        565,
+        513,
+        20193,
+        469,
+        9694,
+        16,
+        3710,
+        270,
+        13981,
+        294,
+        106714,
+        14,
+        19481,
+        19034,
+        305,
+        915,
+        3226,
+        11234,
+        14,
+        8479,
+        34686,
+        477,
+        2413,
+        5686,
+        9497,
+        16,
+        1924,
+        295,
+        1117,
+        3924,
+        270,
+        12644,
+        294,
+        79311,
+        38015,
+        12036,
+        16,
+        660,
+        106714,
+        1635,
+        3645,
+        304,
+        12713,
+        782,
+        24780,
+        305,
+        1856,
+        1754,
+        6009,
+        82,
+        10213,
+        288,
+        344,
+        1855,
+        2953,
+        16,
+        74209,
+        611,
+        4962,
+        34686,
+        304,
+        6599,
+        3480,
+        1482,
+        1754,
+        851,
+        7431,
+        1066,
+        2507,
+        14,
+        582,
+        8474,
+        1747,
+        1277,
+        12050,
+        14,
+        2513,
+        734,
+        1605,
+        1066,
+        362,
+        260,
+        1901,
+        305,
+        1762,
+        1205,
+        579,
+        588,
+        1178,
+        14258,
+        46344,
+        7555,
+        14,
+        855,
+        1902,
+        25488,
+        513,
+        8531,
+        469,
+        915,
+        2950,
+        19940,
+        565,
+        696,
+        782,
+        2455,
+        304,
+        4946,
+        1066,
+        396,
+        440,
+        4750,
+        260,
+        8479,
+        39428,
+        16,
+        4165,
+        1623,
+        8531,
+        12502,
+        7065,
+        8479,
+        10555,
+        27639,
+        14,
+        4499,
+        1623,
+        1482,
+        477,
+        26391,
+        304,
+        8233,
+        295,
+        14891,
+        603,
+        13222,
+        1664,
+        13312,
+        12069,
+        295,
+        260,
+        3817,
+        294,
+        13026,
+        1482,
+        343,
+        9808,
+        1142,
+        366,
+        10957,
+        850,
+        27639,
+        1099,
+        295,
+        270,
+        5937,
+        797,
+        2042,
+        2090,
+        1664,
+        1153,
+        1192,
+        1949,
+        477,
+        851,
+        2887,
+        304,
+        13312,
+        343,
+        458,
+        37133,
+        440,
+        304,
+        13312,
+        418,
+        1066,
+        754,
+        305,
+        1205,
+        22633,
+        1142,
+        20598,
+        1561,
+        396,
+        16,
+        9085,
+        14,
+        1623,
+        105671,
+        3697,
+        14,
+        3459,
+        362,
+        260,
+        5477,
+        19935,
+        362,
+        260,
+        1894,
+        13312,
+        14,
+        588,
+        7097,
+        28907,
+        1345,
+        411,
+        3772,
+        294,
+        127752,
+        396,
+        1114,
+        366,
+        5083,
+        55918,
+        362,
+        270,
+        5004,
+        22753,
+        15,
+        5273,
+        51722,
+        10734,
+        264,
+        603,
+        10032,
+        15925,
+        11590,
+        201,
+        44397,
+        15632,
+        344,
+        260,
+        37408,
+        10524,
+        8536,
+        16,
+        455,
+        4854,
+        294,
+        6939,
+        23908,
+        305,
+        14193,
+        17887,
+        343,
+        39196,
+        316,
+        2760,
+        11,
+        769,
+        14376,
+        6889,
+        305,
+        477,
+        1928,
+        23034,
+        3510,
+        295,
+        710,
+        790,
+        270,
+        19904,
+        29645,
+        16,
+        4147,
+        4085,
+        270,
+        5597,
+        294,
+        270,
+        8126,
+        565,
+        436,
+        1114,
+        366,
+        24218,
+        304,
+        1347,
+        834,
+        396,
+        344,
+        18661,
+        509,
+        260,
+        6939,
+        14,
+        4562,
+        1099,
+        295,
+        2658,
+        21048,
+        16,
+        2042,
+        588,
+        1347,
+        782,
+        8392,
+        10165,
+        343,
+        84162,
+        14244,
+        18986,
+        1051,
+        62792,
+        11,
+        469,
+        10524,
+        5901,
+        21323,
+        7186,
+        16,
+        23609,
+        305,
+        71634,
+        10165,
+        477,
+        12415,
+        513,
+        270,
+        105975,
+        16,
+        2275,
+        5789,
+        1812,
+        782,
+        8392,
+        41545,
+        6000,
+        10165,
+        362,
+        16339,
+        509,
+        67127,
+        295,
+        8577,
+        23721,
+        412,
+        42619,
+        477,
+        554,
+        17024,
+        305,
+        12092,
+        4128,
+        304,
+        5261,
+        782,
+        5474,
+        16276,
+        16,
+        13548,
+        14,
+        436,
+        344,
+        11737,
+        20935,
+        304,
+        3353,
+        10524,
+        16,
+        20568,
+        1290,
+        343,
+        35868,
+        2818,
+        27295,
+        11,
+        7097,
+        7457,
+        14,
+        305,
+        3353,
+        295,
+        10524,
+        13469,
+        5083,
+        65607,
+        3226,
+        295,
+        915,
+        5407,
+        16,
+        7740,
+        1664,
+        19800,
+        14154,
+        4577,
+        295,
+        1750,
+        1345,
+        2352,
+        603,
+        20154,
+        14,
+        436,
+        344,
+        15187,
+        11786,
+        304,
+        5789,
+        2811,
+        343,
+        78053,
+        11,
+        16886,
+        20468,
+        16286,
+        412,
+        1031,
+        477,
+        88852,
+        3611,
+        1345,
+        20468,
+        16286,
+        14,
+        396,
+        1142,
+        6240,
+        3475,
+        440,
+        3072,
+        14,
+        17292,
+        91913,
+        16886,
+        469,
+        990,
+        110186,
+        396,
+        611,
+        1047,
+        53187,
+        538,
+        24676,
+        2918,
+        270,
+        6431,
+        223,
+        1357,
+        18,
+        734,
+        16,
+        6948,
+        3953,
+        9587,
+        2631,
+        48729,
+        343,
+        107948,
+        891,
+        3510,
+        11,
+        305,
+        14664,
+        29,
+        990,
+        366,
+        72598,
+        294,
+        9587,
+        3711,
+        42779,
+        1277,
+        223,
+        23,
+        16,
+        32224,
+        17,
+        23,
+        16,
+        4858,
+        6097,
+        2283,
+        574,
+        6240,
+        294,
+        223,
+        23,
+        16,
+        3318,
+        17,
+        23,
+        16,
+        4858,
+        603,
+        671,
+        3651,
+        305,
+        3076,
+        18151,
+        477,
+        7175,
+        12415,
+        412,
+        9235,
+        6823,
+        294,
+        16886,
+        14,
+        6440,
+        295,
+        32840,
+        4501,
+        16,
+        2359,
+        477,
+        990,
+        270,
+        1473,
+        11737,
+        12415,
+        99745,
+        16886,
+        509,
+        270,
+        9587,
+        2631,
+        48729,
+        14,
+        418,
+        7218,
+        12995,
+        295,
+        4824,
+        2445,
+        603,
+        671,
+        3078,
+        2448,
+        270,
+        628,
+        2396,
+        16,
+        80771,
+        295,
+        7749,
+        340,
+        344,
+        3459,
+        362,
+        270,
+        42670,
+        26876,
+        4691,
+        26740,
+        5159,
+        1482,
+        14,
+        305,
+        1031,
+        611,
+        1047,
+        8773,
+        294,
+        1345,
+        14416,
+        377,
+        1890,
+        460,
+        91,
+        340,
+        15602,
+        14,
+        270,
+        1840,
+        32840,
+        12505,
+        396,
+        12122,
+        538,
+        4207,
+        24767,
+        879,
+        340,
+        33503,
+        1910,
+        1055,
+        22225,
+        321,
+        603,
+        35,
+        803,
+        6621,
+        4848,
+        3217,
+        344,
+        396,
+        1031,
+        344,
+        1093,
+        13716,
+        35735,
+        295,
+        20717,
+        14,
+        12549,
+        1949,
+        294,
+        270,
+        7017,
+        17292,
+        9829,
+        16,
+        29150,
+        16036,
+        538,
+        13139,
+        305,
+        10252,
+        11607,
+        305,
+        26911,
+        1149,
+        517,
+        7306,
+        1949,
+        12721,
+        305,
+        17215,
+        5671,
+        538,
+        270,
+        2486,
+        1303,
+        24356,
+        343,
+        16717,
+        722,
+        797,
+        34634,
+        611,
+        782,
+        50090,
+        343,
+        272,
+        260,
+        52266,
+        3738,
+        294,
+        270,
+        1840,
+        7169,
+        855,
+        440,
+        4750,
+        11568,
+        943,
+        17937,
+        436,
+        469,
+        855,
+        440,
+        4750,
+        21936,
+        295,
+        260,
+        16161,
+        396,
+        344,
+        11630,
+        436,
+        11,
+        412,
+        63022,
+        477,
+        10202,
+        850,
+        112294,
+        1099,
+        3628,
+        16,
+        1162,
+        344,
+        554,
+        304,
+        2507,
+        396,
+        436,
+        344,
+        56141,
+        469,
+        38083,
+        14,
+        790,
+        436,
+        344,
+        2993,
+        304,
+        366,
+        3274,
+        89,
+        1937,
+        284,
+        294,
+        270,
+        45211,
+        603,
+        9468,
+        1031,
+        734,
+        260,
+        3929,
+        294,
+        25992,
+        305,
+        45543,
+        45838,
+        6443,
+        20717,
+        14,
+        3277,
+        2237,
+        27408,
+        344,
+        98944,
+        1202,
+        45994,
+        284,
+        16,
+        983,
+        344,
+        22865,
+        396,
+        440,
+        1114,
+        366,
+        16243,
+        1901,
+        86952,
+        14,
+        4861,
+        377,
+        270,
+        12334,
+        16,
+        6948,
+        1353,
+        27890,
+        7622,
+        43809,
+        603,
+        44397,
+        15632,
+        769,
+        1093,
+        294,
+        270,
+        17072,
+        15572,
+        362,
+        6337,
+        4524,
+        19584,
+        305,
+        17812,
+        295,
+        270,
+        2058,
+        565,
+        832,
+        1214,
+        26932,
+        16,
+        14033,
+        2312,
+        1082,
+        28865,
+        270,
+        21505,
+        29,
+        4961,
+        305,
+        7818,
+        48068,
+        2391,
+        565,
+        366,
+        7058,
+        396,
+        11843,
+        26893,
+        2357,
+        377,
+        1952,
+        270,
+        6352,
+        305,
+        5854,
+        16,
+        18668,
+        440,
+        1749,
+        1309,
+        304,
+        2188,
+        2312,
+        1082,
+        1812,
+        270,
+        120536,
+        14,
+        412,
+        295,
+        30373,
+        104566,
+        270,
+        5159,
+        14,
+        57988,
+        47714,
+        70422,
+        17,
+        19318,
+        57,
+        13645,
+        264,
+        49387,
+        7097,
+        2368,
+        304,
+        5789,
+        270,
+        11843,
+        513,
+        1812,
+        270,
+        7050,
+        120536,
+        29,
+        91292,
+        469,
+        554,
+        16,
+        92583,
+        6761,
+        84,
+        1459,
+        294,
+        14738,
+        14356,
+        611,
+        1047,
+        3459,
+        14,
+        509,
+        2734,
+        14,
+        304,
+        366,
+        850,
+        96741,
+        294,
+        270,
+        6555,
+        294,
+        91292,
+        16,
+        83604,
+        19798,
+        9374,
+        16317,
+        304,
+        91292,
+        28865,
+        260,
+        6337,
+        11482,
+        1031,
+        477,
+        65700,
+        17807,
+        16,
+        34634,
+        6599,
+        798,
+        362,
+        782,
+        6555,
+        603,
+        20154,
+        366,
+        29363,
+        396,
+        38160,
+        3860,
+        17828,
+        295,
+        270,
+        1975,
+        1722,
+        412,
+        270,
+        21505,
+        295,
+        3999,
+        294,
+        107874,
+        12807,
+        16,
+        14033,
+        2312,
+        1082,
+        12893,
+        14,
+        4861,
+        295,
+        270,
+        6406,
+        305,
+        3543,
+        538,
+        270,
+        35957,
+        4501,
+        294,
+        270,
+        1840,
+        11234,
+        343,
+        67,
+        34536,
+        39758,
+        1844,
+        5296,
+        344,
+        260,
+        6923,
+        22294,
+        11,
+        412,
+        270,
+        18814,
+        477,
+        29999,
+        12578,
+        14,
+        412,
+        477,
+        1473,
+        294,
+        270,
+        23914,
+        39619,
+        1779,
+        8807,
+        85,
+        304,
+        13800,
+        14,
+        305,
+        270,
+        12505,
+        305,
+        73819,
+        16838,
+        477,
+        2915,
+        102745,
+        2383,
+        4488,
+        103605,
+        16,
+        7740,
+        1664,
+        3132,
+        377,
+        836,
+        90587,
+        15736,
+        14,
+        412,
+        1305,
+        588,
+        905,
+        45060,
+        9,
+        36903,
+        782,
+        2790,
+        1055,
+        270,
+        15534,
+        418,
+        710,
+        270,
+        3283,
+        17812,
+        8567,
+        4246,
+        344,
+        13879,
+        33585,
+        295,
+        270,
+        55979,
+        538,
+        270,
+        14132,
+        509,
+        899,
+        2450,
+        125413,
+        12613,
+        2333,
+        4514,
+        295,
+        223,
+        1809,
+        24,
+        16,
+        3211,
+        270,
+        1788,
+        344,
+        54537,
+        11482,
+        440,
+        40997,
+        4995,
+        295,
+        899,
+        2450,
+        125413,
+        3078,
+        5231,
+        16,
+        2454,
+        477,
+        1749,
+        39999,
+        304,
+        270,
+        6949,
+        294,
+        109443,
+        12403,
+        268,
+        9,
+        778,
+        344,
+        270,
+        22510,
+        304,
+        270,
+        9466,
+        16,
+        455,
+        6949,
+        344,
+        12734,
+        362,
+        270,
+        81338,
+        57024,
+        294,
+        14889,
+        294,
+        23594,
+        13800,
+        26587,
+        295,
+        223,
+        1809,
+        24,
+        14,
+        1928,
+        12286,
+        798,
+        26292,
+        270,
+        30164,
+        778,
+        127219,
+        538,
+        1737,
+        294,
+        33079,
+        603,
+        8041,
+        554,
+        13312,
+        19171,
+        2237,
+        16,
+        18501,
+        3986,
+        294,
+        566,
+        344,
+        396,
+        2237,
+        344,
+        787,
+        112546,
+        1812,
+        57839,
+        14,
+        832,
+        13132,
+        344,
+        43045,
+        16,
+        56756,
+        3338,
+        6097,
+        96183,
+        2237,
+        14,
+        778,
+        344,
+        27639,
+        3510,
+        305,
+        7175,
+        20370,
+        603,
+        44397,
+        15632,
+        769,
+        270,
+        8740,
+        11021,
+        15908,
+        22399,
+        3711,
+        295,
+        4174,
+        509,
+        8791,
+        223,
+        19,
+        16,
+        23,
+        7,
+        469,
+        223,
+        19,
+        295,
+        223,
+        2574,
+        10902,
+        16,
+        2866,
+        37831,
+        603,
+        3709,
+        3068,
+        270,
+        1577,
+        396,
+        20717,
+        344,
+        411,
+        8181,
+        8589,
+        16,
+        2042,
+        1142,
+        1783,
+        396,
+        1482,
+        477,
+        14071,
+        943,
+        1902,
+        45678,
+        412,
+        582,
+        30215,
+        3697,
+        34258,
+        44397,
+        15632,
+        344,
+        513,
+        1119,
+        3189,
+        260,
+        26128,
+        3924,
+        418,
+        3678,
+        304,
+        20193,
+        14,
+        13689,
+        14,
+        1060,
+        113963,
+        440,
+        855,
+        851,
+        588,
+        1178,
+        513,
+        418,
+        436,
+        14,
+        5499,
+        1205,
+        440,
+        9118,
+        362,
+        343,
+        61678,
+        797,
+        983,
+        344,
+        1855,
+        1688,
+        538,
+        1132,
+        17739,
+        10143,
+        295,
+        1093,
+        294,
+        1305,
+        4599,
+        603,
+        2337,
+        4366,
+        515,
+        2336,
+        22601,
+        509,
+        223,
+        1002,
+        28,
+        2111,
+        14,
+        377,
+        223,
+        1173,
+        5077,
+        223,
+        792,
+        27,
+        513,
+        14943,
+        104788,
+        16,
+        15557,
+        377,
+        1116,
+        513,
+        7558,
+        9785,
+        3429,
+        14,
+        10384,
+        72535,
+        305,
+        35293,
+        83905,
+        14,
+        7566,
+        279,
+        33911,
+        3967,
+        3982,
+        11,
+        342,
+        359,
+        356,
+        12953,
+        14,
+        437,
+        2314,
+        61091,
+        2193,
+        14,
+        66763,
+        328,
+        976,
+        305,
+        9973,
+        13971,
+        1557,
+        14,
+        61611,
+        3967,
+        3982,
+        11,
+        294,
+        7566,
+        279,
+        33911,
+        305,
+        3628,
+        16,
+        1
+      ],
+      "seq_len": 2048
+    }
+  },
+  "config": {
+    "vocab_size": 129280,
+    "dim": 1536,
+    "n_layers": 16,
+    "n_heads": 12,
+    "head_dim": 128,
+    "rope_head_dim": 32,
+    "q_lora_rank": 384,
+    "o_groups": 4,
+    "o_lora_rank": 256,
+    "norm_eps": 1e-06,
+    "n_routed_experts": 12,
+    "n_shared_experts": 1,
+    "n_activated_experts": 2,
+    "moe_inter_dim": 1408,
+    "score_func": "softmax",
+    "route_scale": 1.0,
+    "swiglu_limit": 15.0,
+    "n_dense_layers": 2,
+    "load_balance_alpha": 0.001,
+    "hc_mult": 3,
+    "hc_sinkhorn_iters": 8,
+    "hc_eps": 1e-06,
+    "window_size": 128,
+    "compress_ratio": 4,
+    "n_mtp_layers": 1,
+    "mtp_loss_weight": 0.02,
+    "rope_theta": 10000.0,
+    "rope_factor": 1.0,
+    "original_seq_len": 0,
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "ctx_phase_boundaries": [
+      60000,
+      80000,
+      100000
+    ],
+    "ctx_phase_lengths": [
+      2048,
+      4096,
+      8192
+    ],
+    "yarn_scale": 40.0,
+    "yarn_alpha": 1.0,
+    "yarn_beta": 32.0,
+    "yarn_original_seq_len": 4096,
+    "micro_batch_size": 1,
+    "gradient_accumulation_steps": 32,
+    "max_lr": 0.00015,
+    "min_lr": 1.5e-05,
+    "warmup_steps": 2000,
+    "total_steps": 200000,
+    "decay_steps": 180000,
+    "weight_decay": 0.1,
+    "muon_beta": 0.95,
+    "muon_ns_steps": 7,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.98,
+    "grad_clip_norm": 5.0,
+    "init_std": 0.006,
+    "dtype": "bfloat16",
+    "fineweb_config": "default",
+    "approx_tokens_per_fineweb_doc": 800,
+    "approx_tokens_per_code_doc": 1200,
+    "approx_tokens_per_math_doc": 1500,
+    "data_mix_probs": [
+      0.6,
+      0.25,
+      0.15
+    ],
+    "save_every_steps": 400,
+    "log_every_steps": 10,
+    "push_every_steps": 400,
+    "checkpoint_dir": "/kaggle/working/zenyxv3checkpoints"
+  }
+}

checkpoints/step_22400/opt_state.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a4a3c64826de7db54a2b15ad92e26283fe56e09aee788c652b88364d015a357
+size 3154940698

checkpoints/step_22400/params.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1aa2e5bb44c8a2aeb97490a1729f1a178341cabb4d876327c3e0458673e869b
+size 3154768094