step 18000

checkpoints/step_18000/metadata.json

@@ -0,0 +1,4325 @@
+{
+  "step": 18000,
+  "tokens_consumed": 9437184000,
+  "sequences_consumed": 4608000,
+  "data_state": {
+    "stream_state": {
+      "rng_state": [
+        "MT19937",
+        [
+          520936414,
+          1935065921,
+          3287334355,
+          934970226,
+          3620142959,
+          4018149596,
+          2424068162,
+          4102793907,
+          304779644,
+          387154959,
+          2094431730,
+          3388456831,
+          1640309652,
+          2125183338,
+          1655173501,
+          3488121362,
+          3252165459,
+          3341561334,
+          962404691,
+          3582441241,
+          3064340696,
+          3709113062,
+          3706027814,
+          477267333,
+          2728126526,
+          4071591381,
+          2668203240,
+          840674011,
+          3170096346,
+          2431875279,
+          1372994947,
+          2009219793,
+          3013390696,
+          3349899899,
+          736374100,
+          2096066807,
+          708660273,
+          1791260765,
+          2135116764,
+          2182093731,
+          618572548,
+          1063091886,
+          3669870426,
+          1018693601,
+          1472226938,
+          660194723,
+          2551453160,
+          4249323620,
+          2093173279,
+          3720954236,
+          2885808280,
+          1974409272,
+          1504259282,
+          1283839497,
+          1403678519,
+          2600085028,
+          435523004,
+          2706196346,
+          2571068083,
+          619501040,
+          3115386347,
+          510558013,
+          1677553075,
+          750377796,
+          1196512204,
+          3610680636,
+          1056339688,
+          1174781865,
+          14895470,
+          404407076,
+          1484550145,
+          2968064621,
+          3677807847,
+          3625047443,
+          431497270,
+          81727381,
+          2628685377,
+          3091424983,
+          1977562548,
+          724711600,
+          1458005547,
+          14345844,
+          2188550820,
+          520391323,
+          3089988971,
+          3369732020,
+          195972149,
+          42598673,
+          1088168617,
+          238325671,
+          3133005298,
+          2702089243,
+          4031498870,
+          3677364372,
+          214522008,
+          3460762971,
+          2093641159,
+          1730826461,
+          555319103,
+          4280068490,
+          3775549952,
+          202710017,
+          1332161236,
+          1635920938,
+          490583884,
+          3429098284,
+          3549508302,
+          31420318,
+          814064533,
+          3497670344,
+          1902763128,
+          641606008,
+          3967639370,
+          3036049284,
+          985844861,
+          1492388303,
+          1794019914,
+          2922964557,
+          582222550,
+          1971032792,
+          3141205708,
+          761014450,
+          1247457591,
+          1499011112,
+          4286037672,
+          3068485951,
+          3692029125,
+          1662805315,
+          1395858520,
+          1621219763,
+          2250394641,
+          281948972,
+          2173524021,
+          2148457133,
+          2180130150,
+          4239092095,
+          4018681674,
+          677799854,
+          2498970522,
+          2106746279,
+          3010561268,
+          352766218,
+          2833207544,
+          1442770029,
+          933697951,
+          999017604,
+          2846020058,
+          2283473535,
+          3922011507,
+          4182170880,
+          1183368912,
+          2815871876,
+          3952073925,
+          422919124,
+          1716621381,
+          3471273066,
+          3952512698,
+          2564723516,
+          26803459,
+          856989319,
+          1358641457,
+          9889792,
+          2847148434,
+          1771969138,
+          1559498482,
+          2537706941,
+          1981607046,
+          3481510569,
+          640280255,
+          1679118883,
+          2093798703,
+          1698878300,
+          1533503625,
+          2970100332,
+          2432217614,
+          1241398347,
+          3759877935,
+          129404849,
+          1199157241,
+          3701355836,
+          3337707436,
+          1897969178,
+          270535108,
+          2901344753,
+          779788903,
+          243401726,
+          2848442058,
+          1884752636,
+          2011009573,
+          3511316171,
+          4263612454,
+          664411815,
+          2383829934,
+          4063828574,
+          729547969,
+          789607141,
+          1952870364,
+          669351383,
+          1426360088,
+          321865804,
+          8640653,
+          3579398767,
+          2455310717,
+          997739247,
+          3516241707,
+          1333329399,
+          3752883886,
+          2890578062,
+          4204301674,
+          1813886321,
+          130537003,
+          1843188892,
+          2715143121,
+          3783976882,
+          3974973680,
+          3713834001,
+          2500897168,
+          3731384973,
+          4135817647,
+          1054793462,
+          1341201444,
+          1117130178,
+          1264781129,
+          67493300,
+          3918803679,
+          686298573,
+          1457044493,
+          2214882055,
+          2534508879,
+          4063017638,
+          1951489111,
+          1740637624,
+          2524178613,
+          3425554366,
+          251361741,
+          2545032919,
+          2315434303,
+          2282251934,
+          707467194,
+          2234644867,
+          1516043179,
+          2579961494,
+          1290153003,
+          416915690,
+          1976961315,
+          101109547,
+          3756310557,
+          58441280,
+          1735934959,
+          3000700484,
+          3755372398,
+          1857752850,
+          3314256631,
+          2084152434,
+          3659644310,
+          2267301888,
+          1471156206,
+          2375994447,
+          1406002915,
+          1669536576,
+          2974393574,
+          1623643193,
+          1212908303,
+          1334615416,
+          2294612351,
+          246988151,
+          1426475315,
+          1110077928,
+          249453091,
+          3271653793,
+          1968475373,
+          2299249776,
+          3397487379,
+          2767993819,
+          2623749364,
+          3938851345,
+          3517649231,
+          3877610057,
+          3267971479,
+          3656983632,
+          1930554887,
+          4246925341,
+          4037256849,
+          879754746,
+          2242576742,
+          666328038,
+          1104702461,
+          1242585603,
+          633869220,
+          1622053804,
+          2102234315,
+          1978770642,
+          4134044114,
+          3674394987,
+          3194998864,
+          993636101,
+          3329244233,
+          3069198682,
+          2410462387,
+          1567777637,
+          903450406,
+          1307081780,
+          608810677,
+          1642179859,
+          1266956906,
+          2868696967,
+          2809868661,
+          2653762866,
+          1184023108,
+          2702783338,
+          526052980,
+          3000954620,
+          3743696839,
+          1798442222,
+          1616300179,
+          2743337723,
+          803881245,
+          3626024613,
+          2033515232,
+          1964936361,
+          1580895672,
+          324072122,
+          2756624909,
+          4154776888,
+          4181519551,
+          2896941794,
+          1071021589,
+          1253317585,
+          1636442668,
+          1657030297,
+          2066218607,
+          1063149388,
+          4261050557,
+          3298631802,
+          1117388370,
+          3631273447,
+          2727502668,
+          3010013972,
+          156915621,
+          3527679211,
+          2659476504,
+          1786431158,
+          3745573439,
+          1645537307,
+          626488087,
+          2242419128,
+          1199415022,
+          1167705394,
+          4170840955,
+          4277757691,
+          2250020631,
+          1799128387,
+          1473213748,
+          1556844270,
+          133814163,
+          2529444372,
+          2846890666,
+          1596697428,
+          3505833482,
+          2674460003,
+          377674747,
+          2261562433,
+          2846340057,
+          134706149,
+          83169684,
+          4217235778,
+          4212782980,
+          1518110267,
+          3732432122,
+          1149725392,
+          3957485416,
+          4190200935,
+          3891026130,
+          2649983006,
+          2210585830,
+          3800151070,
+          2452135734,
+          2131707633,
+          2804493525,
+          3638889429,
+          4278297456,
+          3025837053,
+          2693594348,
+          1017185148,
+          1968098416,
+          1396502320,
+          1357453601,
+          1507083527,
+          2696976249,
+          852442464,
+          507486379,
+          2563806878,
+          2460742852,
+          1916970482,
+          1959326159,
+          2755841147,
+          3137992433,
+          3051251163,
+          3663408422,
+          3266867072,
+          255103007,
+          1528113041,
+          1632967646,
+          1082555964,
+          3480145810,
+          370880316,
+          263485222,
+          667112576,
+          3704998487,
+          21531212,
+          2693322443,
+          2676780125,
+          3196187811,
+          252577111,
+          4289871117,
+          1409399007,
+          2830653891,
+          334944054,
+          419232260,
+          1326166296,
+          2329151568,
+          2283453227,
+          2907988349,
+          2680646475,
+          1288369736,
+          1211727775,
+          1899210623,
+          4227039931,
+          3811738993,
+          851679351,
+          4035993239,
+          392130666,
+          2395491786,
+          2723936594,
+          3167644594,
+          252406920,
+          788573379,
+          178850486,
+          2139467421,
+          1910460242,
+          298362846,
+          1390479557,
+          1741062980,
+          91433537,
+          1538075153,
+          3268017378,
+          2238509379,
+          3674106542,
+          1386772869,
+          3086353920,
+          1078477780,
+          1479838004,
+          2298794994,
+          2416594458,
+          3611258311,
+          1544284830,
+          2589891094,
+          534151550,
+          842383627,
+          2078035111,
+          2032689390,
+          3483153195,
+          1449642337,
+          3130994324,
+          226081387,
+          3246569615,
+          3018408666,
+          483375828,
+          2320384124,
+          3576145383,
+          21892665,
+          3661633399,
+          2454232879,
+          1717739425,
+          3943091896,
+          2179228155,
+          762924535,
+          390658276,
+          3842703840,
+          1803374860,
+          440269470,
+          3610580617,
+          781110994,
+          973363489,
+          569972236,
+          498667431,
+          3487619589,
+          2063927621,
+          2995078275,
+          2956979234,
+          2950536426,
+          2500146654,
+          142422508,
+          1224345657,
+          2682992835,
+          2391546736,
+          2697612611,
+          4178852695,
+          4098018019,
+          462319628,
+          3157329910,
+          2250861274,
+          961639096,
+          3886177666,
+          2500725521,
+          680401821,
+          52613616,
+          1048413596,
+          3061113338,
+          2469941865,
+          2107306363,
+          1161420547,
+          3266263235,
+          3636623757,
+          884341433,
+          1676175487,
+          2904792177,
+          4276235416,
+          914041213,
+          3316948499,
+          4061460364,
+          1257378537,
+          419202724,
+          1112868322,
+          469640266,
+          1068019078,
+          481471359,
+          2664632617,
+          2654082561,
+          2444688615,
+          1482634795,
+          970992133,
+          2054069828,
+          2348198798,
+          3938896761,
+          1163826450,
+          3603886965,
+          2513038736,
+          1919522022,
+          2824271161,
+          3089191192,
+          537161672,
+          4226698401,
+          2305654530,
+          1314440999,
+          747077934,
+          3461932309,
+          1548815840,
+          3049801916,
+          2820949495,
+          3067561852,
+          1054640255,
+          2010868361,
+          3166904270,
+          153731677,
+          3165592097,
+          2227941850,
+          1490094680,
+          4007890013,
+          1736442640,
+          2570380766,
+          2562001171,
+          2627079702,
+          3806571285,
+          1498576268,
+          835803120,
+          2713060327,
+          4218858713,
+          3377253202,
+          4059770781,
+          3443979535,
+          751704475,
+          3530109428,
+          607469709,
+          4113612350,
+          1220517255,
+          1202720974,
+          3286833030,
+          2862548071,
+          1746200040,
+          4127903161,
+          3724666652,
+          2868435800,
+          486295002,
+          53249293,
+          2722368005,
+          3218123573,
+          3103124032,
+          3053849429,
+          209461987,
+          1915014238,
+          94832265,
+          362313861,
+          3491354242,
+          3451528111,
+          1667164664,
+          1286457151,
+          3314673712,
+          2855164888,
+          880490788,
+          550953855,
+          3081279625,
+          987931653,
+          1026952421,
+          1432756714,
+          4051676683,
+          2177878651,
+          2805294901,
+          534296598,
+          616496456,
+          3474110479,
+          2459592424,
+          1113620307,
+          88039130,
+          1174165066,
+          1203487117,
+          2747446564,
+          4244986396,
+          2539190083,
+          513398262,
+          2145815378,
+          1722012848,
+          3131839524,
+          3029310238
+        ],
+        522,
+        0,
+        0.0
+      ],
+      "doc_counts": {
+        "fineweb": 974456,
+        "code": 405925,
+        "math": 243216
+      },
+      "code_lang_idx": 27
+    },
+    "token_state": {
+      "buffer_tokens": [
+        29079,
+        15116,
+        5976,
+        12,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        62655,
+        3104,
+        443,
+        4579,
+        294,
+        1660,
+        20368,
+        9043,
+        684,
+        12085,
+        271,
+        1445,
+        3452,
+        65,
+        427,
+        261,
+        65,
+        36916,
+        438,
+        3452,
+        65,
+        427,
+        261,
+        65,
+        36916,
+        940,
+        223,
+        34346,
+        12,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        44457,
+        29079,
+        22924,
+        1282,
+        22785,
+        729,
+        6593,
+        3009,
+        10563,
+        8,
+        201,
+        6118,
+        343,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        666,
+        70,
+        1282,
+        10,
+        992,
+        5203,
+        1445,
+        3452,
+        65,
+        427,
+        261,
+        4731,
+        53359,
+        81142,
+        438,
+        3452,
+        65,
+        427,
+        261,
+        4731,
+        53359,
+        81142,
+        940,
+        34346,
+        12,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        650,
+        65,
+        29495,
+        96388,
+        12,
+        44457,
+        29079,
+        22924,
+        1282,
+        22785,
+        729,
+        6593,
+        3009,
+        88989,
+        201,
+        4694,
+        200,
+        18390,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        666,
+        70,
+        1282,
+        10,
+        992,
+        5203,
+        1445,
+        696,
+        83928,
+        438,
+        223,
+        19,
+        117405,
+        729,
+        6593,
+        3009,
+        201,
+        528,
+        1008,
+        438,
+        343,
+        992,
+        15,
+        19,
+        10563,
+        10064,
+        729,
+        6593,
+        3009,
+        940,
+        83928,
+        4050,
+        66906,
+        304,
+        47041,
+        6946,
+        1167,
+        201,
+        528,
+        4050,
+        261,
+        1625,
+        19580,
+        343,
+        427,
+        1509,
+        682,
+        528,
+        1008,
+        20,
+        438,
+        82,
+        65,
+        68190,
+        85,
+        17,
+        20,
+        940,
+        343,
+        992,
+        15,
+        19,
+        10563,
+        10064,
+        729,
+        6593,
+        3009,
+        940,
+        83928,
+        201,
+        528,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        1008,
+        20,
+        201,
+        528,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        223,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        528,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        528,
+        855,
+        6600,
+        20,
+        4987,
+        83,
+        3225,
+        65,
+        68190,
+        85,
+        11,
+        9594,
+        201,
+        3392,
+        433,
+        11397,
+        6600,
+        20,
+        11,
+        438,
+        3333,
+        40869,
+        201,
+        3392,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        34282,
+        10273,
+        201,
+        3392,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        565,
+        52,
+        30523,
+        73,
+        17,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        3392,
+        855,
+        10,
+        16051,
+        31712,
+        4579,
+        11,
+        9594,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        11,
+        438,
+        223,
+        20,
+        4050,
+        61696,
+        1217,
+        566,
+        834,
+        4086,
+        304,
+        366,
+        11336,
+        305,
+        344,
+        260,
+        35485,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        20,
+        11,
+        438,
+        893,
+        42935,
+        565,
+        16819,
+        940,
+        223,
+        19,
+        4050,
+        8582,
+        9353,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        11,
+        438,
+        1008,
+        20,
+        4050,
+        105604,
+        396,
+        566,
+        8860,
+        344,
+        4515,
+        418,
+        201,
+        775,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        682,
+        3392,
+        67367,
+        201,
+        3392,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        940,
+        223,
+        19,
+        201,
+        528,
+        3006,
+        201,
+        3392,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        1008,
+        20,
+        13,
+        19,
+        201,
+        3392,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        565,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        3392,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        3392,
+        4085,
+        2536,
+        438,
+        223,
+        19,
+        201,
+        3392,
+        1633,
+        2536,
+        438,
+        30988,
+        6980,
+        70,
+        1282,
+        7093,
+        65,
+        68190,
+        85,
+        11,
+        565,
+        283,
+        1282,
+        6600,
+        20,
+        58688,
+        70,
+        1282,
+        22785,
+        729,
+        6593,
+        3009,
+        5203,
+        3392,
+        104827,
+        438,
+        223,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        26584,
+        2704,
+        2536,
+        15,
+        19,
+        682,
+        3392,
+        696,
+        597,
+        438,
+        223,
+        19,
+        23837,
+        82,
+        65,
+        68190,
+        85,
+        565,
+        1008,
+        20,
+        682,
+        775,
+        855,
+        17776,
+        4987,
+        83,
+        14970,
+        82,
+        65,
+        68190,
+        85,
+        565,
+        1008,
+        20,
+        2542,
+        9594,
+        201,
+        6118,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        280,
+        65,
+        68190,
+        85,
+        62105,
+        201,
+        6118,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        433,
+        30523,
+        73,
+        201,
+        6118,
+        855,
+        10,
+        16051,
+        31712,
+        4579,
+        11,
+        9594,
+        201,
+        1507,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        11,
+        438,
+        223,
+        20,
+        4050,
+        61696,
+        1217,
+        566,
+        834,
+        4086,
+        304,
+        366,
+        11336,
+        305,
+        344,
+        260,
+        35485,
+        201,
+        1507,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        20,
+        11,
+        438,
+        893,
+        42935,
+        565,
+        16819,
+        940,
+        223,
+        19,
+        4050,
+        8582,
+        9353,
+        201,
+        1507,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        11,
+        438,
+        1008,
+        20,
+        4050,
+        105604,
+        396,
+        566,
+        8860,
+        344,
+        4515,
+        418,
+        201,
+        1507,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        682,
+        6118,
+        67367,
+        201,
+        6118,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        775,
+        3006,
+        201,
+        6118,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        280,
+        65,
+        68190,
+        85,
+        940,
+        597,
+        201,
+        6118,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        433,
+        30523,
+        73,
+        17,
+        2512,
+        1638,
+        201,
+        6118,
+        855,
+        10,
+        16051,
+        31712,
+        4579,
+        11,
+        9594,
+        201,
+        1507,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        11,
+        438,
+        223,
+        20,
+        4050,
+        61696,
+        1217,
+        566,
+        834,
+        4086,
+        304,
+        366,
+        11336,
+        305,
+        344,
+        260,
+        35485,
+        201,
+        1507,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        20,
+        11,
+        438,
+        893,
+        42935,
+        565,
+        16819,
+        940,
+        223,
+        19,
+        4050,
+        8582,
+        9353,
+        201,
+        1507,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        11,
+        438,
+        1008,
+        20,
+        4050,
+        105604,
+        396,
+        566,
+        8860,
+        344,
+        4515,
+        418,
+        201,
+        1507,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        682,
+        6118,
+        67367,
+        201,
+        6118,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        940,
+        223,
+        19,
+        201,
+        775,
+        67367,
+        201,
+        775,
+        855,
+        17776,
+        4987,
+        83,
+        59664,
+        2536,
+        12,
+        10064,
+        729,
+        6593,
+        3009,
+        11,
+        9594,
+        201,
+        6118,
+        4085,
+        2536,
+        438,
+        4085,
+        2536,
+        940,
+        223,
+        19,
+        201,
+        6118,
+        1633,
+        2536,
+        438,
+        1633,
+        2536,
+        565,
+        223,
+        19,
+        201,
+        6118,
+        104827,
+        438,
+        223,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        26584,
+        2704,
+        2536,
+        15,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        682,
+        775,
+        67367,
+        201,
+        3392,
+        1522,
+        4016,
+        201,
+        3392,
+        1633,
+        2536,
+        438,
+        30988,
+        6980,
+        70,
+        1282,
+        7093,
+        65,
+        68190,
+        85,
+        11,
+        565,
+        283,
+        1282,
+        6600,
+        20,
+        58688,
+        70,
+        1282,
+        22785,
+        729,
+        6593,
+        3009,
+        5203,
+        3392,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        34282,
+        10273,
+        201,
+        3392,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        565,
+        52,
+        30523,
+        73,
+        4557,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        666,
+        2704,
+        2536,
+        682,
+        3392,
+        855,
+        10,
+        16051,
+        31712,
+        4579,
+        11,
+        9594,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        11,
+        438,
+        223,
+        20,
+        4050,
+        61696,
+        1217,
+        566,
+        834,
+        4086,
+        304,
+        366,
+        11336,
+        305,
+        344,
+        260,
+        35485,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        20,
+        11,
+        438,
+        893,
+        42935,
+        565,
+        16819,
+        940,
+        223,
+        19,
+        3,
+        8582,
+        9353,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        11,
+        438,
+        1008,
+        20,
+        4050,
+        105604,
+        396,
+        566,
+        8860,
+        344,
+        4515,
+        418,
+        201,
+        775,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        682,
+        3392,
+        67367,
+        201,
+        3392,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        528,
+        67367,
+        201,
+        528,
+        327,
+        17436,
+        19186,
+        6600,
+        20,
+        13,
+        19,
+        11,
+        438,
+        44917,
+        92,
+        201,
+        1445,
+        1522,
+        4016,
+        201,
+        290,
+        1522,
+        4016,
+        201,
+        361,
+        696,
+        16819,
+        438,
+        223,
+        19,
+        14,
+        893,
+        42935,
+        201,
+        1445,
+        696,
+        83928,
+        438,
+        223,
+        19,
+        117405,
+        729,
+        6593,
+        3009,
+        201,
+        528,
+        4050,
+        24992,
+        26395,
+        21470,
+        201,
+        528,
+        1008,
+        438,
+        343,
+        992,
+        15,
+        19,
+        10563,
+        10064,
+        729,
+        6593,
+        3009,
+        940,
+        83928,
+        4050,
+        66906,
+        304,
+        47041,
+        6946,
+        1167,
+        201,
+        528,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        1008,
+        201,
+        528,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        223,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        528,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        528,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        280,
+        65,
+        68190,
+        85,
+        940,
+        223,
+        19,
+        565,
+        1008,
+        201,
+        528,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        565,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        528,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        528,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        280,
+        65,
+        68190,
+        85,
+        50023,
+        201,
+        528,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        565,
+        52,
+        4941,
+        630,
+        31765,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        528,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        528,
+        327,
+        17436,
+        19186,
+        7093,
+        65,
+        68190,
+        85,
+        50023,
+        13,
+        19,
+        11,
+        438,
+        44917,
+        92,
+        201,
+        1445,
+        1522,
+        4016,
+        201,
+        361,
+        1522,
+        4016,
+        271,
+        361,
+        696,
+        16819,
+        438,
+        223,
+        19,
+        14,
+        893,
+        42935,
+        201,
+        1445,
+        696,
+        83928,
+        438,
+        223,
+        19,
+        117405,
+        729,
+        6593,
+        3009,
+        201,
+        528,
+        4050,
+        12586,
+        8358,
+        294,
+        4629,
+        201,
+        528,
+        1008,
+        438,
+        343,
+        992,
+        15,
+        19,
+        10563,
+        10064,
+        729,
+        6593,
+        3009,
+        940,
+        83928,
+        4050,
+        66906,
+        304,
+        47041,
+        6946,
+        1167,
+        201,
+        528,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        280,
+        65,
+        68190,
+        85,
+        50023,
+        201,
+        528,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        223,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        666,
+        992,
+        201,
+        528,
+        855,
+        10,
+        16051,
+        31712,
+        4579,
+        11,
+        9594,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        11,
+        438,
+        223,
+        21,
+        4050,
+        61696,
+        1217,
+        566,
+        834,
+        4086,
+        304,
+        366,
+        11336,
+        305,
+        344,
+        260,
+        35485,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        20,
+        11,
+        438,
+        16819,
+        3,
+        8582,
+        9353,
+        201,
+        775,
+        9502,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        11,
+        438,
+        1008,
+        4050,
+        105604,
+        396,
+        566,
+        8860,
+        344,
+        4515,
+        418,
+        201,
+        775,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        682,
+        528,
+        67367,
+        201,
+        528,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        1445,
+        1522,
+        4016,
+        201,
+        361,
+        1522,
+        4016,
+        201,
+        361,
+        327,
+        17436,
+        4816,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        34282,
+        10273,
+        201,
+        361,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        565,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        361,
+        44917,
+        92,
+        438,
+        44917,
+        92,
+        13,
+        19,
+        201,
+        361,
+        327,
+        17436,
+        19186,
+        10,
+        29280,
+        10273,
+        13,
+        19,
+        11,
+        438,
+        44917,
+        92,
+        271,
+        361,
+        1926,
+        8987,
+        66534,
+        443,
+        65,
+        109328,
+        4941,
+        84,
+        10,
+        29280,
+        10273,
+        14,
+        10403,
+        60,
+        20134,
+        14,
+        327,
+        17436,
+        19186,
+        14,
+        327,
+        17436,
+        4816,
+        14,
+        327,
+        17436,
+        4617,
+        14,
+        58839,
+        4694,
+        11070,
+        14,
+        433,
+        11397,
+        14,
+        223,
+        3712,
+        14,
+        223,
+        3712,
+        14,
+        19,
+        3845,
+        15,
+        23,
+        14,
+        19,
+        3845,
+        15,
+        22,
+        39550,
+        5896,
+        18016,
+        98928,
+        5743,
+        868,
+        361,
+        4050,
+        67692,
+        27656,
+        6909,
+        1964,
+        22876,
+        18696,
+        11388,
+        433,
+        4730,
+        2518,
+        67692,
+        67692,
+        67692,
+        67692,
+        8567,
+        361,
+        4050,
+        2581,
+        1928,
+        611,
+        27681,
+        565,
+        778,
+        579,
+        588,
+        1347,
+        304,
+        9502,
+        94260,
+        305,
+        10692,
+        3088,
+        305,
+        322,
+        1715,
+        8889,
+        201,
+        361,
+        310,
+        31,
+        18,
+        16,
+        26,
+        4084,
+        82,
+        201,
+        361,
+        67103,
+        730,
+        438,
+        223,
+        23,
+        16,
+        18,
+        71,
+        23,
+        4084,
+        82,
+        8567,
+        361,
+        855,
+        10,
+        16051,
+        31712,
+        4579,
+        11,
+        9594,
+        201,
+        4694,
+        6734,
+        438,
+        223,
+        18,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        4694,
+        11070,
+        5840,
+        438,
+        11070,
+        201,
+        4694,
+        1895,
+        65,
+        1303,
+        438,
+        223,
+        18,
+        201,
+        4694,
+        9415,
+        4295,
+        438,
+        1204,
+        15444,
+        603,
+        1445,
+        696,
+        1901,
+        43516,
+        2869,
+        33375,
+        432,
+        4295,
+        682,
+        4694,
+        696,
+        44917,
+        92,
+        438,
+        223,
+        19,
+        14,
+        19863,
+        60,
+        20134,
+        201,
+        385,
+        855,
+        343,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        797,
+        23225,
+        16,
+        19,
+        11,
+        9594,
+        3,
+        6112,
+        91,
+        201,
+        9819,
+        855,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        797,
+        23225,
+        16,
+        19,
+        11,
+        9594,
+        201,
+        9212,
+        280,
+        12743,
+        438,
+        3333,
+        12743,
+        565,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        3006,
+        201,
+        9212,
+        280,
+        12743,
+        223,
+        438,
+        11070,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        9908,
+        19,
+        9908,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        67367,
+        201,
+        9819,
+        280,
+        40869,
+        438,
+        11070,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        2542,
+        15,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        688,
+        65,
+        29495,
+        96388,
+        438,
+        688,
+        65,
+        2673,
+        65,
+        499,
+        940,
+        343,
+        650,
+        65,
+        2673,
+        20823,
+        606,
+        28855,
+        65,
+        2673,
+        65,
+        499,
+        8930,
+        70,
+        1282,
+        22785,
+        42935,
+        28739,
+        70,
+        1282,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        20,
+        2542,
+        15,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        868,
+        9819,
+        494,
+        12743,
+        438,
+        688,
+        65,
+        29495,
+        96388,
+        13,
+        21,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        20,
+        23375,
+        12743,
+        4557,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        317,
+        648,
+        730,
+        23252,
+        682,
+        9819,
+        494,
+        40869,
+        438,
+        688,
+        65,
+        29495,
+        96388,
+        13,
+        21,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        20,
+        23375,
+        40869,
+        4557,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        317,
+        648,
+        730,
+        23252,
+        682,
+        9819,
+        494,
+        65,
+        1835,
+        438,
+        343,
+        84,
+        12743,
+        940,
+        494,
+        40869,
+        8930,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        28739,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        15116,
+        84,
+        65,
+        1835,
+        666,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        682,
+        385,
+        116508,
+        343,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        797,
+        23225,
+        16,
+        20,
+        11,
+        9594,
+        3,
+        427,
+        261,
+        201,
+        9819,
+        855,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        797,
+        23225,
+        3225,
+        65,
+        68190,
+        85,
+        11,
+        9594,
+        201,
+        9212,
+        280,
+        40869,
+        438,
+        3333,
+        40869,
+        565,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        3006,
+        201,
+        9212,
+        280,
+        40869,
+        223,
+        438,
+        11070,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        12754,
+        19,
+        9908,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        67367,
+        201,
+        9819,
+        280,
+        12743,
+        438,
+        11070,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        2542,
+        15,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        688,
+        65,
+        29495,
+        96388,
+        438,
+        688,
+        65,
+        2673,
+        65,
+        11943,
+        940,
+        343,
+        650,
+        65,
+        2673,
+        19525,
+        606,
+        28855,
+        65,
+        2673,
+        65,
+        11943,
+        8930,
+        70,
+        1282,
+        22785,
+        42935,
+        28739,
+        70,
+        1282,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        20,
+        2542,
+        15,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        868,
+        9819,
+        494,
+        12743,
+        438,
+        688,
+        65,
+        29495,
+        96388,
+        13,
+        21,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        20,
+        23375,
+        12743,
+        4557,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        317,
+        648,
+        730,
+        23252,
+        682,
+        9819,
+        494,
+        40869,
+        438,
+        688,
+        65,
+        29495,
+        96388,
+        13,
+        21,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        20,
+        23375,
+        40869,
+        4557,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        317,
+        648,
+        730,
+        23252,
+        682,
+        9819,
+        494,
+        65,
+        1835,
+        438,
+        343,
+        84,
+        12743,
+        940,
+        494,
+        40869,
+        8930,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9819,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        28739,
+        650,
+        65,
+        29495,
+        96388,
+        666,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        15116,
+        84,
+        65,
+        1835,
+        666,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        682,
+        385,
+        116508,
+        343,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        19,
+        797,
+        23225,
+        16,
+        21,
+        11,
+        9594,
+        3,
+        17237,
+        26395,
+        201,
+        9819,
+        855,
+        6434,
+        630,
+        26395,
+        36666,
+        15136,
+        4987,
+        83,
+        16,
+        20,
+        11,
+        9594,
+        4050,
+        2581,
+        611,
+        260,
+        4654,
+        70387,
+        2645,
+        305,
+        588,
+        9502,
+        58924,
+        10692,
+        356,
+        69,
+        201,
+        9212,
+        280,
+        12743,
+        438,
+        11070,
+        10,
+        16051,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        2542,
+        15,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9212,
+        280,
+        40869,
+        438,
+        11070,
+        7093,
+        65,
+        68190,
+        85,
+        7550,
+        4367,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        14,
+        21,
+        12754,
+        19,
+        9908,
+        779,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9250,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9212,
+        494,
+        12743,
+        438,
+        2342,
+        65,
+        2673,
+        13,
+        21,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        17237,
+        65,
+        2673,
+        666,
+        20,
+        23375,
+        12743,
+        4557,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        317,
+        648,
+        730,
+        23252,
+        682,
+        9212,
+        494,
+        40869,
+        438,
+        2342,
+        65,
+        2673,
+        13,
+        21,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        17237,
+        65,
+        2673,
+        666,
+        20,
+        23375,
+        40869,
+        4557,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        317,
+        648,
+        730,
+        23252,
+        682,
+        9212,
+        494,
+        65,
+        1835,
+        438,
+        343,
+        84,
+        12743,
+        940,
+        494,
+        40869,
+        8930,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        9212,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        28739,
+        17237,
+        65,
+        2673,
+        666,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        15116,
+        84,
+        65,
+        1835,
+        666,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        682,
+        9819,
+        3006,
+        3,
+        2581,
+        611,
+        411,
+        119919,
+        1875,
+        12283,
+        5034,
+        9703,
+        9662,
+        201,
+        12467,
+        327,
+        17436,
+        4617,
+        3913,
+        80,
+        92,
+        11,
+        438,
+        4632,
+        31712,
+        435,
+        3913,
+        80,
+        92,
+        682,
+        9819,
+        1522,
+        855,
+        201,
+        385,
+        67367,
+        201,
+        528,
+        1522,
+        4016,
+        201,
+        4694,
+        4050,
+        22323,
+        1487,
+        201,
+        4694,
+        1926,
+        8987,
+        66534,
+        443,
+        65,
+        109328,
+        4941,
+        84,
+        10,
+        29280,
+        10273,
+        14,
+        10403,
+        60,
+        20134,
+        14,
+        327,
+        17436,
+        19186,
+        14,
+        327,
+        17436,
+        4816,
+        14,
+        327,
+        17436,
+        4617,
+        14,
+        58839,
+        385,
+        11070,
+        5840,
+        14,
+        433,
+        11397,
+        14,
+        223,
+        3712,
+        14,
+        223,
+        3712,
+        14,
+        19,
+        3845,
+        15,
+        23,
+        14,
+        19,
+        3845,
+        15,
+        22,
+        39550,
+        5896,
+        18016,
+        98928,
+        5743,
+        682,
+        4694,
+        696,
+        1008,
+        438,
+        223,
+        19,
+        14,
+        29280,
+        10273,
+        201,
+        385,
+        6734,
+        438,
+        6734,
+        940,
+        343,
+        24574,
+        5840,
+        6600,
+        11,
+        565,
+        11070,
+        6600,
+        117436,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        11370,
+        2944,
+        6600,
+        22604,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        201,
+        4694,
+        1522,
+        4016,
+        201,
+        4694,
+        6734,
+        438,
+        6734,
+        17,
+        29280,
+        10273,
+        201,
+        4694,
+        11070,
+        438,
+        11070,
+        5840,
+        201,
+        4694,
+        855,
+        30506,
+        3314,
+        86,
+        16,
+        19,
+        16,
+        18,
+        71,
+        15,
+        24,
+        4084,
+        82,
+        11,
+        9594,
+        201,
+        385,
+        9415,
+        4295,
+        438,
+        1204,
+        10634,
+        603,
+        4694,
+        3006,
+        855,
+        343,
+        10499,
+        65,
+        1303,
+        3588,
+        86,
+        16,
+        397,
+        11,
+        9594,
+        201,
+        385,
+        9415,
+        4295,
+        438,
+        1204,
+        10634,
+        603,
+        4694,
+        67367,
+        201,
+        4694,
+        1895,
+        65,
+        1303,
+        438,
+        1895,
+        65,
+        1303,
+        940,
+        223,
+        19,
+        201,
+        4694,
+        1522,
+        4016,
+        4050,
+        8848,
+        201,
+        361,
+        67367,
+        201,
+        361,
+        62932,
+        9538,
+        8860,
+        294,
+        58924,
+        1487,
+        201,
+        361,
+        8860,
+        438,
+        343,
+        22883,
+        12743,
+        3095,
+        781,
+        40869,
+        15116,
+        24574,
+        10,
+        29280,
+        10273,
+        5203,
+        361,
+        62932,
+        61910,
+        4892,
+        294,
+        58924,
+        1487,
+        201,
+        361,
+        16170,
+        29079,
+        438,
+        8860,
+        18390,
+        23043,
+        18390,
+        18,
+        16,
+        3600,
+        4084,
+        82,
+        22604,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        15116,
+        26,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        9159,
+        868,
+        361,
+        855,
+        10,
+        7449,
+        65,
+        32527,
+        1290,
+        11,
+        9594,
+        201,
+        4694,
+        4050,
+        25837,
+        304,
+        4245,
+        201,
+        528,
+        37877,
+        26764,
+        10,
+        553,
+        53540,
+        201,
+        6118,
+        93345,
+        43,
+        24,
+        44261,
+        14,
+        20,
+        23445,
+        27,
+        16,
+        21,
+        44261,
+        754,
+        40,
+        27,
+        16,
+        23,
+        44261,
+        101284,
+        45304,
+        438,
+        905,
+        4484,
+        9,
+        1900,
+        8,
+        47913,
+        43,
+        24,
+        44261,
+        14,
+        22,
+        23445,
+        26,
+        16,
+        20,
+        44261,
+        754,
+        22,
+        23445,
+        26,
+        16,
+        23,
+        44261,
+        754,
+        8,
+        20,
+        23445,
+        553,
+        16,
+        20,
+        44261,
+        754,
+        21,
+        23445,
+        26,
+        16,
+        22,
+        44261,
+        754,
+        43,
+        24,
+        44261,
+        14,
+        20,
+        23445,
+        553,
+        16,
+        23,
+        44261,
+        754,
+        20,
+        23445,
+        26,
+        16,
+        22,
+        44261,
+        754,
+        8,
+        10,
+        40,
+        553,
+        16,
+        20,
+        44261,
+        2542,
+        6528,
+        8,
+        201,
+        385,
+        522,
+        9211,
+        14,
+        22883,
+        12743,
+        14,
+        22883,
+        40869,
+        14,
+        24574,
+        10,
+        29280,
+        10273,
+        682,
+        385,
+        696,
+        1008,
+        438,
+        223,
+        19,
+        14,
+        29280,
+        10273,
+        15,
+        19,
+        201,
+        775,
+        855,
+        6600,
+        3314,
+        86,
+        3225,
+        65,
+        68190,
+        85,
+        11,
+        9594,
+        201,
+        6118,
+        37877,
+        26764,
+        10,
+        553,
+        9943,
+        10,
+        40,
+        27,
+        16,
+        21,
+        44261,
+        101284,
+        45304,
+        438,
+        905,
+        4484,
+        6528,
+        11070,
+        6600,
+        682,
+        775,
+        3006,
+        201,
+        6118,
+        37877,
+        26764,
+        10,
+        553,
+        9943,
+        10,
+        40,
+        27,
+        16,
+        23,
+        44261,
+        101284,
+        45304,
+        438,
+        905,
+        4484,
+        6528,
+        11070,
+        6600,
+        682,
+        775,
+        67367,
+        201,
+        385,
+        1522,
+        4016,
+        201,
+        385,
+        5085,
+        10,
+        553,
+        14,
+        93345,
+        20,
+        26101,
+        24,
+        44261,
+        2542,
+        6528,
+        893,
+        42935,
+        117405,
+        729,
+        6593,
+        3009,
+        201,
+        361,
+        67367,
+        271,
+        361,
+        392,
+        110194,
+        14055,
+        17436,
+        4816,
+        14,
+        59817,
+        31,
+        5480,
+        39997,
+        12329,
+        682,
+        361,
+        392,
+        110194,
+        14055,
+        17436,
+        4617,
+        14,
+        59817,
+        31,
+        5480,
+        39997,
+        12329,
+        682,
+        361,
+        392,
+        110194,
+        14055,
+        17436,
+        19186,
+        14,
+        59817,
+        31,
+        5480,
+        39997,
+        12329,
+        682,
+        361,
+        392,
+        110194,
+        14055,
+        2944,
+        14,
+        59817,
+        31,
+        5480,
+        39997,
+        12329,
+        682,
+        361,
+        392,
+        110194,
+        16569,
+        11397,
+        14,
+        59817,
+        31,
+        5480,
+        39997,
+        12329,
+        682,
+        361,
+        855,
+        10,
+        16051,
+        31712,
+        4579,
+        11,
+        9594,
+        201,
+        4694,
+        392,
+        110194,
+        343,
+        16051,
+        31712,
+        435,
+        14,
+        32759,
+        438,
+        2899,
+        39997,
+        12329,
+        682,
+        4694,
+        392,
+        110194,
+        343,
+        23844,
+        31712,
+        435,
+        14,
+        32759,
+        438,
+        2899,
+        39997,
+        12329,
+        682,
+        4694,
+        392,
+        110194,
+        343,
+        24574,
+        5840,
+        14,
+        32759,
+        438,
+        2899,
+        39997,
+        12329,
+        682,
+        361,
+        67367,
+        201,
+        361,
+        4050,
+        18618,
+        7855,
+        362,
+        6651,
+        305,
+        4433,
+        3078,
+        343,
+        16640,
+        29717,
+        513,
+        1234,
+        3504,
+        304,
+        14531,
+        377,
+        10451,
+        1055,
+        58924,
+        1487,
+        682,
+        361,
+        68006,
+        55291,
+        10,
+        803,
+        65,
+        6593,
+        14,
+        803,
+        9211,
+        11,
+        438,
+        3452,
+        4941,
+        630,
+        65,
+        36916,
+        201,
+        361,
+        68006,
+        55291,
+        10,
+        803,
+        4731,
+        67,
+        14,
+        803,
+        9211,
+        11,
+        438,
+        3452,
+        4941,
+        630,
+        4731,
+        53359,
+        81142,
+        201,
+        361,
+        68006,
+        55291,
+        10,
+        803,
+        65,
+        521,
+        6593,
+        14,
+        803,
+        9211,
+        11,
+        438,
+        3452,
+        65,
+        521,
+        65,
+        36916,
+        201,
+        361,
+        68006,
+        55291,
+        10,
+        803,
+        65,
+        9839,
+        67,
+        14,
+        803,
+        9211,
+        11,
+        438,
+        3452,
+        65,
+        521,
+        4731,
+        53359,
+        81142,
+        201,
+        361,
+        68006,
+        55291,
+        10,
+        803,
+        65,
+        427,
+        261,
+        6593,
+        14,
+        803,
+        9211,
+        11,
+        438,
+        3452,
+        65,
+        427,
+        261,
+        65,
+        36916,
+        201,
+        361,
+        68006,
+        55291,
+        10,
+        803,
+        65,
+        427,
+        1509,
+        67,
+        14,
+        803,
+        9211,
+        11,
+        438,
+        3452,
+        65,
+        427,
+        261,
+        4731,
+        53359,
+        81142,
+        201,
+        361,
+        68006,
+        55291,
+        10,
+        803,
+        29079,
+        14,
+        803,
+        9211,
+        11,
+        438,
+        16170,
+        29079,
+        271,
+        361,
+        1522,
+        855,
+        271,
+        361,
+        1926,
+        6601,
+        24814,
+        279,
+        52529,
+        7907,
+        14,
+        20,
+        682,
+        523,
+        114957,
+        58924,
+        31712,
+        4579,
+        271,
+        8567,
+        3,
+        92334,
+        112236,
+        2596,
+        8567,
+        3,
+        32683,
+        15649,
+        125973,
+        5180,
+        21,
+        201,
+        8567,
+        5635,
+        86653,
+        40187,
+        39711,
+        65,
+        29495,
+        10,
+        29495,
+        58575,
+        68,
+        36323,
+        3487,
+        118528,
+        682,
+        361,
+        1347,
+        30784,
+        201,
+        361,
+        1347,
+        31939,
+        201,
+        361,
+        1347,
+        70077,
+        271,
+        361,
+        2041,
+        61787,
+        754,
+        12756,
+        27680,
+        11,
+        20268,
+        15857,
+        201,
+        361,
+        2041,
+        61787,
+        754,
+        12756,
+        27680,
+        11,
+        20268,
+        310,
+        68,
+        201,
+        361,
+        2041,
+        61787,
+        754,
+        12756,
+        27680,
+        606,
+        11,
+        20268,
+        2332,
+        69,
+        118528,
+        6328,
+        361,
+        2041,
+        61787,
+        11,
+        20268,
+        2703,
+        66580,
+        201,
+        361,
+        2041,
+        61787,
+        11,
+        20268,
+        21474,
+        271,
+        361,
+        3053,
+        23555,
+        31,
+        1683,
+        11,
+        20268,
+        1241,
+        7907,
+        201,
+        361,
+        9132,
+        20268,
+        70077,
+        59238,
+        271,
+        223,
+        1241,
+        7907,
+        438,
+        905,
+        10865,
+        65,
+        10107,
+        15093,
+        65,
+        32527,
+        1290,
+        5832,
+        223,
+        1926,
+        6601,
+        24814,
+        279,
+        52529,
+        7907,
+        14,
+        19,
+        682,
+        223,
+        1926,
+        1178,
+        4084,
+        88729,
+        59238,
+        7164,
+        88729,
+        59238,
+        14532,
+        223,
+        2703,
+        66580,
+        438,
+        223,
+        18,
+        16,
+        1883,
+        4084,
+        82,
+        201,
+        223,
+        21474,
+        438,
+        223,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        4557,
+        19,
+        13,
+        6580,
+        7414,
+        18,
+        16,
+        32224,
+        21,
+        4084,
+        82,
+        18390,
+        20,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        29495,
+        15,
+        24,
+        16,
+        6048,
+        4084,
+        82,
+        28951,
+        223,
+        2332,
+        69,
+        118528,
+        438,
+        343,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        940,
+        343,
+        6580,
+        19177,
+        68,
+        12,
+        13400,
+        9908,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        15116,
+        6580,
+        10,
+        28747,
+        66580,
+        12,
+        13400,
+        9908,
+        19,
+        16,
+        18,
+        4084,
+        82,
+        11,
+        58839,
+        3392,
+        75608,
+        5234,
+        16,
+        18,
+        4084,
+        82,
+        12,
+        6580,
+        7414,
+        20,
+        16,
+        30236,
+        4084,
+        82,
+        12,
+        29495,
+        11,
+        940,
+        223,
+        21,
+        16,
+        18,
+        4084,
+        82,
+        565,
+        223,
+        21,
+        16,
+        1883,
+        4084,
+        82,
+        12,
+        6580,
+        7414,
+        18,
+        16,
+        3315,
+        4084,
+        82,
+        12,
+        29495,
+        2542,
+        8930,
+        22,
+        16,
+        18,
+        4084,
+        82,
+        271,
+        223,
+        1926,
+        6601,
+        24814,
+        279,
+        52529,
+        7907,
+        14,
+        20,
+        682,
+        523,
+        114957,
+        40187,
+        39711,
+        65,
+        29495,
+        13318,
+        523,
+        12124,
+        4935,
+        31712,
+        4579,
+        65,
+        10107,
+        271,
+        1
+      ],
+      "seq_len": 2048
+    }
+  },
+  "config": {
+    "vocab_size": 129280,
+    "dim": 1536,
+    "n_layers": 16,
+    "n_heads": 12,
+    "head_dim": 128,
+    "rope_head_dim": 32,
+    "q_lora_rank": 384,
+    "o_groups": 4,
+    "o_lora_rank": 256,
+    "norm_eps": 1e-06,
+    "n_routed_experts": 12,
+    "n_shared_experts": 1,
+    "n_activated_experts": 2,
+    "moe_inter_dim": 1408,
+    "score_func": "softmax",
+    "route_scale": 1.0,
+    "swiglu_limit": 15.0,
+    "n_dense_layers": 2,
+    "load_balance_alpha": 0.003,
+    "hc_mult": 3,
+    "hc_sinkhorn_iters": 8,
+    "hc_eps": 1e-06,
+    "window_size": 128,
+    "compress_ratio": 4,
+    "n_mtp_layers": 1,
+    "mtp_loss_weight": 0.02,
+    "rope_theta": 10000.0,
+    "rope_factor": 1.0,
+    "original_seq_len": 0,
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "ctx_phase_boundaries": [
+      60000,
+      80000,
+      100000
+    ],
+    "ctx_phase_lengths": [
+      2048,
+      4096,
+      8192
+    ],
+    "yarn_scale": 40.0,
+    "yarn_alpha": 1.0,
+    "yarn_beta": 32.0,
+    "yarn_original_seq_len": 4096,
+    "micro_batch_size": 1,
+    "gradient_accumulation_steps": 32,
+    "max_lr": 0.00015,
+    "min_lr": 1.5e-05,
+    "warmup_steps": 2000,
+    "total_steps": 200000,
+    "decay_steps": 180000,
+    "weight_decay": 0.1,
+    "muon_beta": 0.95,
+    "muon_ns_steps": 7,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.98,
+    "grad_clip_norm": 5.0,
+    "init_std": 0.006,
+    "dtype": "bfloat16",
+    "fineweb_config": "default",
+    "approx_tokens_per_fineweb_doc": 800,
+    "approx_tokens_per_code_doc": 1200,
+    "approx_tokens_per_math_doc": 1500,
+    "data_mix_probs": [
+      0.6,
+      0.25,
+      0.15
+    ],
+    "save_every_steps": 400,
+    "log_every_steps": 10,
+    "push_every_steps": 400,
+    "checkpoint_dir": "/kaggle/working/zenyxv3checkpoints"
+  }
+}

checkpoints/step_18000/opt_state.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83765729e1c94a73460f43f8929e4bf2e808abb7abafc02adc9a1999000db509
+size 3154940698

checkpoints/step_18000/params.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ab0c2ebe725396f0812b86ce985907356d6a9932898b5aa2c15e06adfe95ab5
+size 3154768094