step 20000

checkpoints/step_20000/metadata.json

@@ -0,0 +1,3927 @@
+{
+  "step": 20000,
+  "tokens_consumed": 10485760000,
+  "sequences_consumed": 5120000,
+  "data_state": {
+    "stream_state": {
+      "rng_state": [
+        "MT19937",
+        [
+          307596749,
+          2202902619,
+          562540679,
+          1290061492,
+          2055979458,
+          4145815445,
+          87275957,
+          1772554215,
+          3953466586,
+          1685852852,
+          2995645353,
+          877747995,
+          176946635,
+          552447351,
+          1324785644,
+          2553093553,
+          3214770300,
+          1687227130,
+          4208783966,
+          726987633,
+          1253471070,
+          4138141505,
+          1969911958,
+          2064309688,
+          3627112164,
+          1904278439,
+          3541111013,
+          1592364366,
+          4048656942,
+          2930432068,
+          4112525574,
+          1303331741,
+          3746600496,
+          2093551297,
+          1174502232,
+          159460320,
+          3931592377,
+          520179310,
+          1433219609,
+          2108831070,
+          2522423136,
+          3770132898,
+          3345871754,
+          69833312,
+          2923183758,
+          1660552655,
+          455406244,
+          4153660253,
+          861061288,
+          2234846053,
+          1608334077,
+          3733633781,
+          2353853261,
+          2117962616,
+          4252925939,
+          2646950898,
+          2436816312,
+          3237171672,
+          444481653,
+          3553305577,
+          3650479593,
+          3225213666,
+          1594247659,
+          2435665352,
+          208520227,
+          133384386,
+          1482978130,
+          1549641975,
+          747184455,
+          1808124102,
+          990541901,
+          2216939763,
+          1832830547,
+          1785075103,
+          2317531142,
+          4011379234,
+          989081166,
+          288344782,
+          3684435668,
+          3548380636,
+          2234777005,
+          2235948627,
+          2053863458,
+          841348408,
+          332572955,
+          2163945070,
+          3986456211,
+          853964589,
+          965742164,
+          3158059623,
+          1694701336,
+          2475618199,
+          3808555507,
+          28631214,
+          1074435016,
+          1303357889,
+          1565922909,
+          3835812264,
+          1398004770,
+          3806712997,
+          498669285,
+          1380969428,
+          1130909527,
+          2744877933,
+          1658397617,
+          3260690839,
+          2011694135,
+          1565618942,
+          78430138,
+          4149840759,
+          532596315,
+          3251948409,
+          570713048,
+          3153422496,
+          1113872973,
+          1799773701,
+          1782781267,
+          2050045273,
+          1830341542,
+          1827641635,
+          1190685268,
+          2634856595,
+          2489192274,
+          1460930013,
+          3742756404,
+          3910285013,
+          1207068692,
+          3390996597,
+          3424943024,
+          2557711785,
+          443844393,
+          1058853888,
+          187548635,
+          1801613851,
+          1641790168,
+          1956842456,
+          390276393,
+          2846836828,
+          3021751295,
+          2655576644,
+          3756304961,
+          2724323916,
+          2124852870,
+          3225407868,
+          834017670,
+          1656475964,
+          3102071598,
+          4135194603,
+          1825014407,
+          2553301626,
+          2243590382,
+          1532084132,
+          3702089115,
+          1595610131,
+          2077067511,
+          1770471579,
+          2851834581,
+          2460030622,
+          3473908695,
+          885981515,
+          1298552051,
+          1777250820,
+          546143812,
+          1747842042,
+          2588431885,
+          4070971485,
+          1775148956,
+          1736392216,
+          853131692,
+          193644399,
+          2270115083,
+          191086826,
+          2461305582,
+          1695899617,
+          1933239606,
+          1302959270,
+          3437749851,
+          271803498,
+          3536296435,
+          2848477298,
+          2756433464,
+          3922789332,
+          1903388161,
+          1401817713,
+          577045056,
+          1931919020,
+          3921123152,
+          884235916,
+          2751855271,
+          713829103,
+          3565759016,
+          3329190684,
+          378599207,
+          3541373900,
+          3547814514,
+          136602000,
+          4175239587,
+          129698133,
+          430586456,
+          2859814420,
+          1629183481,
+          48594781,
+          3152677108,
+          912149397,
+          3299717462,
+          3155309887,
+          313469470,
+          3265234326,
+          515708874,
+          2953213339,
+          1014454018,
+          2997428655,
+          1001202084,
+          3628324242,
+          488149736,
+          876085336,
+          2675094200,
+          3932739186,
+          1682237414,
+          3708821729,
+          3455217745,
+          935345140,
+          698123838,
+          1808863117,
+          1962278040,
+          2342126729,
+          1325258171,
+          1402569890,
+          56371371,
+          956875872,
+          1978869945,
+          1204896365,
+          2850078345,
+          4226359797,
+          419712600,
+          3083309478,
+          2752710928,
+          4248904052,
+          222772237,
+          3672352916,
+          4006761008,
+          4292738961,
+          4180426458,
+          2720669584,
+          1400374206,
+          3150770230,
+          912911938,
+          1072744454,
+          4094842377,
+          3971238704,
+          1403322517,
+          428832968,
+          655409795,
+          2501438342,
+          55336971,
+          2212667158,
+          920517634,
+          3452302589,
+          365763965,
+          3347685739,
+          1123911940,
+          582784035,
+          393984599,
+          4284355092,
+          3152909351,
+          2787628108,
+          133632606,
+          2230841988,
+          3986413738,
+          2135085480,
+          4008833330,
+          914414334,
+          1204425264,
+          4148478933,
+          3564626296,
+          3048107413,
+          2917927314,
+          2015131957,
+          3617105527,
+          593926532,
+          7134900,
+          816662038,
+          3656467531,
+          3248900886,
+          3376928071,
+          2792703789,
+          4172426590,
+          2687874900,
+          2188615899,
+          3684562226,
+          2365565864,
+          542969519,
+          1477128817,
+          1721710308,
+          1227759365,
+          1093082608,
+          2973396387,
+          643839726,
+          2604331384,
+          2205186306,
+          1869536233,
+          53465334,
+          3214617124,
+          296298064,
+          2231374007,
+          3517594302,
+          3362827587,
+          1366061067,
+          1913215902,
+          628459952,
+          2985464455,
+          2902967395,
+          656526915,
+          2198827420,
+          340538648,
+          212605511,
+          672785587,
+          2711088453,
+          513176714,
+          3777995094,
+          1161163483,
+          4270231389,
+          1169246189,
+          2511536960,
+          3124225562,
+          2480995022,
+          665439048,
+          4025959610,
+          2509449978,
+          4252715408,
+          3506845615,
+          3219157435,
+          2181427109,
+          4083881070,
+          1181910780,
+          3983536720,
+          4000086140,
+          1252337192,
+          2912865558,
+          2860891074,
+          1874987094,
+          53145710,
+          2089698642,
+          3505910388,
+          3900853721,
+          1537585816,
+          674862660,
+          2432213419,
+          1196195602,
+          3980484754,
+          1577410683,
+          3346542500,
+          1399290486,
+          3702260349,
+          1304296431,
+          1576621146,
+          3095114865,
+          752169100,
+          1048964345,
+          2231638684,
+          2119051106,
+          3422478799,
+          3186215902,
+          3751096670,
+          3706883100,
+          2588187739,
+          2451364379,
+          1895328299,
+          2595640826,
+          982390485,
+          2087345471,
+          2667793689,
+          4191186514,
+          2652729974,
+          4211035299,
+          1616285291,
+          2717345822,
+          3318675010,
+          2188594245,
+          2004546531,
+          3646116251,
+          1649895761,
+          1294153478,
+          744669201,
+          1265401811,
+          3580437877,
+          1407381303,
+          1225163875,
+          104539436,
+          2357502269,
+          564280270,
+          119332124,
+          2118633341,
+          3475677646,
+          3669623903,
+          4137358699,
+          224785264,
+          1676100553,
+          407471092,
+          4052659666,
+          274934719,
+          1407847957,
+          3714372461,
+          227011967,
+          3309250612,
+          1581173153,
+          4190121097,
+          2374170723,
+          2911423511,
+          3365185341,
+          865780674,
+          4017411785,
+          486527569,
+          2015214655,
+          2876049140,
+          2223846773,
+          3394535413,
+          3285984347,
+          2439442390,
+          3773591596,
+          3034638501,
+          972831434,
+          3614664479,
+          2662596855,
+          2773497578,
+          2061383459,
+          3428719318,
+          3809303105,
+          3932086068,
+          2186446717,
+          2124686883,
+          3611233483,
+          338503366,
+          3995674817,
+          2320223907,
+          3588254571,
+          4168586413,
+          1450090361,
+          2498094522,
+          2394419786,
+          3927819523,
+          2308061189,
+          4134636584,
+          3361731027,
+          3391296133,
+          1223352957,
+          3493822156,
+          963571798,
+          2497657878,
+          1787935245,
+          1125418473,
+          3448777737,
+          519406088,
+          3263873803,
+          116979727,
+          3218558437,
+          2322323098,
+          3755746415,
+          2176879816,
+          133718908,
+          1835151283,
+          3636882754,
+          4241095681,
+          3925120947,
+          4116408796,
+          3863385143,
+          3952977471,
+          1404371810,
+          2240897218,
+          1557854224,
+          2373582838,
+          2644915511,
+          3108971841,
+          2343582401,
+          4024782508,
+          3786876508,
+          1268107346,
+          2369248444,
+          609011860,
+          3149168731,
+          1267912421,
+          2199134604,
+          2411320377,
+          2717495275,
+          2667289194,
+          1937720256,
+          1018127008,
+          1700790574,
+          4211873355,
+          3211785761,
+          77756122,
+          2240799426,
+          1842501309,
+          4028271689,
+          2713764582,
+          2448343096,
+          1353545770,
+          462307346,
+          3879528963,
+          2644963549,
+          3302121842,
+          704899648,
+          1439961782,
+          2362892056,
+          3535319411,
+          1005067536,
+          3285634272,
+          208994660,
+          3603551763,
+          1303076860,
+          2422389971,
+          2681262751,
+          3575564484,
+          1061293583,
+          843098612,
+          502230571,
+          3576953385,
+          401982165,
+          730463542,
+          1652440328,
+          3613429858,
+          842615284,
+          3863192708,
+          2537761130,
+          1304337173,
+          95989160,
+          1716013358,
+          2632893910,
+          4063519613,
+          3490901468,
+          2624701036,
+          2464531963,
+          143379171,
+          4083487602,
+          1238995198,
+          3076925047,
+          3146550608,
+          2398561287,
+          3961369582,
+          3378649239,
+          2469024605,
+          2780809040,
+          3727198437,
+          2203060586,
+          2568908803,
+          3421485329,
+          4141936259,
+          2513128666,
+          3193474556,
+          793397960,
+          20185831,
+          2100384132,
+          470938561,
+          2796317406,
+          379526115,
+          1557035123,
+          1832824061,
+          1485010816,
+          533431939,
+          3899647714,
+          2791668408,
+          1973576618,
+          1811988218,
+          791462312,
+          4126009880,
+          4094999711,
+          2788188043,
+          1127143665,
+          494941855,
+          1950586185,
+          2669554660,
+          2002072206,
+          660943915,
+          3721431210,
+          431417558,
+          1831971714,
+          680370914,
+          871415802,
+          1920370842,
+          1297333633,
+          97778404,
+          2219047576,
+          1447414112,
+          2890680120,
+          3270521614,
+          4254857992,
+          191115110,
+          1182152594,
+          197100844,
+          4222385456,
+          4145128785,
+          3722554011,
+          2942493214,
+          2041325907,
+          1695394021,
+          3642264900,
+          3442193708,
+          1191843335,
+          1402845083,
+          4039046550,
+          4163871115,
+          2227435090,
+          3672861405,
+          1990257901,
+          670286256,
+          4255315325,
+          915713579,
+          432920534,
+          287711596,
+          1017508896,
+          590711324,
+          1113039753,
+          529787789,
+          3981564495,
+          243681313,
+          3072141016,
+          3641196906,
+          2656370607,
+          8494873,
+          3378568957,
+          2356793839,
+          817431460,
+          2179725849,
+          2638710089
+        ],
+        478,
+        0,
+        0.0
+      ],
+      "doc_counts": {
+        "fineweb": 1474002,
+        "code": 613764,
+        "math": 367913
+      },
+      "code_lang_idx": 27
+    },
+    "token_state": {
+      "buffer_tokens": [
+        20765,
+        14,
+        11920,
+        201,
+        223,
+        9132,
+        20268,
+        17966,
+        25517,
+        61915,
+        223,
+        3701,
+        31,
+        7042,
+        5868,
+        4951,
+        10,
+        20,
+        12,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        6761,
+        90,
+        12754,
+        20,
+        201,
+        223,
+        855,
+        10,
+        20,
+        12,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        16,
+        4392,
+        77537,
+        20,
+        12,
+        18437,
+        11,
+        3701,
+        31,
+        21,
+        201,
+        3,
+        223,
+        855,
+        10,
+        20,
+        12,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        6761,
+        90,
+        6547,
+        54,
+        16,
+        19,
+        16,
+        23,
+        11,
+        1720,
+        439,
+        31,
+        476,
+        439,
+        18390,
+        20,
+        12,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        6761,
+        90,
+        17,
+        19,
+        16,
+        23,
+        22604,
+        20,
+        29406,
+        223,
+        855,
+        36035,
+        16,
+        302,
+        8146,
+        90,
+        16,
+        12943,
+        12571,
+        16,
+        302,
+        16,
+        2928,
+        11,
+        1539,
+        271,
+        262,
+        1926,
+        3609,
+        82,
+        6600,
+        20765,
+        14,
+        36036,
+        14,
+        1589,
+        14,
+        31847,
+        74690,
+        868,
+        262,
+        855,
+        95626,
+        439,
+        6172,
+        51,
+        16,
+        18,
+        11,
+        1539,
+        201,
+        361,
+        1926,
+        4491,
+        10253,
+        39122,
+        14,
+        36036,
+        14,
+        11920,
+        14,
+        31847,
+        74690,
+        28558,
+        54623,
+        25517,
+        682,
+        262,
+        3006,
+        14320,
+        361,
+        1926,
+        39269,
+        755,
+        10253,
+        39122,
+        14,
+        36036,
+        14,
+        11920,
+        14,
+        31847,
+        74690,
+        682,
+        262,
+        67367,
+        201,
+        14320,
+        262,
+        1926,
+        1258,
+        20859,
+        439,
+        10253,
+        11220,
+        359,
+        14,
+        1425,
+        14,
+        10208,
+        20197,
+        8137,
+        14,
+        476,
+        83333,
+        27993,
+        20765,
+        14,
+        11920,
+        28558,
+        624,
+        14,
+        476,
+        439,
+        682,
+        14320,
+        223,
+        67367,
+        201,
+        14320,
+        1522,
+        114957,
+        271,
+        114957,
+        1258,
+        82,
+        10253,
+        11220,
+        359,
+        14,
+        1425,
+        14,
+        10208,
+        27993,
+        20765,
+        14,
+        11920,
+        29532,
+        10253,
+        28558,
+        624,
+        682,
+        223,
+        9132,
+        3157,
+        10,
+        20,
+        754,
+        1425,
+        10,
+        20,
+        754,
+        75,
+        20765,
+        14,
+        11920,
+        19306,
+        19,
+        19306,
+        20,
+        28558,
+        19,
+        28558,
+        20,
+        201,
+        3,
+        223,
+        9132,
+        3157,
+        14,
+        1425,
+        27993,
+        20765,
+        14,
+        11920,
+        19306,
+        19,
+        19306,
+        20,
+        28558,
+        19,
+        28558,
+        20,
+        201,
+        223,
+        2041,
+        4273,
+        29532,
+        10253,
+        27526,
+        62155,
+        754,
+        10208,
+        27526,
+        62155,
+        682,
+        2143,
+        313,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19577,
+        10,
+        19,
+        9908,
+        75,
+        3171,
+        313,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        1425,
+        10,
+        19,
+        9908,
+        75,
+        682,
+        292,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19577,
+        10,
+        20,
+        9908,
+        76,
+        3171,
+        292,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        1425,
+        10,
+        20,
+        9908,
+        76,
+        682,
+        3,
+        313,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19577,
+        24207,
+        3171,
+        313,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        1425,
+        24207,
+        682,
+        3,
+        292,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19,
+        19477,
+        3171,
+        292,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        2928,
+        19477,
+        682,
+        33196,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        10208,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        12754,
+        49967,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        682,
+        2143,
+        1522,
+        114957,
+        271,
+        114957,
+        1258,
+        20859,
+        439,
+        10253,
+        11220,
+        359,
+        14,
+        1425,
+        14,
+        10208,
+        29532,
+        10253,
+        20197,
+        1751,
+        27993,
+        20765,
+        14,
+        11920,
+        28558,
+        624,
+        14,
+        476,
+        439,
+        682,
+        223,
+        9132,
+        3157,
+        10,
+        20,
+        754,
+        1425,
+        10,
+        20,
+        754,
+        75,
+        20765,
+        14,
+        11920,
+        19306,
+        19,
+        19306,
+        20,
+        28558,
+        19,
+        28558,
+        20,
+        201,
+        3,
+        223,
+        9132,
+        3157,
+        14,
+        1425,
+        27993,
+        20765,
+        14,
+        11920,
+        19306,
+        19,
+        19306,
+        20,
+        28558,
+        19,
+        28558,
+        20,
+        201,
+        223,
+        2041,
+        4273,
+        14,
+        476,
+        439,
+        29532,
+        10253,
+        27526,
+        62155,
+        754,
+        10208,
+        27526,
+        62155,
+        754,
+        22871,
+        27526,
+        62155,
+        682,
+        2143,
+        313,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19577,
+        10,
+        19,
+        9908,
+        75,
+        3171,
+        313,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        1425,
+        10,
+        19,
+        9908,
+        75,
+        682,
+        292,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19577,
+        10,
+        20,
+        9908,
+        76,
+        3171,
+        292,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        1425,
+        10,
+        20,
+        9908,
+        76,
+        682,
+        3,
+        313,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19577,
+        24207,
+        3171,
+        313,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        1425,
+        24207,
+        682,
+        3,
+        292,
+        19,
+        118411,
+        10,
+        19,
+        14,
+        19,
+        19477,
+        3171,
+        292,
+        20,
+        31,
+        1930,
+        36035,
+        14,
+        2928,
+        19477,
+        868,
+        3,
+        855,
+        95626,
+        439,
+        3588,
+        86,
+        16,
+        18,
+        11,
+        67656,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        22871,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        10563,
+        6580,
+        7414,
+        476,
+        439,
+        17,
+        20,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        5203,
+        3,
+        855,
+        95626,
+        439,
+        3588,
+        86,
+        16,
+        18,
+        11,
+        67656,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        22871,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        10563,
+        6580,
+        7414,
+        476,
+        439,
+        17,
+        20,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        15531,
+        223,
+        855,
+        5444,
+        624,
+        72944,
+        16,
+        18,
+        16,
+        458,
+        85098,
+        439,
+        3588,
+        86,
+        16,
+        18,
+        11,
+        1539,
+        201,
+        3,
+        262,
+        67656,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        22871,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        28739,
+        19,
+        9824,
+        1930,
+        10,
+        19,
+        1042,
+        476,
+        439,
+        17,
+        20,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        28951,
+        262,
+        33196,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        10208,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        12754,
+        49967,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        10563,
+        22871,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        682,
+        262,
+        67656,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        22871,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        28739,
+        19,
+        9824,
+        1930,
+        10,
+        19,
+        1042,
+        476,
+        439,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        28951,
+        262,
+        1354,
+        201,
+        223,
+        67367,
+        201,
+        223,
+        855,
+        5444,
+        624,
+        72944,
+        16,
+        18,
+        11,
+        33196,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        10208,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        12754,
+        49967,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        10563,
+        22871,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        682,
+        223,
+        855,
+        95626,
+        439,
+        3588,
+        86,
+        16,
+        18,
+        11,
+        67656,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        6912,
+        22871,
+        6600,
+        47137,
+        19,
+        72004,
+        47137,
+        20,
+        20765,
+        63065,
+        19,
+        28,
+        76,
+        63065,
+        20,
+        28739,
+        19,
+        9824,
+        1930,
+        10,
+        19,
+        1042,
+        476,
+        439,
+        23375,
+        10253,
+        3913,
+        19,
+        33405,
+        20,
+        28558,
+        19,
+        97717,
+        20,
+        28951,
+        2143,
+        1522,
+        114957,
+        6328,
+        114957,
+        39269,
+        755,
+        10253,
+        7093,
+        10253,
+        14,
+        36036,
+        14,
+        11920,
+        14,
+        31847,
+        74690,
+        682,
+        223,
+        9132,
+        20268,
+        3701,
+        201,
+        223,
+        2041,
+        20268,
+        19469,
+        10,
+        20,
+        754,
+        31847,
+        74690,
+        29532,
+        10253,
+        27526,
+        62155,
+        682,
+        223,
+        2041,
+        20268,
+        494,
+        14,
+        11648,
+        32244,
+        76,
+        32244,
+        767,
+        19306,
+        640,
+        14,
+        34642,
+        88,
+        201,
+        223,
+        2041,
+        20268,
+        41498,
+        15034,
+        91,
+        40513,
+        90,
+        10,
+        21,
+        754,
+        52597,
+        10,
+        21,
+        682,
+        223,
+        9132,
+        20268,
+        1008,
+        20765,
+        67130,
+        223,
+        855,
+        36035,
+        4987,
+        83,
+        16,
+        21,
+        11,
+        1539,
+        201,
+        3,
+        96,
+        361,
+        1008,
+        31,
+        20,
+        30208,
+        31,
+        20,
+        201,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        19,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        6547,
+        54,
+        3845,
+        90,
+        17,
+        20,
+        11,
+        1008,
+        31,
+        19,
+        201,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        20,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        6547,
+        54,
+        3845,
+        90,
+        17,
+        20,
+        11,
+        597,
+        31,
+        19,
+        201,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        19,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8995,
+        54,
+        16,
+        21,
+        12,
+        18437,
+        17,
+        20,
+        5136,
+        1008,
+        31,
+        21,
+        201,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        20,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8995,
+        54,
+        16,
+        21,
+        12,
+        18437,
+        17,
+        20,
+        5136,
+        597,
+        31,
+        21,
+        201,
+        3,
+        96,
+        361,
+        280,
+        10253,
+        10,
+        19,
+        28,
+        21,
+        14,
+        19,
+        28,
+        21,
+        6912,
+        18,
+        201,
+        3,
+        96,
+        361,
+        280,
+        10253,
+        6600,
+        20765,
+        6912,
+        19,
+        201,
+        262,
+        41498,
+        14574,
+        36036,
+        10,
+        19,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8930,
+        18437,
+        201,
+        262,
+        104289,
+        14574,
+        36036,
+        10,
+        20,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8930,
+        18437,
+        14320,
+        262,
+        106376,
+        10,
+        19,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        15,
+        7234,
+        5203,
+        262,
+        106376,
+        10,
+        20,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        15,
+        10284,
+        4042,
+        90,
+        15,
+        19,
+        28951,
+        262,
+        106376,
+        10,
+        21,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        7234,
+        15,
+        19,
+        5203,
+        262,
+        96975,
+        10,
+        19,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        5200,
+        91,
+        5203,
+        262,
+        96975,
+        10,
+        20,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        15,
+        10284,
+        15138,
+        91,
+        15,
+        19,
+        28951,
+        262,
+        96975,
+        10,
+        21,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        20044,
+        15,
+        19,
+        5203,
+        262,
+        696,
+        1008,
+        31,
+        19,
+        14,
+        21,
+        201,
+        290,
+        696,
+        597,
+        31,
+        19,
+        14,
+        21,
+        201,
+        4694,
+        280,
+        10253,
+        6600,
+        20765,
+        6912,
+        58599,
+        6600,
+        10563,
+        52597,
+        17776,
+        682,
+        290,
+        1522,
+        4016,
+        201,
+        262,
+        1522,
+        4016,
+        201,
+        262,
+        1354,
+        201,
+        223,
+        67367,
+        1004,
+        223,
+        3829,
+        31,
+        18,
+        603,
+        223,
+        38991,
+        88,
+        31,
+        19,
+        3625,
+        85,
+        74690,
+        201,
+        223,
+        696,
+        597,
+        31,
+        19,
+        14,
+        11920,
+        201,
+        262,
+        108586,
+        31,
+        7414,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        14864,
+        3284,
+        10,
+        20,
+        62024,
+        76,
+        15,
+        19,
+        10563,
+        18437,
+        22604,
+        20,
+        201,
+        262,
+        696,
+        1008,
+        31,
+        19,
+        14,
+        11920,
+        201,
+        1445,
+        1298,
+        19042,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        14864,
+        3284,
+        10,
+        19,
+        62024,
+        75,
+        15,
+        19,
+        10563,
+        18437,
+        5191,
+        1445,
+        494,
+        79095,
+        51,
+        23085,
+        7164,
+        75,
+        666,
+        20,
+        61963,
+        76,
+        10563,
+        34642,
+        88,
+        201,
+        1445,
+        21635,
+        49016,
+        87846,
+        13027,
+        682,
+        1445,
+        3829,
+        31,
+        31948,
+        13,
+        47707,
+        201,
+        1445,
+        280,
+        10253,
+        6600,
+        20765,
+        6912,
+        47707,
+        201,
+        262,
+        1522,
+        4016,
+        201,
+        223,
+        1522,
+        4016,
+        2143,
+        223,
+        855,
+        3913,
+        640,
+        6172,
+        51,
+        16,
+        18,
+        11,
+        223,
+        1926,
+        20669,
+        10,
+        24,
+        682,
+        223,
+        280,
+        10253,
+        10,
+        19,
+        28,
+        11920,
+        14,
+        19,
+        28,
+        11920,
+        6912,
+        82,
+        10253,
+        10,
+        19,
+        28,
+        11920,
+        14,
+        19,
+        28,
+        11920,
+        8930,
+        31948,
+        271,
+        1522,
+        114957,
+        271,
+        114957,
+        3609,
+        82,
+        6600,
+        20765,
+        14,
+        36036,
+        14,
+        1589,
+        14,
+        31847,
+        74690,
+        682,
+        223,
+        9132,
+        20268,
+        1008,
+        20765,
+        201,
+        223,
+        2041,
+        20268,
+        19469,
+        10,
+        20,
+        754,
+        1589,
+        10,
+        21,
+        754,
+        31847,
+        74690,
+        201,
+        223,
+        2041,
+        1527,
+        15034,
+        201,
+        223,
+        1527,
+        31,
+        1589,
+        10,
+        19,
+        11,
+        3749,
+        383,
+        31,
+        1589,
+        10,
+        20,
+        682,
+        5191,
+        223,
+        1008,
+        72722,
+        10973,
+        2508,
+        6980,
+        90,
+        105027,
+        90,
+        23252,
+        85,
+        74690,
+        34705,
+        11920,
+        17,
+        20,
+        5136,
+        17,
+        18437,
+        62655,
+        15,
+        19,
+        201,
+        223,
+        597,
+        72722,
+        10973,
+        2508,
+        6980,
+        91,
+        105027,
+        90,
+        23252,
+        85,
+        74690,
+        13,
+        922,
+        1387,
+        17,
+        20,
+        5136,
+        17,
+        18437,
+        62655,
+        15,
+        19,
+        201,
+        223,
+        19469,
+        10,
+        19,
+        51046,
+        90,
+        105027,
+        90,
+        23252,
+        85,
+        74690,
+        34705,
+        11920,
+        17,
+        20,
+        5136,
+        17219,
+        75,
+        10563,
+        18437,
+        4050,
+        10,
+        75,
+        13,
+        19,
+        682,
+        223,
+        19469,
+        10,
+        20,
+        51046,
+        91,
+        105027,
+        90,
+        23252,
+        85,
+        74690,
+        13,
+        922,
+        1387,
+        17,
+        20,
+        5136,
+        17219,
+        76,
+        10563,
+        18437,
+        4050,
+        10,
+        76,
+        13,
+        19,
+        682,
+        1522,
+        114957,
+        271,
+        5635,
+        86653,
+        31358,
+        37691,
+        10225,
+        201,
+        2041,
+        20268,
+        2263,
+        14,
+        5920,
+        14,
+        664,
+        28558,
+        10,
+        18,
+        71112,
+        318,
+        1896,
+        630,
+        682,
+        9132,
+        20268,
+        313,
+        27993,
+        20765,
+        77968,
+        313,
+        87413,
+        318,
+        1896,
+        630,
+        201,
+        2263,
+        31,
+        9884,
+        10,
+        20,
+        5136,
+        17,
+        80,
+        201,
+        11881,
+        31,
+        19,
+        6984,
+        80,
+        77968,
+        292,
+        10,
+        18,
+        6912,
+        18,
+        201,
+        696,
+        1008,
+        31,
+        19,
+        19306,
+        201,
+        223,
+        292,
+        6600,
+        6912,
+        79,
+        6600,
+        15,
+        19,
+        12754,
+        16,
+        23,
+        12,
+        15979,
+        18390,
+        15979,
+        18390,
+        75,
+        15,
+        19,
+        10563,
+        85,
+        87846,
+        7164,
+        84,
+        18390,
+        75,
+        15,
+        19,
+        127522,
+        15979,
+        22409,
+        21494,
+        87846,
+        7164,
+        84,
+        22409,
+        5203,
+        1522,
+        4016,
+        271,
+        4924,
+        10,
+        18,
+        6912,
+        18,
+        603,
+        4924,
+        3913,
+        6912,
+        15979,
+        20735,
+        201,
+        696,
+        1008,
+        31,
+        19,
+        19306,
+        15,
+        19,
+        201,
+        223,
+        597,
+        31,
+        18,
+        201,
+        223,
+        696,
+        1901,
+        5444,
+        17776,
+        8930,
+        79,
+        3913,
+        797,
+        6306,
+        3845,
+        72,
+        22409,
+        682,
+        262,
+        597,
+        96984,
+        13,
+        19,
+        201,
+        223,
+        1522,
+        4016,
+        201,
+        223,
+        2937,
+        14574,
+        5920,
+        22409,
+        23236,
+        3913,
+        9908,
+        79,
+        17776,
+        15,
+        19,
+        46835,
+        79,
+        17776,
+        9908,
+        79,
+        17776,
+        15,
+        19,
+        5203,
+        223,
+        4924,
+        6600,
+        51046,
+        19,
+        59151,
+        10563,
+        15979,
+        18390,
+        76,
+        15,
+        19,
+        12754,
+        664,
+        12,
+        15979,
+        24580,
+        201,
+        1522,
+        4016,
+        1004,
+        523,
+        114957,
+        271,
+        5635,
+        86653,
+        4491,
+        10253,
+        7093,
+        10253,
+        14,
+        36036,
+        14,
+        11920,
+        14,
+        31847,
+        74690,
+        28558,
+        54623,
+        25517,
+        682,
+        223,
+        9132,
+        20268,
+        3701,
+        201,
+        223,
+        2041,
+        20268,
+        19469,
+        10,
+        20,
+        754,
+        31847,
+        74690,
+        29532,
+        10253,
+        27526,
+        62155,
+        682,
+        223,
+        9132,
+        20268,
+        473,
+        27993,
+        20765,
+        14,
+        1423,
+        14,
+        10042,
+        14,
+        328,
+        201,
+        223,
+        2041,
+        20268,
+        116720,
+        21674,
+        10,
+        20,
+        754,
+        15979,
+        28558,
+        388,
+        16181,
+        2459,
+        19306,
+        640,
+        29532,
+        52,
+        4928,
+        201,
+        223,
+        2041,
+        20268,
+        41498,
+        15034,
+        91,
+        40513,
+        90,
+        10,
+        21,
+        754,
+        52597,
+        10,
+        21,
+        682,
+        223,
+        9132,
+        14,
+        10767,
+        20268,
+        39350,
+        21611,
+        31,
+        22,
+        201,
+        223,
+        2041,
+        14,
+        10767,
+        20268,
+        11533,
+        61237,
+        31,
+        19,
+        16,
+        23,
+        201,
+        223,
+        9132,
+        17966,
+        25517,
+        271,
+        223,
+        855,
+        36035,
+        4987,
+        83,
+        16,
+        21,
+        11,
+        1539,
+        201,
+        3,
+        96,
+        361,
+        1008,
+        31,
+        20,
+        30208,
+        31,
+        20,
+        14320,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        19,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        6547,
+        54,
+        3845,
+        90,
+        17,
+        20,
+        11,
+        1008,
+        31,
+        19,
+        201,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        20,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        6547,
+        54,
+        3845,
+        90,
+        17,
+        20,
+        11,
+        597,
+        31,
+        19,
+        201,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        19,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8995,
+        54,
+        16,
+        21,
+        12,
+        18437,
+        17,
+        20,
+        5136,
+        1008,
+        31,
+        21,
+        201,
+        3,
+        96,
+        361,
+        855,
+        10,
+        36036,
+        10,
+        20,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8995,
+        54,
+        16,
+        21,
+        12,
+        18437,
+        17,
+        20,
+        5136,
+        597,
+        31,
+        21,
+        201,
+        3,
+        96,
+        361,
+        280,
+        10253,
+        10,
+        19,
+        28,
+        21,
+        14,
+        19,
+        28,
+        21,
+        6912,
+        18,
+        201,
+        3,
+        96,
+        361,
+        280,
+        10253,
+        6600,
+        20765,
+        6912,
+        19,
+        201,
+        262,
+        41498,
+        14574,
+        36036,
+        10,
+        19,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8930,
+        18437,
+        201,
+        262,
+        104289,
+        14574,
+        36036,
+        10,
+        20,
+        12754,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        8930,
+        18437,
+        14320,
+        262,
+        106376,
+        10,
+        19,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        15,
+        7234,
+        5203,
+        262,
+        106376,
+        10,
+        20,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        15,
+        10284,
+        4042,
+        90,
+        15,
+        19,
+        28951,
+        262,
+        106376,
+        10,
+        21,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        7234,
+        15,
+        19,
+        5203,
+        262,
+        96975,
+        10,
+        19,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        5200,
+        91,
+        5203,
+        262,
+        96975,
+        10,
+        20,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        19,
+        15,
+        10284,
+        15138,
+        91,
+        15,
+        19,
+        28951,
+        262,
+        96975,
+        10,
+        21,
+        6912,
+        9047,
+        10,
+        18,
+        1042,
+        1930,
+        10,
+        19,
+        1042,
+        20044,
+        15,
+        19,
+        5203,
+        262,
+        696,
+        1008,
+        31,
+        19,
+        14,
+        21,
+        201,
+        290,
+        696,
+        597,
+        31,
+        19,
+        14,
+        21,
+        201,
+        4694,
+        280,
+        10253,
+        6600,
+        20765,
+        6912,
+        58599,
+        6600,
+        10563,
+        52597,
+        17776,
+        682,
+        290,
+        1522,
+        4016,
+        201,
+        262,
+        1522,
+        4016,
+        201,
+        262,
+        1354,
+        201,
+        223,
+        67367,
+        1004,
+        223,
+        641,
+        31,
+        18,
+        201,
+        223,
+        3829,
+        31,
+        18,
+        603,
+        223,
+        280,
+        10253,
+        10,
+        19,
+        28,
+        11920,
+        14,
+        19,
+        28,
+        11920,
+        6912,
+        18,
+        603,
+        223,
+        696,
+        473,
+        31,
+        18,
+        14,
+        289,
+        21611,
+        201,
+        262,
+        11533,
+        31,
+        19,
+        6984,
+        92555,
+        61237,
+        666,
+        77,
+        201,
+        262,
+        696,
+        1008,
+        31,
+        19,
+        14,
+        20,
+        666,
+        77,
+        201,
+        361,
+        116720,
+        31,
+        20,
+        201,
+        361,
+        696,
+        1901,
+        13027,
+        84,
+        3588,
+        86,
+        16,
+        19,
+        682,
+        1445,
+        1527,
+        10,
+        19,
+        6912,
+        82,
+        52,
+        4928,
+        5444,
+        54623,
+        25517,
+        13,
+        20,
+        12,
+        328,
+        682,
+        1445,
+        1527,
+        10,
+        20,
+        6912,
+        82,
+        52,
+        4928,
+        10,
+        21,
+        23236,
+        54623,
+        25517,
+        13,
+        20,
+        12,
+        328,
+        13,
+        19,
+        682,
+        1445,
+        1527,
+        33577,
+        12,
+        20,
+        15,
+        19,
+        201,
+        1445,
+        116720,
+        31,
+        5674,
+        4042,
+        666,
+        20,
+        682,
+        1445,
+        641,
+        31,
+        328,
+        13,
+        19,
+        201,
+        361,
+        1522,
+        4016,
+        201,
+        361,
+        597,
+        31,
+        31386,
+        12,
+        48,
+        318,
+        1896,
+        630,
+        201,
+        361,
+        2263,
+        31,
+        31386,
+        12,
+        48,
+        318,
+        1896,
+        630,
+        19477,
+        201,
+        361,
+        13611,
+        14574,
+        19,
+        18960,
+        10563,
+        7216,
+        17776,
+        12754,
+        15979,
+        12,
+        7216,
+        17776,
+        13,
+        19,
+        682,
+        361,
+        1527,
+        31,
+        31847,
+        74690,
+        13912,
+        23236,
+        388,
+        2283,
+        7968,
+        13027,
+        84,
+        682,
+        361,
+        1527,
+        33577,
+        13,
+        36036,
+        13,
+        9047,
+        90,
+        23252,
+        85,
+        74690,
+        201,
+        361,
+        82866,
+        87413,
+        15918,
+        4042,
+        10,
+        19,
+        8930,
+        18437,
+        12754,
+        19,
+        201,
+        361,
+        1008,
+        91,
+        87413,
+        15918,
+        4042,
+        10,
+        20,
+        8930,
+        18437,
+        12754,
+        19,
+        201,
+        361,
+        855,
+        10,
+        1423,
+        3314,
+        86,
+        16,
+        19,
+        115496,
+        16,
+        1423,
+        3588,
+        86,
+        12571,
+        115496,
+        16,
+        10042,
+        3314,
+        86,
+        16,
+        19,
+        115496,
+        16,
+        10042,
+        3588,
+        86,
+        12571,
+        11,
+        1539,
+        201,
+        3,
+        6,
+        36760,
+        6490,
+        201,
+        1445,
+        2777,
+        12,
+        9943,
+        27965,
+        798,
+        294,
+        3291,
+        5832,
+        1445,
+        2777,
+        35260,
+        1527,
+        14,
+        1423,
+        14,
+        10042,
+        201,
+        1445,
+        6409,
+        201,
+        3,
+        6,
+        36760,
+        1522,
+        6490,
+        40637,
+        361,
+        67367,
+        539,
+        361,
+        280,
+        10253,
+        10,
+        1423,
+        14,
+        10042,
+        6912,
+        82,
+        10253,
+        10,
+        1423,
+        14,
+        10042,
+        12754,
+        92555,
+        201,
+        361,
+        3829,
+        31,
+        31948,
+        25846,
+        2459,
+        201,
+        262,
+        1522,
+        4016,
+        201,
+        223,
+        1522,
+        4016,
+        201,
+        223,
+        855,
+        3913,
+        640,
+        16,
+        302,
+        16,
+        18,
+        11,
+        1539,
+        201,
+        262,
+        2777,
+        35260,
+        31948,
+        201,
+        262,
+        6409,
+        201,
+        223,
+        67367,
+        2143,
+        223,
+        280,
+        10253,
+        10,
+        19,
+        28,
+        11920,
+        14,
+        19,
+        28,
+        11920,
+        6912,
+        82,
+        10253,
+        10,
+        19,
+        28,
+        11920,
+        14,
+        19,
+        28,
+        11920,
+        8930,
+        31948,
+        201,
+        523,
+        114957,
+        80267,
+        2019,
+        13899,
+        560,
+        13027,
+        11,
+        1529,
+        19078,
+        682,
+        223,
+        2041,
+        20268,
+        494,
+        57479,
+        201,
+        223,
+        281,
+        31,
+        18,
+        201,
+        3,
+        223,
+        855,
+        13027,
+        8995,
+        54,
+        16,
+        20,
+        11,
+        1354,
+        201,
+        223,
+        855,
+        13027,
+        8995,
+        54,
+        77537,
+        20,
+        11,
+        1354,
+        201,
+        223,
+        281,
+        31,
+        19,
+        15,
+        21,
+        17,
+        20,
+        16550,
+        84,
+        666,
+        20,
+        13,
+        9884,
+        20,
+        75,
+        24010,
+        666,
+        21,
+        271,
+        3,
+        223,
+        855,
+        13027,
+        6547,
+        54,
+        16,
+        19,
+        11,
+        1539,
+        201,
+        3,
+        262,
+        281,
+        31,
+        19,
+        15,
+        21,
+        17,
+        20,
+        16550,
+        84,
+        666,
+        20,
+        13,
+        21,
+        17,
+        22,
+        16550,
+        84,
+        666,
+        21,
+        201,
+        3,
+        223,
+        3006,
+        201,
+        3,
+        262,
+        281,
+        31,
+        19,
+        17,
+        22,
+        16550,
+        10,
+        20,
+        9954,
+        22604,
+        21,
+        201,
+        3,
+        223,
+        67367,
+        201,
+        3,
+        262,
+        281,
+        31,
+        89,
+        12,
+        553,
+        4557,
+        25,
+        12,
+        36375,
+        682,
+        1522,
+        2019,
+        77968,
+        2041,
+        2019,
+        3829,
+        3913,
+        682,
+        223,
+        2041,
+        313,
+        10,
+        21,
+        682,
+        262,
+        3829,
+        79095,
+        51,
+        23085,
+        47876,
+        3913,
+        666,
+        20,
+        5203,
+        1522,
+        2019,
+        271,
+        114957,
+        82182,
+        3913,
+        682,
+        223,
+        2041,
+        313,
+        10,
+        21,
+        754,
+        88,
+        201,
+        262,
+        374,
+        31,
+        31948,
+        3913,
+        682,
+        262,
+        313,
+        10,
+        19,
+        6912,
+        80,
+        10,
+        19,
+        8930,
+        88,
+        201,
+        262,
+        313,
+        10,
+        20,
+        6912,
+        80,
+        10,
+        20,
+        8930,
+        88,
+        201,
+        262,
+        313,
+        10,
+        21,
+        6912,
+        80,
+        10,
+        21,
+        8930,
+        88,
+        201,
+        1522,
+        114957,
+        271,
+        114957,
+        15924,
+        39053,
+        36323,
+        57479,
+        682,
+        223,
+        2041,
+        374,
+        10,
+        21,
+        754,
+        89,
+        10,
+        21,
+        754,
+        81,
+        10,
+        21,
+        682,
+        223,
+        277,
+        10,
+        19,
+        6912,
+        88,
+        10,
+        20,
+        10563,
+        89,
+        10,
+        21,
+        9908,
+        89,
+        10,
+        20,
+        10563,
+        88,
+        10,
+        21,
+        682,
+        223,
+        277,
+        10,
+        20,
+        6912,
+        88,
+        10,
+        21,
+        10563,
+        89,
+        10,
+        19,
+        9908,
+        89,
+        10,
+        21,
+        10563,
+        88,
+        10,
+        19,
+        682,
+        223,
+        277,
+        10,
+        21,
+        6912,
+        88,
+        10,
+        19,
+        10563,
+        89,
+        10,
+        20,
+        9908,
+        89,
+        10,
+        19,
+        10563,
+        88,
+        10,
+        20,
+        682,
+        1522,
+        114957,
+        6328,
+        5635,
+        86653,
+        71543,
+        317,
+        6600,
+        19306,
+        19306,
+        90,
+        14,
+        2928,
+        14,
+        19577,
+        14,
+        1425,
+        682,
+        9132,
+        2366,
+        313,
+        32244,
+        10446,
+        19,
+        32244,
+        10446,
+        20,
+        19306,
+        90,
+        14,
+        2928,
+        27993,
+        14,
+        1423,
+        14,
+        10042,
+        201,
+        9132,
+        20268,
+        3157,
+        10,
+        20,
+        754,
+        1425,
+        10,
+        20,
+        682,
+        539,
+        1543,
+        264,
+        19,
+        36259,
+        26584,
+        18,
+        16,
+        23,
+        682,
+        696,
+        1901,
+        127146,
+        3913,
+        32244,
+        10446,
+        19,
+        797,
+        18916,
+        16,
+        18,
+        16,
+        12943,
+        3845,
+        10446,
+        19,
+        8995,
+        54,
+        16,
+        19,
+        682,
+        223,
+        1543,
+        264,
+        19,
+        59107,
+        10446,
+        19,
+        15,
+        19,
+        201,
+        1522,
+        4016,
+        201,
+        1543,
+        264,
+        20,
+        36259,
+        6761,
+        10446,
+        19,
+        271,
+        223,
+        1008,
+        91,
+        31,
+        5158,
+        6600,
+        32244,
+        10446,
+        19,
+        682,
+        223,
+        82866,
+        51959,
+        6761,
+        10446,
+        19,
+        271,
+        223,
+        3157,
+        10,
+        19,
+        6912,
+        19,
+        27374,
+        65469,
+        12,
+        1423,
+        8930,
+        70,
+        10446,
+        20,
+        201,
+        223,
+        890,
+        10,
+        19,
+        51046,
+        65469,
+        18390,
+        1423,
+        13,
+        19,
+        58688,
+        70,
+        10446,
+        20,
+        201,
+        223,
+        3157,
+        10,
+        20,
+        6912,
+        19,
+        27374,
+        2928,
+        12,
+        10042,
+        8930,
+        70,
+        10446,
+        19,
+        201,
+        223,
+        890,
+        10,
+        20,
+        51046,
+        2928,
+        18390,
+        10042,
+        13,
+        19,
+        58688,
+        70,
+        10446,
+        19,
+        271,
+        523,
+        114957,
+        6328,
+        523,
+        12124,
+        201,
+        1
+      ],
+      "seq_len": 2048
+    }
+  },
+  "config": {
+    "vocab_size": 129280,
+    "dim": 1536,
+    "n_layers": 16,
+    "n_heads": 12,
+    "head_dim": 128,
+    "rope_head_dim": 32,
+    "q_lora_rank": 384,
+    "o_groups": 4,
+    "o_lora_rank": 256,
+    "norm_eps": 1e-06,
+    "n_routed_experts": 12,
+    "n_shared_experts": 1,
+    "n_activated_experts": 2,
+    "moe_inter_dim": 1408,
+    "score_func": "softmax",
+    "route_scale": 1.0,
+    "swiglu_limit": 15.0,
+    "n_dense_layers": 2,
+    "load_balance_alpha": 0.003,
+    "hc_mult": 3,
+    "hc_sinkhorn_iters": 8,
+    "hc_eps": 1e-06,
+    "window_size": 128,
+    "compress_ratio": 4,
+    "n_mtp_layers": 1,
+    "mtp_loss_weight": 0.02,
+    "rope_theta": 10000.0,
+    "rope_factor": 1.0,
+    "original_seq_len": 0,
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "ctx_phase_boundaries": [
+      60000,
+      80000,
+      100000
+    ],
+    "ctx_phase_lengths": [
+      2048,
+      4096,
+      8192
+    ],
+    "yarn_scale": 40.0,
+    "yarn_alpha": 1.0,
+    "yarn_beta": 32.0,
+    "yarn_original_seq_len": 4096,
+    "micro_batch_size": 1,
+    "gradient_accumulation_steps": 32,
+    "max_lr": 0.00015,
+    "min_lr": 1.5e-05,
+    "warmup_steps": 2000,
+    "total_steps": 200000,
+    "decay_steps": 180000,
+    "weight_decay": 0.1,
+    "muon_beta": 0.95,
+    "muon_ns_steps": 7,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.98,
+    "grad_clip_norm": 5.0,
+    "init_std": 0.006,
+    "dtype": "bfloat16",
+    "fineweb_config": "default",
+    "approx_tokens_per_fineweb_doc": 800,
+    "approx_tokens_per_code_doc": 1200,
+    "approx_tokens_per_math_doc": 1500,
+    "data_mix_probs": [
+      0.6,
+      0.25,
+      0.15
+    ],
+    "save_every_steps": 400,
+    "log_every_steps": 10,
+    "push_every_steps": 400,
+    "checkpoint_dir": "/kaggle/working/zenyxv3checkpoints"
+  }
+}

checkpoints/step_20000/opt_state.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412b5bfa482e8a5ab20abee04b97b0bbefde81970e6489e72e39c344487994f9
+size 3154940698

checkpoints/step_20000/params.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed68b983dfd885a233a5e4402a3f01c62a858ea387c7a73f043016da4d68366
+size 3154768094