Upload add_sub_sorl_v1_abs30_K1_100K

add_sub_sorl_v1_abs30_K1_100K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_K1_100K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_K1_100K/metrics.json

@@ -0,0 +1,3537 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1613,
+      1663,
+      1713,
+      1763,
+      1813,
+      1863,
+      1913,
+      1963,
+      2013,
+      2063,
+      2113,
+      2163,
+      2213,
+      2263,
+      2313,
+      2363,
+      2413,
+      2463,
+      2513,
+      2563,
+      2613,
+      2663,
+      2713,
+      2763,
+      2813,
+      2863,
+      2913,
+      2963,
+      3013,
+      3063,
+      3113,
+      3176,
+      3226,
+      3276,
+      3326,
+      3376,
+      3426,
+      3476,
+      3526,
+      3576,
+      3626,
+      3676,
+      3726,
+      3776,
+      3826,
+      3876,
+      3926,
+      3976,
+      4026,
+      4076,
+      4126,
+      4176,
+      4226,
+      4276,
+      4326,
+      4376,
+      4426,
+      4476,
+      4526,
+      4576,
+      4626,
+      4676,
+      4739,
+      4789,
+      4839,
+      4889,
+      4939,
+      4989,
+      5039,
+      5089,
+      5139,
+      5189,
+      5239,
+      5289,
+      5339,
+      5389,
+      5439,
+      5489,
+      5539,
+      5589,
+      5639,
+      5689,
+      5739,
+      5789,
+      5839,
+      5889,
+      5939,
+      5989,
+      6039,
+      6089,
+      6139,
+      6189,
+      6239,
+      6302,
+      6352,
+      6402,
+      6452,
+      6502,
+      6552,
+      6602,
+      6652,
+      6702,
+      6752,
+      6802,
+      6852,
+      6902,
+      6952,
+      7002,
+      7052,
+      7102,
+      7152,
+      7202,
+      7252,
+      7302,
+      7352,
+      7402,
+      7452,
+      7502,
+      7552,
+      7602,
+      7652,
+      7702,
+      7752,
+      7802,
+      7865,
+      7915,
+      7965,
+      8015,
+      8065,
+      8115,
+      8165,
+      8215,
+      8265,
+      8315,
+      8365,
+      8415,
+      8465,
+      8515,
+      8565,
+      8615,
+      8665,
+      8715,
+      8765,
+      8815,
+      8865,
+      8915,
+      8965,
+      9015,
+      9065,
+      9115,
+      9165,
+      9215,
+      9265,
+      9315,
+      9365,
+      9428,
+      9478,
+      9528,
+      9578,
+      9628,
+      9678,
+      9728,
+      9778,
+      9828,
+      9878,
+      9928,
+      9978,
+      10028,
+      10078,
+      10128,
+      10178,
+      10228,
+      10278,
+      10328,
+      10378,
+      10428,
+      10478,
+      10528,
+      10578,
+      10628,
+      10678,
+      10728,
+      10778,
+      10828,
+      10878,
+      10928,
+      10991,
+      11041,
+      11091,
+      11141,
+      11191,
+      11241,
+      11291,
+      11341,
+      11391,
+      11441,
+      11491,
+      11541,
+      11591,
+      11641,
+      11691,
+      11741,
+      11791,
+      11841,
+      11891,
+      11941,
+      11991,
+      12041,
+      12091,
+      12141,
+      12191,
+      12241,
+      12291,
+      12341,
+      12391,
+      12441,
+      12491,
+      12554,
+      12604,
+      12654,
+      12704,
+      12754,
+      12804,
+      12854,
+      12904,
+      12954,
+      13004,
+      13054,
+      13104,
+      13154,
+      13204,
+      13254,
+      13304,
+      13354,
+      13404,
+      13454,
+      13504,
+      13554,
+      13604,
+      13654,
+      13704,
+      13754,
+      13804,
+      13854,
+      13904,
+      13954,
+      14004,
+      14054,
+      14117,
+      14167,
+      14217,
+      14267,
+      14317,
+      14367,
+      14417,
+      14467,
+      14517,
+      14567,
+      14617,
+      14667,
+      14717,
+      14767,
+      14817,
+      14867,
+      14917,
+      14967,
+      15017,
+      15067,
+      15117,
+      15167,
+      15217,
+      15267,
+      15317,
+      15367,
+      15417,
+      15467,
+      15517,
+      15567,
+      15617
+    ],
+    "loss": [
+      0.6740970611572266,
+      -4.550182819366455,
+      3.4813899993896484,
+      6.129793643951416,
+      4.952874660491943,
+      3.551248073577881,
+      2.7374675273895264,
+      2.642364501953125,
+      2.546827793121338,
+      2.222029209136963,
+      2.18642520904541,
+      1.692894458770752,
+      1.2222495079040527,
+      -0.4533480405807495,
+      -5.11752986907959,
+      -6.007235527038574,
+      -5.759991645812988,
+      -4.846492290496826,
+      -4.504001140594482,
+      -3.289268970489502,
+      -3.108947515487671,
+      -2.94197416305542,
+      -2.7386274337768555,
+      -2.536914348602295,
+      -2.4570322036743164,
+      -2.0626423358917236,
+      -2.5411453247070312,
+      -2.001025915145874,
+      -1.9104180335998535,
+      -1.7866570949554443,
+      -1.4493488073349,
+      -2.2201766967773438,
+      -1.3762651681900024,
+      -1.2861659526824951,
+      -0.9900193214416504,
+      -0.6757111549377441,
+      -1.3362367153167725,
+      -1.2853680849075317,
+      -1.2487475872039795,
+      -0.751352071762085,
+      -1.162947654724121,
+      -1.3089065551757812,
+      -0.717445969581604,
+      -0.6190880537033081,
+      -0.8564563989639282,
+      -0.5846812725067139,
+      -0.4223213195800781,
+      -0.2847200632095337,
+      -0.2850603461265564,
+      -0.6715193390846252,
+      -0.4275590777397156,
+      -0.24846726655960083,
+      -0.298381507396698,
+      -0.21033816039562225,
+      -0.6978311538696289,
+      -0.5567100644111633,
+      -0.08622289448976517,
+      -0.46259573101997375,
+      -0.11854743212461472,
+      -0.2933117747306824,
+      -0.18288816511631012,
+      -0.028837069869041443,
+      -0.10079759359359741,
+      -0.10146437585353851,
+      -0.04020918905735016,
+      -0.17104049026966095,
+      0.03080505132675171,
+      -0.06636752933263779,
+      -0.09336651861667633,
+      0.04093829542398453,
+      -0.18856458365917206,
+      -0.5159808397293091,
+      -0.1291065216064453,
+      0.005189757794141769,
+      0.03898356854915619,
+      0.04612784460186958,
+      -0.035799235105514526,
+      0.017833951860666275,
+      0.008807066828012466,
+      -0.06113648787140846,
+      0.05030018463730812,
+      0.008559092879295349,
+      -0.03814322128891945,
+      -0.35033321380615234,
+      0.022228460758924484,
+      -0.14529463648796082,
+      -0.2573607861995697,
+      0.1615849733352661,
+      0.005920305848121643,
+      -0.17471525073051453,
+      0.04031040892004967,
+      0.15831109881401062,
+      0.017531119287014008,
+      0.025253821164369583,
+      -0.03998721018433571,
+      -0.007496364414691925,
+      -0.08198462426662445,
+      0.005123011767864227,
+      0.013146581128239632,
+      0.021802786737680435,
+      0.010888367891311646,
+      0.01744697242975235,
+      -0.011754129081964493,
+      0.010674640536308289,
+      0.026038404554128647,
+      -0.008711300790309906,
+      -0.026323799043893814,
+      -0.0012233927845954895,
+      0.0051741283386945724,
+      -0.05305163562297821,
+      0.022504277527332306,
+      -0.13647110760211945,
+      -0.07036244124174118,
+      0.03079453483223915,
+      -0.003495320677757263,
+      -0.004044299945235252,
+      0.07291208952665329,
+      0.0158882737159729,
+      -0.05404190719127655,
+      0.031853821128606796,
+      -0.00931493379175663,
+      0.030048897489905357,
+      0.03535217419266701,
+      0.02074359357357025,
+      0.02245134674012661,
+      0.018024209886789322,
+      0.019024260342121124,
+      0.016671666875481606,
+      0.02318689413368702,
+      -0.16695775091648102,
+      -0.09018514305353165,
+      -0.02566644549369812,
+      0.04127392917871475,
+      0.024764355272054672,
+      -0.09843560308218002,
+      -0.011365063488483429,
+      0.03550856187939644,
+      0.017145691439509392,
+      -0.13998593389987946,
+      0.038096681237220764,
+      0.021543879061937332,
+      0.04749413579702377,
+      0.033979713916778564,
+      0.03674383834004402,
+      -0.27052831649780273,
+      0.02465873956680298,
+      0.011796783655881882,
+      0.014996586367487907,
+      0.004961196333169937,
+      0.023028558120131493,
+      0.026358719915151596,
+      0.01939976215362549,
+      -0.005309898406267166,
+      0.020909907296299934,
+      0.0012779179960489273,
+      0.01730952039361,
+      0.02640022523701191,
+      0.0036910362541675568,
+      0.027950383722782135,
+      0.01147831417620182,
+      -0.05264598876237869,
+      0.03575543314218521,
+      0.03973792493343353,
+      0.02252863347530365,
+      -0.009360365569591522,
+      0.015801431611180305,
+      0.02285904251039028,
+      -0.08570974320173264,
+      -0.024168234318494797,
+      -0.048223428428173065,
+      -0.01638774946331978,
+      0.0006880722939968109,
+      0.03306081146001816,
+      -0.043127525597810745,
+      0.0020329076796770096,
+      0.01305706799030304,
+      0.02981213480234146,
+      0.029193194583058357,
+      0.02433847077190876,
+      -0.005482485517859459,
+      0.014438088983297348,
+      0.028228456154465675,
+      0.01625111512839794,
+      0.019915776327252388,
+      0.034516170620918274,
+      0.008582454174757004,
+      0.024511806666851044,
+      -0.024253854528069496,
+      0.01701337844133377,
+      -0.0373363271355629,
+      0.027363358065485954,
+      0.03041806071996689,
+      0.12457426637411118,
+      0.027725394815206528,
+      0.013912304304540157,
+      0.023901334032416344,
+      0.020304588600993156,
+      0.028389938175678253,
+      -0.06357190012931824,
+      0.033661819994449615,
+      -0.07012784481048584,
+      0.00801593717187643,
+      -0.0050064027309417725,
+      -0.09235908091068268,
+      -0.01229124329984188,
+      0.006246935576200485,
+      0.02044505998492241,
+      -0.0047648195177316666,
+      0.025742169469594955,
+      0.0010393057018518448,
+      0.03094959259033203,
+      0.02283795177936554,
+      -0.036882538348436356,
+      0.02033250965178013,
+      0.015916980803012848,
+      0.005782921798527241,
+      0.018086083233356476,
+      -0.0076748523861169815,
+      0.02724587917327881,
+      0.024482186883687973,
+      -0.09868386387825012,
+      0.0167199969291687,
+      0.012935709208250046,
+      0.02163371443748474,
+      0.021752143278717995,
+      0.020082633942365646,
+      0.019570238888263702,
+      0.03249555826187134,
+      0.027163658291101456,
+      -0.014401596039533615,
+      0.022197101265192032,
+      0.022188032045960426,
+      0.03264985978603363,
+      0.024040594696998596,
+      0.011879267171025276,
+      0.03255181014537811,
+      0.023215238004922867,
+      0.0023117028176784515,
+      0.024495478719472885,
+      0.019720498472452164,
+      0.023135492578148842,
+      0.014651769772171974,
+      -0.07951529324054718,
+      0.03233766555786133,
+      0.024470800533890724,
+      0.012430749833583832,
+      0.034896910190582275,
+      0.0302310511469841,
+      0.030738510191440582,
+      0.026226412504911423,
+      0.020528079941868782,
+      0.05512729659676552,
+      0.023782307282090187,
+      0.03561301529407501,
+      0.03899824991822243,
+      0.04859517514705658,
+      0.025169283151626587,
+      0.020543096587061882,
+      0.029567118734121323,
+      0.02862093225121498,
+      0.024188119918107986,
+      0.008323835209012032,
+      0.01983008161187172,
+      0.02144598215818405,
+      0.019778920337557793,
+      0.02909342385828495,
+      0.018559956923127174,
+      0.02560761198401451,
+      0.020072082057595253,
+      0.027100972831249237,
+      0.02295202761888504,
+      0.029123857617378235,
+      0.026096206158399582,
+      0.021702082827687263,
+      0.016750920563936234,
+      0.021258926019072533,
+      0.02797173708677292,
+      0.03243739902973175,
+      0.016977261751890182,
+      0.02330954745411873,
+      0.019976291805505753,
+      0.013898602686822414,
+      0.01689036749303341,
+      0.017199471592903137,
+      0.017278049141168594,
+      0.0165029875934124,
+      0.027514660730957985,
+      0.014729876071214676,
+      0.016875755041837692,
+      0.03120841272175312,
+      0.014980831183493137,
+      0.01650414802134037,
+      0.0152245769277215,
+      0.01621233858168125,
+      0.017918068915605545,
+      0.024394424632191658,
+      0.020674878731369972,
+      0.022252261638641357,
+      0.02144944667816162,
+      0.021988647058606148,
+      0.012099644169211388,
+      0.018198803067207336,
+      0.020583491772413254,
+      0.013832390308380127,
+      0.016547255218029022,
+      0.014451662078499794,
+      0.023647719994187355,
+      0.01148418989032507,
+      0.014027438126504421,
+      0.01586928404867649
+    ],
+    "base_loss": [
+      11.356026649475098,
+      9.631194114685059,
+      8.192168235778809,
+      6.82120418548584,
+      5.469183444976807,
+      3.667222499847412,
+      2.1800343990325928,
+      2.0134799480438232,
+      1.8902651071548462,
+      1.8171883821487427,
+      1.831566572189331,
+      1.7631242275238037,
+      1.8302593231201172,
+      1.7793418169021606,
+      1.7493199110031128,
+      1.6062908172607422,
+      1.4234923124313354,
+      1.2260541915893555,
+      1.0503933429718018,
+      0.8701945543289185,
+      0.7965256571769714,
+      0.7610203623771667,
+      0.7659856081008911,
+      0.6486948728561401,
+      0.6037761569023132,
+      0.5409098267555237,
+      0.6157110333442688,
+      0.537693202495575,
+      0.4848516285419464,
+      0.47311052680015564,
+      0.36529630422592163,
+      0.4196758270263672,
+      0.34379932284355164,
+      0.3498077988624573,
+      0.27728471159935,
+      0.3462373912334442,
+      0.2944621741771698,
+      0.29158657789230347,
+      0.2628433406352997,
+      0.2456883043050766,
+      0.2572735846042633,
+      0.24886596202850342,
+      0.15836580097675323,
+      0.13796664774417877,
+      0.15766572952270508,
+      0.13893279433250427,
+      0.09202127903699875,
+      0.0762753039598465,
+      0.09009210020303726,
+      0.10502225905656815,
+      0.08243220299482346,
+      0.05596935749053955,
+      0.07416750490665436,
+      0.056125104427337646,
+      0.10202639549970627,
+      0.0857960507273674,
+      0.033538710325956345,
+      0.07196591049432755,
+      0.0300368070602417,
+      0.04979466274380684,
+      0.034712500870227814,
+      0.021323591470718384,
+      0.03061729297041893,
+      0.0243806391954422,
+      0.017203472554683685,
+      0.03176433965563774,
+      0.009061001241207123,
+      0.017964502796530724,
+      0.023406967520713806,
+      0.0077172196470201015,
+      0.03076944686472416,
+      0.06775210797786713,
+      0.024297479540109634,
+      0.00829256046563387,
+      0.003892586100846529,
+      0.0024990288075059652,
+      0.011014272458851337,
+      0.004383252467960119,
+      0.006406955420970917,
+      0.01326407678425312,
+      0.0017964880680665374,
+      0.005481006111949682,
+      0.012037997134029865,
+      0.044263992458581924,
+      0.005022068973630667,
+      0.021807847544550896,
+      0.03292359039187431,
+      0.003572768298909068,
+      0.00446776719763875,
+      0.024249836802482605,
+      0.0017572713550180197,
+      0.004465059842914343,
+      0.003973008599132299,
+      0.003093535779044032,
+      0.009311909787356853,
+      0.005410215817391872,
+      0.014211111702024937,
+      0.0034575473982840776,
+      0.0019197268411517143,
+      0.0033427230082452297,
+      0.003699967637658119,
+      0.001604084623977542,
+      0.005739237181842327,
+      0.003080210415646434,
+      0.0015003395965322852,
+      0.004763317760080099,
+      0.006590183824300766,
+      0.004715103190392256,
+      0.0035070686135441065,
+      0.008657963015139103,
+      0.0003322688862681389,
+      0.01885424368083477,
+      0.010693478398025036,
+      0.0005817242781631649,
+      0.004215598572045565,
+      0.003816906362771988,
+      0.003886388149112463,
+      0.003007725114002824,
+      0.010286539793014526,
+      0.0009298769873566926,
+      0.004372141789644957,
+      0.00039718716288916767,
+      0.0013855819124728441,
+      0.000452018080977723,
+      0.0010533257154747844,
+      0.0029780082404613495,
+      0.001559087890200317,
+      0.0014608645578846335,
+      0.00307484227232635,
+      0.02185017243027687,
+      0.013785426504909992,
+      0.006048695184290409,
+      0.00217792228795588,
+      0.001179423532448709,
+      0.017274845391511917,
+      0.0050835153087973595,
+      0.0009023203165270388,
+      0.00184809067286551,
+      0.01991746947169304,
+      0.00039157853461802006,
+      0.003423531772568822,
+      0.0001724983740132302,
+      0.000554570578970015,
+      0.00017678069707471877,
+      0.03262777253985405,
+      0.00029608188197016716,
+      0.004467927850782871,
+      0.0026619676500558853,
+      0.00304457382299006,
+      0.0005487267626449466,
+      0.00014045450370758772,
+      0.000306745117995888,
+      0.0030850092880427837,
+      0.0002520345733501017,
+      0.003663628827780485,
+      0.0009150320547632873,
+      0.0009810777846723795,
+      0.004521961789578199,
+      0.0005316039896570146,
+      0.0028678581584244967,
+      0.009407954290509224,
+      0.0010586724383756518,
+      9.7081639978569e-05,
+      0.0011830349685624242,
+      0.003912817686796188,
+      0.00036425379221327603,
+      0.00018008511688094586,
+      0.013768239878118038,
+      0.006027285009622574,
+      0.008561733178794384,
+      0.005584522150456905,
+      0.004016578663140535,
+      0.00027379707898944616,
+      0.007990238256752491,
+      0.0037911939434707165,
+      0.0017629734938964248,
+      0.00024030738859437406,
+      0.0005787690752185881,
+      0.00018286806880496442,
+      0.0031564664095640182,
+      0.0010887740645557642,
+      0.0006292345351539552,
+      0.00043399655260145664,
+      0.00029165399610064924,
+      0.0008086164598353207,
+      0.002190670231357217,
+      0.00016634409257676452,
+      0.005654097069054842,
+      0.0007038725889287889,
+      0.007417007349431515,
+      0.00011201239976799116,
+      0.00010763022146420553,
+      0.0022711276542395353,
+      0.0004151062748860568,
+      0.0001313961111009121,
+      8.628344221506268e-05,
+      0.00045848326408304274,
+      0.00032122276024892926,
+      0.009824968874454498,
+      0.00012736987264361233,
+      0.010177732445299625,
+      0.0010128446156159043,
+      0.004551018122583628,
+      0.013036270625889301,
+      0.0037082901690155268,
+      0.002745306584984064,
+      9.197835606755689e-05,
+      0.0026039115618914366,
+      0.00013443033094517887,
+      0.0028798342682421207,
+      8.376793266506866e-05,
+      3.4521566703915596e-05,
+      0.006287913303822279,
+      0.000488495163153857,
+      0.0015168219106271863,
+      0.001650623744353652,
+      0.00019628273730631918,
+      0.0032842964865267277,
+      0.00045201057218946517,
+      0.0003367347235325724,
+      0.012855315580964088,
+      0.0005910869804210961,
+      0.0011569021735340357,
+      0.00038534580380655825,
+      0.00017291684343945235,
+      3.4607968700584024e-05,
+      7.030936831142753e-05,
+      0.00021958326396998018,
+      6.0223665059311315e-05,
+      0.005668822210282087,
+      2.0555753508233465e-05,
+      1.8682430891203694e-05,
+      2.363364546908997e-05,
+      5.7613910030340776e-05,
+      0.0010363028850406408,
+      3.1130286515690386e-05,
+      1.534678813186474e-05,
+      0.0031705431174486876,
+      0.00017218641005456448,
+      0.0001772747200448066,
+      0.0002666492946445942,
+      9.376927482662722e-05,
+      0.011100794188678265,
+      1.4641766028944403e-05,
+      1.0835289685928728e-05,
+      0.0026217966806143522,
+      0.0001616269291844219,
+      6.171682616695762e-05,
+      9.293656330555677e-05,
+      4.332200842327438e-05,
+      4.211247505736537e-05,
+      0.00012382295972201973,
+      5.505144872586243e-05,
+      1.5595851436955854e-05,
+      0.0001040818024193868,
+      0.0015936119016259909,
+      0.0002965541207231581,
+      0.0002465788566041738,
+      7.969176658662036e-05,
+      7.859185279812664e-05,
+      1.8917620764113963e-05,
+      0.0011198128340765834,
+      4.6502420445904136e-05,
+      0.00011284415086265653,
+      0.00010124589607585222,
+      1.891675310616847e-05,
+      0.00014527658640872687,
+      5.916170994169079e-05,
+      1.7848889910965227e-05,
+      1.8014457964454778e-05,
+      0.000127076476928778,
+      4.3773507059086114e-05,
+      2.9668290153495036e-05,
+      4.1452207369729877e-05,
+      1.0970002222165931e-05,
+      1.6135436453623697e-05,
+      1.2064027941960376e-05,
+      0.00010817877046065405,
+      6.877558917040005e-06,
+      6.24669628450647e-05,
+      9.740904715727083e-06,
+      6.209537332324544e-06,
+      4.491790241445415e-05,
+      7.744681170152035e-06,
+      0.000721240823622793,
+      0.0005756255122832954,
+      7.592778729303973e-06,
+      0.000425457808887586,
+      8.788871491560712e-05,
+      1.8696180632105097e-05,
+      1.0503771591174882e-05,
+      8.51023432915099e-05,
+      1.927906669152435e-05,
+      2.6535662982496433e-05,
+      8.09933953860309e-06,
+      1.4273643500928301e-05,
+      1.1488074960652739e-05,
+      6.521457544295117e-05,
+      1.4513118912873324e-05,
+      6.31172297289595e-05,
+      0.00020963601127732545,
+      7.592925157950958e-06,
+      7.2776251727191266e-06,
+      7.568458386231214e-06,
+      0.00020133117504883558,
+      8.974142474471591e-06,
+      1.848995452746749e-05,
+      7.909872692835052e-06,
+      1.0007915079768281e-05,
+      7.120363534340868e-06
+    ],
+    "info_loss": [
+      -1.7890558242797852,
+      -2.0338239669799805,
+      -0.9482707977294922,
+      -0.40160417556762695,
+      -0.2615690231323242,
+      -0.15079426765441895,
+      -0.06246805191040039,
+      -0.05041790008544922,
+      -0.046514034271240234,
+      -0.07114255428314209,
+      -0.0759742259979248,
+      -0.1182030439376831,
+      -0.17184579372406006,
+      -0.3341081142425537,
+      -0.7977063655853271,
+      -0.8726139068603516,
+      -0.828260600566864,
+      -0.7154039144515991,
+      -0.6624167561531067,
+      -0.5188260078430176,
+      -0.4914211332798004,
+      -0.4694363474845886,
+      -0.44739606976509094,
+      -0.4149097204208374,
+      -0.39935627579689026,
+      -0.3504805564880371,
+      -0.40426820516586304,
+      -0.3374651074409485,
+      -0.320678174495697,
+      -0.30643635988235474,
+      -0.2568308711051941,
+      -0.3357636332511902,
+      -0.23645959794521332,
+      -0.23092350363731384,
+      -0.19038887321949005,
+      -0.16156063973903656,
+      -0.22003400325775146,
+      -0.2115722894668579,
+      -0.20114350318908691,
+      -0.15098732709884644,
+      -0.19291964173316956,
+      -0.20082661509513855,
+      -0.13588814437389374,
+      -0.12022731453180313,
+      -0.14502620697021484,
+      -0.11288445442914963,
+      -0.08485924452543259,
+      -0.07067755609750748,
+      -0.06549526751041412,
+      -0.10249876976013184,
+      -0.07931748777627945,
+      -0.05338749662041664,
+      -0.06161873787641525,
+      -0.04923267289996147,
+      -0.09974635392427444,
+      -0.08387186378240585,
+      -0.026406487450003624,
+      -0.06957025825977325,
+      -0.029383406043052673,
+      -0.047899384051561356,
+      -0.034090615808963776,
+      -0.02017339877784252,
+      -0.02552923932671547,
+      -0.023796377703547478,
+      -0.016712982207536697,
+      -0.03141834959387779,
+      -0.008702496066689491,
+      -0.01777610555291176,
+      -0.023175155743956566,
+      -0.007410112768411636,
+      -0.030521877110004425,
+      -0.06679011136293411,
+      -0.024130171164870262,
+      -0.008060572668910027,
+      -0.0038049675058573484,
+      -0.002234472194686532,
+      -0.010922570712864399,
+      -0.004309913143515587,
+      -0.00633973628282547,
+      -0.013188357464969158,
+      -0.0017262588953599334,
+      -0.005346525926142931,
+      -0.011774897575378418,
+      -0.04415065050125122,
+      -0.004944274667650461,
+      -0.021722447127103806,
+      -0.032871976494789124,
+      0.010226638056337833,
+      -0.00442873639985919,
+      -0.024205442517995834,
+      -0.0017210857477039099,
+      0.009605957195162773,
+      -0.0038327304646372795,
+      -0.003049204358831048,
+      -0.009267322719097137,
+      -0.005381999537348747,
+      -0.014152013696730137,
+      -0.0033899759873747826,
+      -0.001888719736598432,
+      -0.0033195270225405693,
+      -0.0036388984881341457,
+      -0.0015598938334733248,
+      -0.005652206018567085,
+      -0.0030528190545737743,
+      -0.0014716533478349447,
+      -0.004738667979836464,
+      -0.006567519623786211,
+      -0.00465953117236495,
+      -0.003490664064884186,
+      -0.008616232313215733,
+      -0.00014296182780526578,
+      -0.018821515142917633,
+      -0.0105670765042305,
+      -0.0005634925328195095,
+      -0.0041562579572200775,
+      -0.0037870830856263638,
+      -0.0037928519304841757,
+      -0.002935892902314663,
+      -0.010225500911474228,
+      -0.0009113152627833188,
+      -0.004355199635028839,
+      -0.00035779280005954206,
+      -0.0011598645942285657,
+      -0.0004282161535229534,
+      -0.001037601730786264,
+      -0.002960778307169676,
+      -0.0015378620009869337,
+      -0.0014182236045598984,
+      -0.003061524359509349,
+      -0.021842218935489655,
+      -0.013775412924587727,
+      -0.0060298447497189045,
+      -0.0003302695695310831,
+      -0.0011622959282249212,
+      -0.01726355403661728,
+      -0.005036278162151575,
+      -0.00041794925346039236,
+      -0.0018197885947301984,
+      -0.01988215371966362,
+      -0.0003762421547435224,
+      -0.003400198882445693,
+      -0.00015940802404657006,
+      -0.0005233503761701286,
+      -0.00015287620772141963,
+      -0.03261752426624298,
+      -0.0002691608387976885,
+      -0.0044571529142558575,
+      -0.0026539356913417578,
+      -0.003012097207829356,
+      -0.0005078242975287139,
+      -0.00013505083916243166,
+      -0.0003015396650880575,
+      -0.003070201026275754,
+      -0.0002374972536927089,
+      -0.0036586765199899673,
+      -0.0009081086027435958,
+      -0.0003338689566589892,
+      -0.004436817951500416,
+      -0.0004963281098753214,
+      -0.002862064866349101,
+      -0.009401691146194935,
+      -0.0010455616284161806,
+      -8.600996079621837e-05,
+      -0.0011703305644914508,
+      -0.0038957232609391212,
+      -0.00034908443922176957,
+      -0.00017510104225948453,
+      -0.013707173988223076,
+      -0.006002034991979599,
+      -0.008538494817912579,
+      -0.005569901317358017,
+      -0.004009208641946316,
+      -0.0002337438054382801,
+      -0.007973904721438885,
+      -0.00378547259606421,
+      -0.0015358731616288424,
+      -0.00023604519083164632,
+      -0.0005697747110389173,
+      -0.00016817497089505196,
+      -0.003141829976812005,
+      -0.0010833129053935409,
+      -0.000625795335508883,
+      -0.0004284863534849137,
+      -0.00027779044467024505,
+      -0.0007803748594596982,
+      -0.0012746010906994343,
+      -0.00015991127293091267,
+      -0.005650668870657682,
+      -0.0006989692919887602,
+      -0.007410449907183647,
+      -0.00010782053141156211,
+      -8.742442878428847e-05,
+      0.008178682066500187,
+      -0.00041095964843407273,
+      -0.0001164748755400069,
+      -6.856003892607987e-05,
+      -0.0004533177416305989,
+      -0.00031492739799432456,
+      -0.009818059392273426,
+      -0.00012356028310023248,
+      -0.010169540531933308,
+      -0.0010074570309370756,
+      -0.00452058669179678,
+      -0.013032909482717514,
+      -0.003703873371705413,
+      -0.0027421782724559307,
+      -8.945290755946189e-05,
+      -0.002599295461550355,
+      -0.00012219540076330304,
+      -0.0028765909373760223,
+      -6.239667709451169e-05,
+      -3.049292354262434e-05,
+      -0.006285164505243301,
+      -0.000484316871734336,
+      -0.001514196745119989,
+      -0.0016474260482937098,
+      -0.00019226217409595847,
+      -0.003281482495367527,
+      -0.000449836632469669,
+      -0.0003332652267999947,
+      -0.012853292748332024,
+      -0.0005873491172678769,
+      -0.0011497307568788528,
+      -0.00037652216269634664,
+      -0.00016018647875171155,
+      -3.0155661079334095e-05,
+      -6.62122038193047e-05,
+      -0.00020255916751921177,
+      -5.408488505054265e-05,
+      -0.005652152467519045,
+      -1.6025685908971354e-05,
+      -1.5044855899759568e-05,
+      -2.1465304598677903e-05,
+      -5.105826130602509e-05,
+      -0.0010347074130550027,
+      -1.907377372845076e-05,
+      -1.2802254786947742e-05,
+      -0.0017700940370559692,
+      -0.0001699523563729599,
+      -0.00017307884991168976,
+      -0.0002653632836882025,
+      -8.952543430496007e-05,
+      -0.011098955757915974,
+      -1.267616062250454e-05,
+      -9.07962748897262e-06,
+      -0.0026156750973314047,
+      -0.00015660403005313128,
+      -4.651540075428784e-05,
+      -8.610812074039131e-05,
+      -3.715619823196903e-05,
+      -3.755074067157693e-05,
+      -0.00012043094466207549,
+      -5.0913546147057787e-05,
+      -1.0800137715705205e-05,
+      -9.554681309964508e-05,
+      -0.0015626936219632626,
+      -0.00028642217512242496,
+      -0.0002437218645354733,
+      -6.973075505811721e-05,
+      -7.534190081059933e-05,
+      -1.5272318705683574e-05,
+      -0.0011173563543707132,
+      -3.8153659261297435e-05,
+      -9.573547140462324e-05,
+      -7.302961603272706e-05,
+      -1.7509401004645042e-05,
+      -0.00014374923193827271,
+      -5.755531674367376e-05,
+      -1.6221218174905516e-05,
+      3.5054064937867224e-05,
+      -0.00012420724669937044,
+      -3.818568802671507e-05,
+      -2.7303436581860296e-05,
+      -4.013958459836431e-05,
+      -7.484359230147675e-06,
+      -1.4186858606990427e-05,
+      -7.645062396477442e-06,
+      -0.000106018444057554,
+      -4.252270173310535e-06,
+      -5.9775567933684215e-05,
+      -8.242283911386039e-06,
+      -5.06002697875374e-06,
+      -4.421116318553686e-05,
+      -6.299577762547415e-06,
+      -0.000720062293112278,
+      -0.0005746351089328527,
+      -6.277238753682468e-06,
+      -0.00042429924360476434,
+      -8.700289617991075e-05,
+      -1.789098678273149e-05,
+      -9.702303941594437e-06,
+      -8.410264126723632e-05,
+      -1.8023398297373205e-05,
+      -2.533986116759479e-05,
+      -7.160304903663928e-06,
+      -1.340539711236488e-05,
+      -1.0124642358277924e-05,
+      -6.414462404791266e-05,
+      -1.3188522643758915e-05,
+      -6.239931099116802e-05,
+      -0.0002082925202557817,
+      -6.2832409639668185e-06,
+      -6.361206487781601e-06,
+      -6.34949947198038e-06,
+      -0.00020040091476403177,
+      -7.874392395024188e-06,
+      -1.763314321578946e-05,
+      -6.871585355838761e-06,
+      -8.002768481674138e-06,
+      -6.212992502696579e-06
+    ],
+    "abs_loss": [
+      3.400221586227417,
+      3.3749444484710693,
+      3.3227717876434326,
+      3.2700469493865967,
+      3.0865707397460938,
+      2.835861921310425,
+      2.728372573852539,
+      2.7506744861602783,
+      2.7388916015625,
+      2.7243969440460205,
+      2.6911118030548096,
+      2.7227325439453125,
+      2.711573839187622,
+      2.7065513134002686,
+      2.6780731678009033,
+      2.6901721954345703,
+      2.667034149169922,
+      2.5448572635650635,
+      2.4869515895843506,
+      2.359622001647949,
+      2.235144853591919,
+      2.180715799331665,
+      2.1124141216278076,
+      2.011976480484009,
+      2.0321614742279053,
+      1.9616875648498535,
+      1.801836609840393,
+      1.8774360418319702,
+      1.72232186794281,
+      1.5954824686050415,
+      1.6342706680297852,
+      1.4825626611709595,
+      1.3536456823349,
+      1.3639774322509766,
+      1.3342081308364868,
+      1.3440603017807007,
+      1.254590630531311,
+      1.1744147539138794,
+      1.065630316734314,
+      1.0658882856369019,
+      1.0787917375564575,
+      0.9862263798713684,
+      1.0132602453231812,
+      0.9369902610778809,
+      0.9065744280815125,
+      0.7865585684776306,
+      0.80427485704422,
+      0.7036848664283752,
+      0.5578082203865051,
+      0.5439424514770508,
+      0.5409030318260193,
+      0.47011399269104004,
+      0.5740960240364075,
+      0.5397010445594788,
+      0.4535597264766693,
+      0.3936477601528168,
+      0.4094356596469879,
+      0.3892175257205963,
+      0.3264791965484619,
+      0.3573044240474701,
+      0.2997516691684723,
+      0.3188786506652832,
+      0.30981138348579407,
+      0.24736468493938446,
+      0.2696608901023865,
+      0.3189583718776703,
+      0.2266036719083786,
+      0.22413313388824463,
+      0.19217394292354584,
+      0.1975734680891037,
+      0.18984945118427277,
+      0.18902957439422607,
+      0.16657519340515137,
+      0.16911743581295013,
+      0.13720349967479706,
+      0.11513227224349976,
+      0.14049004018306732,
+      0.13792578876018524,
+      0.12941564619541168,
+      0.12365423887968063,
+      0.10792890936136246,
+      0.08864489942789078,
+      0.1043628379702568,
+      0.11446789652109146,
+      0.0968158021569252,
+      0.11080390214920044,
+      0.092313252389431,
+      0.10519183427095413,
+      0.08616107702255249,
+      0.09144207835197449,
+      0.0939544215798378,
+      0.075531966984272,
+      0.09000606089830399,
+      0.07730904221534729,
+      0.10152538865804672,
+      0.07238713651895523,
+      0.06638116389513016,
+      0.0714423879981041,
+      0.05998113751411438,
+      0.08143870532512665,
+      0.07830570638179779,
+      0.061825867742300034,
+      0.05181339755654335,
+      0.05899941548705101,
+      0.09622979909181595,
+      0.05658048391342163,
+      0.07066275924444199,
+      0.04587157070636749,
+      0.08620896190404892,
+      0.04473419487476349,
+      0.05273726209998131,
+      0.04513105750083923,
+      0.0452953577041626,
+      0.0503426156938076,
+      0.04764794185757637,
+      0.041065242141485214,
+      0.0642910972237587,
+      0.0393511988222599,
+      0.053640592843294144,
+      0.04719572141766548,
+      0.03421347215771675,
+      0.050639551132917404,
+      0.051145922392606735,
+      0.04705527797341347,
+      0.034941766411066055,
+      0.04037773981690407,
+      0.04842052981257439,
+      0.05336771532893181,
+      0.06259221583604813,
+      0.026474887505173683,
+      0.031480368226766586,
+      0.06849294155836105,
+      0.031607963144779205,
+      0.03826194629073143,
+      0.04014688357710838,
+      0.03380918502807617,
+      0.030318809673190117,
+      0.02713879942893982,
+      0.06298081576824188,
+      0.03328637406229973,
+      0.061558797955513,
+      0.039092469960451126,
+      0.04050593823194504,
+      0.03157830610871315,
+      0.039200589060783386,
+      0.04502244293689728,
+      0.04802577570080757,
+      0.033281903713941574,
+      0.0406477265059948,
+      0.04972783103585243,
+      0.037740204483270645,
+      0.04166874289512634,
+      0.02565709315240383,
+      0.03303562104701996,
+      0.03330441191792488,
+      0.03580398112535477,
+      0.03196847438812256,
+      0.046718478202819824,
+      0.04452933743596077,
+      0.024944854900240898,
+      0.0416223406791687,
+      0.036224447190761566,
+      0.05375756695866585,
+      0.0367773212492466,
+      0.03235439583659172,
+      0.02875889278948307,
+      0.03506853058934212,
+      0.04399366304278374,
+      0.03728214651346207,
+      0.0427677147090435,
+      0.0378706157207489,
+      0.035004809498786926,
+      0.04873510077595711,
+      0.035581763833761215,
+      0.04483659937977791,
+      0.03673766180872917,
+      0.037086594849824905,
+      0.03477691113948822,
+      0.027956614270806313,
+      0.03586661070585251,
+      0.033001434057950974,
+      0.031290311366319656,
+      0.035908281803131104,
+      0.030315974727272987,
+      0.03552045300602913,
+      0.04022013023495674,
+      0.026197584345936775,
+      0.035174209624528885,
+      0.02384295128285885,
+      0.037081893533468246,
+      0.0395648367702961,
+      0.03571036830544472,
+      0.04348386451601982,
+      0.02467021346092224,
+      0.02426413632929325,
+      0.029572337865829468,
+      0.03587673231959343,
+      0.04099087044596672,
+      0.025093600153923035,
+      0.03426919877529144,
+      0.024828441441059113,
+      0.027611741796135902,
+      0.02356303296983242,
+      0.026904771104454994,
+      0.028047552332282066,
+      0.029466355219483376,
+      0.03148570656776428,
+      0.021179335191845894,
+      0.04117191955447197,
+      0.03302908316254616,
+      0.026304379105567932,
+      0.04027620330452919,
+      0.033575210720300674,
+      0.03448949381709099,
+      0.021397188305854797,
+      0.028684427961707115,
+      0.032612934708595276,
+      0.019171735271811485,
+      0.030285999178886414,
+      0.019913824275135994,
+      0.01965104602277279,
+      0.02198127470910549,
+      0.032737139612436295,
+      0.025666287168860435,
+      0.023947492241859436,
+      0.026168445125222206,
+      0.02097104676067829,
+      0.030548997223377228,
+      0.027204474434256554,
+      0.0273570716381073,
+      0.0234281737357378,
+      0.030982812866568565,
+      0.024150425568223,
+      0.030628904700279236,
+      0.027146128937602043,
+      0.02773093245923519,
+      0.028193825855851173,
+      0.026094062253832817,
+      0.028047211468219757,
+      0.03213946893811226,
+      0.02489076741039753,
+      0.027458617463707924,
+      0.02401149272918701,
+      0.04969260096549988,
+      0.040320202708244324,
+      0.0385587178170681,
+      0.03392435237765312,
+      0.03432046249508858,
+      0.04117554798722267,
+      0.045425236225128174,
+      0.027294665575027466,
+      0.04759006202220917,
+      0.03798606991767883,
+      0.04059753939509392,
+      0.038912393152713776,
+      0.0981680378317833,
+      0.048965733498334885,
+      0.050136175006628036,
+      0.026505114510655403,
+      0.02815972827374935,
+      0.025639228522777557,
+      0.02983478270471096,
+      0.03243863955140114,
+      0.027228472754359245,
+      0.03602074086666107,
+      0.03893369436264038,
+      0.02168719470500946,
+      0.027577867731451988,
+      0.027573436498641968,
+      0.027087293565273285,
+      0.031455520540475845,
+      0.03223405405879021,
+      0.028805658221244812,
+      0.02321797050535679,
+      0.02031596750020981,
+      0.028041882440447807,
+      0.023989439010620117,
+      0.02367476187646389,
+      0.02858881466090679,
+      0.025053301826119423,
+      0.018464459106326103,
+      0.017213404178619385,
+      0.02135978639125824,
+      0.01722577027976513,
+      0.02366158552467823,
+      0.018046531826257706,
+      0.036419834941625595,
+      0.019340110942721367,
+      0.03182953596115112,
+      0.017846064642071724,
+      0.02648819424211979,
+      0.025615304708480835,
+      0.021788349375128746,
+      0.028488459065556526,
+      0.02397303842008114,
+      0.029911711812019348,
+      0.023171788081526756,
+      0.030988125130534172,
+      0.023679383099079132,
+      0.02411339245736599,
+      0.025272300466895103,
+      0.022435227409005165,
+      0.02144625596702099,
+      0.024688227102160454,
+      0.02549501322209835,
+      0.018564846366643906,
+      0.020725158974528313,
+      0.022303933277726173,
+      0.015030582435429096,
+      0.018608180806040764
+    ],
+    "zipf_loss": [
+      6.8686065673828125,
+      5.8193678855896,
+      4.439652442932129,
+      2.997626543045044,
+      1.790724277496338,
+      1.1083821058273315,
+      0.9092764258384705,
+      0.8579962253570557,
+      0.8478137254714966,
+      0.8438266515731812,
+      0.8454896211624146,
+      0.839527428150177,
+      0.8392906785011292,
+      0.8377361297607422,
+      0.8424060940742493,
+      0.843595564365387,
+      0.8324185609817505,
+      0.8270072937011719,
+      0.8210777044296265,
+      0.7928341627120972,
+      0.785223662853241,
+      0.7732972502708435,
+      0.7581063508987427,
+      0.762290358543396,
+      0.7295382022857666,
+      0.7050848603248596,
+      0.7056423425674438,
+      0.648188054561615,
+      0.6392800807952881,
+      0.645047664642334,
+      0.5902365446090698,
+      0.5695276260375977,
+      0.5091668367385864,
+      0.5368636846542358,
+      0.5031638145446777,
+      0.45925191044807434,
+      0.44418203830718994,
+      0.42132675647735596,
+      0.39328092336654663,
+      0.40624403953552246,
+      0.40109601616859436,
+      0.3518710434436798,
+      0.3817436695098877,
+      0.3515194058418274,
+      0.3454824388027191,
+      0.3265745937824249,
+      0.253822386264801,
+      0.2754117250442505,
+      0.2240193635225296,
+      0.19405187666416168,
+      0.2290932536125183,
+      0.18242695927619934,
+      0.18622878193855286,
+      0.17189337313175201,
+      0.15224997699260712,
+      0.1568477600812912,
+      0.10335970669984818,
+      0.12221911549568176,
+      0.11260189861059189,
+      0.10015691816806793,
+      0.09333030879497528,
+      0.11968545615673065,
+      0.09289635717868805,
+      0.08738230168819427,
+      0.08275105804204941,
+      0.07948283851146698,
+      0.08610863983631134,
+      0.07101570814847946,
+      0.09576068818569183,
+      0.0875648558139801,
+      0.06689979135990143,
+      0.06526514887809753,
+      0.07124019414186478,
+      0.06059117987751961,
+      0.059420306235551834,
+      0.054460309445858,
+      0.04836319386959076,
+      0.04275725409388542,
+      0.05285590887069702,
+      0.045117590576410294,
+      0.05497339367866516,
+      0.047678858041763306,
+      0.05713147297501564,
+      0.03546249866485596,
+      0.056967560201883316,
+      0.03904160484671593,
+      0.029204068705439568,
+      0.045226648449897766,
+      0.037123795598745346,
+      0.03394513577222824,
+      0.0463685542345047,
+      0.05023325979709625,
+      0.042884811758995056,
+      0.044921424239873886,
+      0.033221568912267685,
+      0.0336747020483017,
+      0.03868628293275833,
+      0.028420986607670784,
+      0.02411593869328499,
+      0.043511465191841125,
+      0.03574681654572487,
+      0.025259237736463547,
+      0.03384735435247421,
+      0.032222680747509,
+      0.029631618410348892,
+      0.028254013508558273,
+      0.025694940239191055,
+      0.036069657653570175,
+      0.027952803298830986,
+      0.01997930370271206,
+      0.018327901139855385,
+      0.02837669476866722,
+      0.02008531056344509,
+      0.03081347420811653,
+      0.029086865484714508,
+      0.02590310014784336,
+      0.10052511096000671,
+      0.03830435872077942,
+      0.0325625017285347,
+      0.03531752526760101,
+      0.026443572714924812,
+      0.028165683150291443,
+      0.040450647473335266,
+      0.019868209958076477,
+      0.02827986143529415,
+      0.04061621055006981,
+      0.02800174057483673,
+      0.024056266993284225,
+      0.044468075037002563,
+      0.02696678228676319,
+      0.030635518953204155,
+      0.02173401415348053,
+      0.03923790529370308,
+      0.031381696462631226,
+      0.05291040986776352,
+      0.030533283948898315,
+      0.03575385361909866,
+      0.030781608074903488,
+      0.032620057463645935,
+      0.03813888877630234,
+      0.04596645385026932,
+      0.04500647261738777,
+      0.03460805490612984,
+      0.034937988966703415,
+      0.019099101424217224,
+      0.022552020847797394,
+      0.04709780961275101,
+      0.03554578498005867,
+      0.02797282300889492,
+      0.022585291415452957,
+      0.02379475347697735,
+      0.017941538244485855,
+      0.019741391763091087,
+      0.019729282706975937,
+      0.03087061271071434,
+      0.02189517766237259,
+      0.02556098997592926,
+      0.03886540234088898,
+      0.02792912721633911,
+      0.034736618399620056,
+      0.027800733223557472,
+      0.041529931128025055,
+      0.035125184804201126,
+      0.029371172189712524,
+      0.02244860678911209,
+      0.016052132472395897,
+      0.02092311531305313,
+      0.033194392919540405,
+      0.026096615940332413,
+      0.02432301640510559,
+      0.0299396775662899,
+      0.03326310217380524,
+      0.030250942334532738,
+      0.025063108652830124,
+      0.03161277994513512,
+      0.02297906018793583,
+      0.028223620727658272,
+      0.030834481120109558,
+      0.02304169163107872,
+      0.019192688167095184,
+      0.020882301032543182,
+      0.030728142708539963,
+      0.016511153429746628,
+      0.0193704292178154,
+      0.037959255278110504,
+      0.01511578168720007,
+      0.023324817419052124,
+      0.02308131568133831,
+      0.020914902910590172,
+      0.02564297616481781,
+      0.024373067542910576,
+      0.02761363796889782,
+      0.0361679270863533,
+      0.028952863067388535,
+      0.01251924317330122,
+      0.021543417125940323,
+      0.020791608840227127,
+      0.027118902653455734,
+      0.022274363785982132,
+      0.03134313225746155,
+      0.018906977027654648,
+      0.014316488988697529,
+      0.033292144536972046,
+      0.02224327251315117,
+      0.018234441056847572,
+      0.027976777404546738,
+      0.018099039793014526,
+      0.016506290063261986,
+      0.022712500765919685,
+      0.02362247370183468,
+      0.028859354555606842,
+      0.019080739468336105,
+      0.016323670744895935,
+      0.021238233894109726,
+      0.027402406558394432,
+      0.017738115042448044,
+      0.016551129519939423,
+      0.019938500598073006,
+      0.02826363407075405,
+      0.025486722588539124,
+      0.015028645284473896,
+      0.019804272800683975,
+      0.020002400502562523,
+      0.02244696207344532,
+      0.020786341279745102,
+      0.017732737585902214,
+      0.018064947798848152,
+      0.031246665865182877,
+      0.024923836812376976,
+      0.033715397119522095,
+      0.019993985071778297,
+      0.019221516326069832,
+      0.030425837263464928,
+      0.021430673077702522,
+      0.018475424498319626,
+      0.02993832528591156,
+      0.020508531481027603,
+      0.014232693240046501,
+      0.023218095302581787,
+      0.018060065805912018,
+      0.023033399134874344,
+      0.012707392685115337,
+      0.017972322180867195,
+      0.02748052403330803,
+      0.020518740639090538,
+      0.032109834253787994,
+      0.032908886671066284,
+      0.027202442288398743,
+      0.027389099821448326,
+      0.022012129426002502,
+      0.018132008612155914,
+      0.05144877731800079,
+      0.020437784492969513,
+      0.03164566680788994,
+      0.03595839813351631,
+      0.052811697125434875,
+      0.022840376943349838,
+      0.017720118165016174,
+      0.02753422223031521,
+      0.02647978626191616,
+      0.021758003160357475,
+      0.015394107438623905,
+      0.016921252012252808,
+      0.019567646086215973,
+      0.016805896535515785,
+      0.02535623125731945,
+      0.017683453857898712,
+      0.02336621657013893,
+      0.017459101974964142,
+      0.024023689329624176,
+      0.02092147059738636,
+      0.026238536462187767,
+      0.02345900610089302,
+      0.019740229472517967,
+      0.01478319801390171,
+      0.018580470234155655,
+      0.025637179613113403,
+      0.031021928414702415,
+      0.014154024422168732,
+      0.021339505910873413,
+      0.0182025283575058,
+      0.012221653014421463,
+      0.015151582658290863,
+      0.015532146207988262,
+      0.02139127254486084,
+      0.019869059324264526,
+      0.023927856236696243,
+      0.016613399609923363,
+      0.014474941417574883,
+      0.029584020376205444,
+      0.012418530881404877,
+      0.014698540791869164,
+      0.013206697069108486,
+      0.013590356335043907,
+      0.015584267675876617,
+      0.02152303420007229,
+      0.018447458744049072,
+      0.019729681313037872,
+      0.019198879599571228,
+      0.02013818360865116,
+      0.011445702984929085,
+      0.016010519117116928,
+      0.01849520020186901,
+      0.011419493705034256,
+      0.01580043137073517,
+      0.012664947658777237,
+      0.02173304557800293,
+      0.009314602240920067,
+      0.012594399973750114,
+      0.014063475653529167
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.9333709478378296,
+      0.6308308839797974,
+      0.4150587022304535,
+      0.2892041802406311,
+      0.19332674145698547,
+      0.17707014083862305,
+      0.166446715593338,
+      0.13733863830566406,
+      0.11083301901817322,
+      0.10711774230003357,
+      0.10654493421316147,
+      0.1081528589129448,
+      0.11822911351919174,
+      0.12990574538707733,
+      0.15268974006175995,
+      0.16605304181575775,
+      0.17477574944496155,
+      0.18307840824127197,
+      0.19441837072372437,
+      0.2060604691505432,
+      0.21536315977573395,
+      0.22143158316612244,
+      0.2279053032398224,
+      0.23052586615085602,
+      0.23592153191566467,
+      0.24382653832435608,
+      0.24329723417758942,
+      0.24600692093372345,
+      0.2511166036128998,
+      0.2505258619785309,
+      0.25530147552490234,
+      0.25758427381515503,
+      0.25553104281425476,
+      0.2592415511608124,
+      0.2596992254257202,
+      0.2601006031036377,
+      0.2595624625682831,
+      0.2595066428184509,
+      0.26065850257873535,
+      0.2611340284347534,
+      0.2594366669654846,
+      0.26323431730270386,
+      0.2660321891307831,
+      0.2679878771305084,
+      0.2684157192707062,
+      0.26988834142684937,
+      0.2704735994338989,
+      0.2655894160270691,
+      0.2642352879047394,
+      0.2643618583679199,
+      0.26621660590171814,
+      0.26454290747642517,
+      0.262870728969574,
+      0.26233384013175964,
+      0.26171162724494934,
+      0.2598952353000641,
+      0.2565430998802185,
+      0.254987508058548,
+      0.2577458918094635,
+      0.25746971368789673,
+      0.25535234808921814,
+      0.2547993063926697,
+      0.24966923892498016,
+      0.25087061524391174,
+      0.2469557821750641,
+      0.24440476298332214,
+      0.24528679251670837,
+      0.25163495540618896,
+      0.25011399388313293,
+      0.25165289640426636,
+      0.2533286511898041,
+      0.2535499036312103,
+      0.24993078410625458,
+      0.25142911076545715,
+      0.24880218505859375,
+      0.24764417111873627,
+      0.24868673086166382,
+      0.25567251443862915,
+      0.2550661265850067,
+      0.25474682450294495,
+      0.2570652663707733,
+      0.25641271471977234,
+      0.260037362575531,
+      0.25931286811828613,
+      0.2570188641548157,
+      0.25402596592903137,
+      0.24845820665359497,
+      0.24738140404224396,
+      0.24976491928100586,
+      0.24371632933616638,
+      0.24283857643604279,
+      0.2415243536233902,
+      0.2389516532421112,
+      0.23993976414203644,
+      0.23685067892074585,
+      0.23623093962669373,
+      0.23535040020942688,
+      0.23545700311660767,
+      0.23676586151123047,
+      0.2359374463558197,
+      0.23601751029491425,
+      0.23786631226539612,
+      0.23994474112987518,
+      0.23437577486038208,
+      0.2316410094499588,
+      0.22984708845615387,
+      0.2367476522922516,
+      0.23675338923931122,
+      0.23577436804771423,
+      0.23775532841682434,
+      0.23460887372493744,
+      0.23389208316802979,
+      0.23258386552333832,
+      0.2268686443567276,
+      0.2264840453863144,
+      0.22687679529190063,
+      0.2271483689546585,
+      0.22968031466007233,
+      0.22700731456279755,
+      0.2234393209218979,
+      0.22132118046283722,
+      0.21585044264793396,
+      0.22008442878723145,
+      0.22143056988716125,
+      0.2230714112520218,
+      0.22327327728271484,
+      0.2229500561952591,
+      0.2197694629430771,
+      0.22258102893829346,
+      0.2234944850206375,
+      0.22243764996528625,
+      0.2204165905714035,
+      0.22256164252758026,
+      0.22307437658309937,
+      0.22309176623821259,
+      0.22266785800457,
+      0.2229180932044983,
+      0.22392676770687103,
+      0.22399261593818665,
+      0.2252259999513626,
+      0.22278866171836853,
+      0.22075329720973969,
+      0.22390493750572205,
+      0.22366319596767426,
+      0.22134855389595032,
+      0.22104404866695404,
+      0.22058901190757751,
+      0.22168795764446259,
+      0.21861295402050018,
+      0.21891270577907562,
+      0.2173292189836502,
+      0.21650256216526031,
+      0.2147790789604187,
+      0.2152273803949356,
+      0.2126801759004593,
+      0.21074366569519043,
+      0.2088627964258194,
+      0.20668046176433563,
+      0.2081356644630432,
+      0.20418785512447357,
+      0.2027214765548706,
+      0.20179711282253265,
+      0.2047148197889328,
+      0.20528995990753174,
+      0.2082604467868805,
+      0.20916081964969635,
+      0.20792485773563385,
+      0.20808105170726776,
+      0.2057202160358429,
+      0.20713961124420166,
+      0.2063913643360138,
+      0.20946520566940308,
+      0.21049541234970093,
+      0.20999348163604736,
+      0.21008725464344025,
+      0.20702557265758514,
+      0.20705953240394592,
+      0.2053450644016266,
+      0.20888005197048187,
+      0.20728805661201477,
+      0.2072432041168213,
+      0.2075180858373642,
+      0.20379330217838287,
+      0.20430469512939453,
+      0.203228160738945,
+      0.2039157599210739,
+      0.19957931339740753,
+      0.20222312211990356,
+      0.20464438199996948,
+      0.20372842252254486,
+      0.20377814769744873,
+      0.2038251906633377,
+      0.20239806175231934,
+      0.20012249052524567,
+      0.19935493171215057,
+      0.19491460919380188,
+      0.1950601041316986,
+      0.1933392435312271,
+      0.1931627243757248,
+      0.194549560546875,
+      0.19782963395118713,
+      0.1952061504125595,
+      0.19629552960395813,
+      0.19705578684806824,
+      0.1944155991077423,
+      0.19655749201774597,
+      0.19557835161685944,
+      0.19652190804481506,
+      0.19533191621303558,
+      0.19499757885932922,
+      0.19416728615760803,
+      0.19508403539657593,
+      0.192694753408432,
+      0.19308702647686005,
+      0.19311173260211945,
+      0.1946423202753067,
+      0.19308170676231384,
+      0.19140398502349854,
+      0.1922903060913086,
+      0.1936880648136139,
+      0.19247913360595703,
+      0.1918903887271881,
+      0.19200348854064941,
+      0.1914215385913849,
+      0.19212058186531067,
+      0.1931421309709549,
+      0.19275791943073273,
+      0.1919129341840744,
+      0.19288034737110138,
+      0.19191424548625946,
+      0.1923104077577591,
+      0.19141116738319397,
+      0.19080063700675964,
+      0.1909828782081604,
+      0.19203798472881317,
+      0.19173741340637207,
+      0.19222232699394226,
+      0.19140870869159698,
+      0.1906118392944336,
+      0.19115376472473145,
+      0.19102847576141357,
+      0.1911878138780594,
+      0.19050511717796326,
+      0.18949173390865326,
+      0.18837480247020721,
+      0.18857349455356598,
+      0.19203560054302216,
+      0.19109252095222473,
+      0.19306501746177673,
+      0.19350570440292358,
+      0.19490084052085876,
+      0.1954215168952942,
+      0.19767098128795624,
+      0.19803857803344727,
+      0.19837264716625214,
+      0.19853557646274567,
+      0.19824384152889252,
+      0.199829563498497,
+      0.19941304624080658,
+      0.2001725435256958,
+      0.20109978318214417,
+      0.20163023471832275,
+      0.20199553668498993,
+      0.20207828283309937,
+      0.2020748108625412,
+      0.2010546326637268,
+      0.2014133781194687,
+      0.20234157145023346,
+      0.20224446058273315,
+      0.20123662054538727,
+      0.20157521963119507,
+      0.201634019613266,
+      0.20164793729782104,
+      0.20207831263542175,
+      0.20247094333171844,
+      0.2016265094280243,
+      0.20163075625896454,
+      0.20101690292358398,
+      0.20207585394382477,
+      0.2025221586227417,
+      0.20208317041397095,
+      0.20152181386947632,
+      0.2012781947851181,
+      0.20215976238250732,
+      0.20119856297969818,
+      0.20118388533592224,
+      0.20139957964420319,
+      0.2009301483631134,
+      0.2007567584514618,
+      0.20078420639038086,
+      0.20057371258735657,
+      0.20089110732078552,
+      0.20103169977664948,
+      0.20069797337055206,
+      0.2004106193780899,
+      0.20041851699352264,
+      0.20023564994335175,
+      0.20043036341667175,
+      0.20020100474357605,
+      0.20024654269218445,
+      0.20010225474834442,
+      0.1999865621328354,
+      0.1999346911907196,
+      0.2005068063735962,
+      0.20037813484668732,
+      0.20055745542049408,
+      0.20020297169685364,
+      0.20010267198085785,
+      0.19998785853385925,
+      0.2001042664051056
+    ],
+    "lr": [
+      4.188034188034189e-06,
+      8.461538461538462e-06,
+      1.2735042735042738e-05,
+      1.700854700854701e-05,
+      2.1282051282051282e-05,
+      2.5555555555555557e-05,
+      2.9829059829059833e-05,
+      3.4102564102564105e-05,
+      3.837606837606838e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.992995647012267e-05,
+      3.963316185199842e-05,
+      3.933636723387416e-05,
+      3.90395726157499e-05,
+      3.874277799762565e-05,
+      3.844598337950139e-05,
+      3.814918876137713e-05,
+      3.785239414325287e-05,
+      3.7555599525128613e-05,
+      3.725880490700435e-05,
+      3.696201028888009e-05,
+      3.666521567075585e-05,
+      3.6368421052631585e-05,
+      3.607162643450732e-05,
+      3.577483181638307e-05,
+      3.547803719825881e-05,
+      3.518124258013455e-05,
+      3.4884447962010287e-05,
+      3.458765334388604e-05,
+      3.4290858725761776e-05,
+      3.399406410763751e-05,
+      3.3697269489513265e-05,
+      3.3400474871389e-05,
+      3.310368025326474e-05,
+      3.280688563514048e-05,
+      3.251009101701623e-05,
+      3.2213296398891967e-05,
+      3.191650178076771e-05,
+      3.1542540561931145e-05,
+      3.124574594380688e-05,
+      3.094895132568263e-05,
+      3.065215670755838e-05,
+      3.0355362089434113e-05,
+      3.0058567471309854e-05,
+      2.97617728531856e-05,
+      2.9464978235061336e-05,
+      2.9168183616937077e-05,
+      2.8871388998812818e-05,
+      2.857459438068857e-05,
+      2.8277799762564307e-05,
+      2.798100514444005e-05,
+      2.7684210526315793e-05,
+      2.738741590819153e-05,
+      2.709062129006727e-05,
+      2.6793826671943013e-05,
+      2.6497032053818764e-05,
+      2.62002374356945e-05,
+      2.5903442817570243e-05,
+      2.5606648199445987e-05,
+      2.5309853581321725e-05,
+      2.5013058963197466e-05,
+      2.4716264345073214e-05,
+      2.441946972694896e-05,
+      2.4122675108824693e-05,
+      2.3825880490700434e-05,
+      2.352908587257618e-05,
+      2.3232291254451916e-05,
+      2.293549663632766e-05,
+      2.2638702018203408e-05,
+      2.226474079936684e-05,
+      2.196794618124258e-05,
+      2.1671151563118328e-05,
+      2.1374356944994066e-05,
+      2.1077562326869803e-05,
+      2.0780767708745554e-05,
+      2.0483973090621296e-05,
+      2.018717847249703e-05,
+      1.989038385437277e-05,
+      1.959358923624852e-05,
+      1.929679461812426e-05,
+      1.8999999999999998e-05,
+      1.8703205381875746e-05,
+      1.8406410763751487e-05,
+      1.8109616145627224e-05,
+      1.7812821527502965e-05,
+      1.7516026909378713e-05,
+      1.7219232291254454e-05,
+      1.6922437673130192e-05,
+      1.662564305500594e-05,
+      1.632884843688168e-05,
+      1.603205381875742e-05,
+      1.573525920063316e-05,
+      1.5438464582508908e-05,
+      1.5141669964384647e-05,
+      1.4844875346260385e-05,
+      1.4548080728136134e-05,
+      1.4251286110011875e-05,
+      1.3954491491887611e-05,
+      1.3657696873763363e-05,
+      1.33609022556391e-05,
+      1.298694103680253e-05,
+      1.2690146418678277e-05,
+      1.2393351800554018e-05,
+      1.2096557182429756e-05,
+      1.1799762564305506e-05,
+      1.1502967946181245e-05,
+      1.1206173328056984e-05,
+      1.0909378709932724e-05,
+      1.0612584091808472e-05,
+      1.0315789473684213e-05,
+      1.0018994855559949e-05,
+      9.7222002374357e-06,
+      9.425405619311438e-06,
+      9.128611001187177e-06,
+      8.831816383062916e-06,
+      8.535021764938666e-06,
+      8.238227146814405e-06,
+      7.941432528690143e-06,
+      7.644637910565893e-06,
+      7.347843292441632e-06,
+      7.0510486743173715e-06,
+      6.75425405619311e-06,
+      6.45745943806886e-06,
+      6.160664819944598e-06,
+      5.8638702018203375e-06,
+      5.567075583696086e-06,
+      5.270280965571826e-06,
+      4.973486347447565e-06,
+      4.676691729323314e-06,
+      4.379897111199053e-06,
+      4.083102493074792e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      1550,
+      3113,
+      4676,
+      6239,
+      7802,
+      9365,
+      10928,
+      12491,
+      14054,
+      15617
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 1.0,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 1.0,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 1.0,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 1.0,
+  "sft_overall_accuracy": 1.0
+}

add_sub_sorl_v1_abs30_K1_100K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e8b4c1e17d72c9091961d6426ecc8aecb49fc0016bf701e6079d6a99ddd0ef8
+size 650385300

add_sub_sorl_v1_abs30_K1_100K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 1562,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K1_100K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 100000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162519662,
+  "run_name": "add_sub_sorl_v1_abs30_K1_100K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T15:30:36.095637+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "4zimijm6",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/4zimijm6",
+  "final_accuracy": 1.0,
+  "sft_accuracy": 1.0,
+  "eval_method": "ArithmeticEvaluator"
+}