Upload add_sub_sorl_v1_abs30_100K

add_sub_sorl_v1_abs30_100K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_100K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_100K/metrics.json

@@ -0,0 +1,3537 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1613,
+      1663,
+      1713,
+      1763,
+      1813,
+      1863,
+      1913,
+      1963,
+      2013,
+      2063,
+      2113,
+      2163,
+      2213,
+      2263,
+      2313,
+      2363,
+      2413,
+      2463,
+      2513,
+      2563,
+      2613,
+      2663,
+      2713,
+      2763,
+      2813,
+      2863,
+      2913,
+      2963,
+      3013,
+      3063,
+      3113,
+      3176,
+      3226,
+      3276,
+      3326,
+      3376,
+      3426,
+      3476,
+      3526,
+      3576,
+      3626,
+      3676,
+      3726,
+      3776,
+      3826,
+      3876,
+      3926,
+      3976,
+      4026,
+      4076,
+      4126,
+      4176,
+      4226,
+      4276,
+      4326,
+      4376,
+      4426,
+      4476,
+      4526,
+      4576,
+      4626,
+      4676,
+      4739,
+      4789,
+      4839,
+      4889,
+      4939,
+      4989,
+      5039,
+      5089,
+      5139,
+      5189,
+      5239,
+      5289,
+      5339,
+      5389,
+      5439,
+      5489,
+      5539,
+      5589,
+      5639,
+      5689,
+      5739,
+      5789,
+      5839,
+      5889,
+      5939,
+      5989,
+      6039,
+      6089,
+      6139,
+      6189,
+      6239,
+      6302,
+      6352,
+      6402,
+      6452,
+      6502,
+      6552,
+      6602,
+      6652,
+      6702,
+      6752,
+      6802,
+      6852,
+      6902,
+      6952,
+      7002,
+      7052,
+      7102,
+      7152,
+      7202,
+      7252,
+      7302,
+      7352,
+      7402,
+      7452,
+      7502,
+      7552,
+      7602,
+      7652,
+      7702,
+      7752,
+      7802,
+      7865,
+      7915,
+      7965,
+      8015,
+      8065,
+      8115,
+      8165,
+      8215,
+      8265,
+      8315,
+      8365,
+      8415,
+      8465,
+      8515,
+      8565,
+      8615,
+      8665,
+      8715,
+      8765,
+      8815,
+      8865,
+      8915,
+      8965,
+      9015,
+      9065,
+      9115,
+      9165,
+      9215,
+      9265,
+      9315,
+      9365,
+      9428,
+      9478,
+      9528,
+      9578,
+      9628,
+      9678,
+      9728,
+      9778,
+      9828,
+      9878,
+      9928,
+      9978,
+      10028,
+      10078,
+      10128,
+      10178,
+      10228,
+      10278,
+      10328,
+      10378,
+      10428,
+      10478,
+      10528,
+      10578,
+      10628,
+      10678,
+      10728,
+      10778,
+      10828,
+      10878,
+      10928,
+      10991,
+      11041,
+      11091,
+      11141,
+      11191,
+      11241,
+      11291,
+      11341,
+      11391,
+      11441,
+      11491,
+      11541,
+      11591,
+      11641,
+      11691,
+      11741,
+      11791,
+      11841,
+      11891,
+      11941,
+      11991,
+      12041,
+      12091,
+      12141,
+      12191,
+      12241,
+      12291,
+      12341,
+      12391,
+      12441,
+      12491,
+      12554,
+      12604,
+      12654,
+      12704,
+      12754,
+      12804,
+      12854,
+      12904,
+      12954,
+      13004,
+      13054,
+      13104,
+      13154,
+      13204,
+      13254,
+      13304,
+      13354,
+      13404,
+      13454,
+      13504,
+      13554,
+      13604,
+      13654,
+      13704,
+      13754,
+      13804,
+      13854,
+      13904,
+      13954,
+      14004,
+      14054,
+      14117,
+      14167,
+      14217,
+      14267,
+      14317,
+      14367,
+      14417,
+      14467,
+      14517,
+      14567,
+      14617,
+      14667,
+      14717,
+      14767,
+      14817,
+      14867,
+      14917,
+      14967,
+      15017,
+      15067,
+      15117,
+      15167,
+      15217,
+      15267,
+      15317,
+      15367,
+      15417,
+      15467,
+      15517,
+      15567,
+      15617
+    ],
+    "loss": [
+      16.353931427001953,
+      11.875846862792969,
+      9.565509796142578,
+      7.342390060424805,
+      4.512979507446289,
+      4.0690813064575195,
+      2.862144947052002,
+      2.254006862640381,
+      2.57066011428833,
+      1.998836874961853,
+      2.1581692695617676,
+      2.091899871826172,
+      2.1439857482910156,
+      1.2456738948822021,
+      1.5288512706756592,
+      1.3524699211120605,
+      1.3221745491027832,
+      1.198272943496704,
+      0.07600802183151245,
+      0.367651104927063,
+      -0.7961059808731079,
+      -3.9763927459716797,
+      -7.191513538360596,
+      -7.797391891479492,
+      -8.402938842773438,
+      -8.945942878723145,
+      -9.609200477600098,
+      -9.026281356811523,
+      -9.448883056640625,
+      -9.915501594543457,
+      -10.245144844055176,
+      -11.20373249053955,
+      -11.372387886047363,
+      -11.372745513916016,
+      -12.334482192993164,
+      -12.151872634887695,
+      -12.037510871887207,
+      -12.56748104095459,
+      -13.159852027893066,
+      -12.560744285583496,
+      -12.327042579650879,
+      -13.098484992980957,
+      -13.265353202819824,
+      -13.08821964263916,
+      -12.599993705749512,
+      -13.491194725036621,
+      -13.431854248046875,
+      -13.41433334350586,
+      -13.12167739868164,
+      -13.244621276855469,
+      -14.1083345413208,
+      -13.758782386779785,
+      -13.726495742797852,
+      -13.88464641571045,
+      -13.157135963439941,
+      -13.105184555053711,
+      -12.805330276489258,
+      -13.57339096069336,
+      -14.142427444458008,
+      -14.489392280578613,
+      -14.23194694519043,
+      -14.446044921875,
+      -13.665212631225586,
+      -14.616517066955566,
+      -14.436517715454102,
+      -13.845361709594727,
+      -13.804733276367188,
+      -13.730453491210938,
+      -14.256049156188965,
+      -13.56989574432373,
+      -13.875513076782227,
+      -13.14440631866455,
+      -14.191486358642578,
+      -14.01298999786377,
+      -14.247776985168457,
+      -13.457542419433594,
+      -14.019203186035156,
+      -13.876285552978516,
+      -13.785099983215332,
+      -14.36813735961914,
+      -14.329707145690918,
+      -13.76140308380127,
+      -13.874360084533691,
+      -13.979138374328613,
+      -14.142152786254883,
+      -13.983951568603516,
+      -14.2293119430542,
+      -14.110427856445312,
+      -13.481212615966797,
+      -13.352953910827637,
+      -13.77885627746582,
+      -13.672810554504395,
+      -14.271621704101562,
+      -14.000099182128906,
+      -13.626543998718262,
+      -13.962502479553223,
+      -13.572952270507812,
+      -13.996663093566895,
+      -14.078392028808594,
+      -13.966607093811035,
+      -13.693870544433594,
+      -14.399123191833496,
+      -14.0098876953125,
+      -13.81348991394043,
+      -14.445164680480957,
+      -13.953023910522461,
+      -13.390979766845703,
+      -14.02158260345459,
+      -13.58531665802002,
+      -14.071144104003906,
+      -14.009268760681152,
+      -14.096712112426758,
+      -14.132247924804688,
+      -13.767648696899414,
+      -13.715518951416016,
+      -13.647896766662598,
+      -14.147568702697754,
+      -13.230199813842773,
+      -13.99448013305664,
+      -13.923420906066895,
+      -13.942337989807129,
+      -13.804627418518066,
+      -13.392175674438477,
+      -12.837953567504883,
+      -13.070643424987793,
+      -13.120101928710938,
+      -13.86976432800293,
+      -13.643270492553711,
+      -13.369363784790039,
+      -12.956626892089844,
+      -13.1434965133667,
+      -13.507857322692871,
+      -13.006118774414062,
+      -12.692553520202637,
+      -12.813122749328613,
+      -12.225473403930664,
+      -11.903809547424316,
+      -10.268694877624512,
+      -8.495654106140137,
+      -7.033803462982178,
+      -6.582006454467773,
+      -5.818358898162842,
+      -5.196183681488037,
+      -4.915243148803711,
+      -4.249300956726074,
+      -4.208031177520752,
+      -4.276210784912109,
+      -4.114187240600586,
+      -4.079606056213379,
+      -3.9176602363586426,
+      -3.651959180831909,
+      -3.572800636291504,
+      -3.899899959564209,
+      -3.253357172012329,
+      -3.1612918376922607,
+      -3.3208858966827393,
+      -3.3848090171813965,
+      -3.2022788524627686,
+      -3.420503616333008,
+      -3.2170166969299316,
+      -3.079822301864624,
+      -2.739346504211426,
+      -2.9486916065216064,
+      -3.182239532470703,
+      -2.7178120613098145,
+      -3.3026247024536133,
+      -3.1974449157714844,
+      -2.9400928020477295,
+      -3.053345203399658,
+      -2.314910888671875,
+      -2.9756245613098145,
+      -2.937688112258911,
+      -2.677002429962158,
+      -2.8224380016326904,
+      -2.768467426300049,
+      -2.4176738262176514,
+      -2.5374021530151367,
+      -3.2440459728240967,
+      -2.8884079456329346,
+      -2.809415578842163,
+      -2.4866552352905273,
+      -2.7317116260528564,
+      -1.8125377893447876,
+      -2.617854595184326,
+      -1.9995617866516113,
+      -2.7783727645874023,
+      -3.1240453720092773,
+      -2.210836887359619,
+      -2.5533175468444824,
+      -2.9647233486175537,
+      -2.007883071899414,
+      -2.233142137527466,
+      -2.3278913497924805,
+      -2.538228750228882,
+      -2.2664778232574463,
+      -1.813431739807129,
+      -2.4405009746551514,
+      -2.3692400455474854,
+      -2.074908494949341,
+      -2.0504086017608643,
+      -2.2472660541534424,
+      -2.0656344890594482,
+      -2.357393503189087,
+      -1.9783142805099487,
+      -2.097346305847168,
+      -2.335299491882324,
+      -1.8233290910720825,
+      -2.2929933071136475,
+      -1.7413837909698486,
+      -2.384492874145508,
+      -2.1335105895996094,
+      -1.5464704036712646,
+      -1.9119561910629272,
+      -1.6973811388015747,
+      -2.2817766666412354,
+      -2.3222947120666504,
+      -2.296046018600464,
+      -1.7667940855026245,
+      -1.6567730903625488,
+      -1.6351410150527954,
+      -1.4980250597000122,
+      -2.189903497695923,
+      -1.7353428602218628,
+      -1.9462618827819824,
+      -1.4697139263153076,
+      -1.7280217409133911,
+      -2.411081075668335,
+      -2.1487488746643066,
+      -1.7597498893737793,
+      -1.771265983581543,
+      -1.8300777673721313,
+      -1.9071307182312012,
+      -1.5723772048950195,
+      -2.0543878078460693,
+      -1.7049565315246582,
+      -1.7995303869247437,
+      -1.7827564477920532,
+      -2.0536272525787354,
+      -1.9578628540039062,
+      -1.6535711288452148,
+      -1.9253246784210205,
+      -1.9923954010009766,
+      -1.931577444076538,
+      -1.4669015407562256,
+      -1.898777961730957,
+      -1.882079005241394,
+      -1.8351067304611206,
+      -1.8482004404067993,
+      -1.4887627363204956,
+      -1.5922356843948364,
+      -1.6491912603378296,
+      -1.5675148963928223,
+      -1.720570683479309,
+      -1.4480400085449219,
+      -1.584964632987976,
+      -1.2510169744491577,
+      -1.4396697282791138,
+      -1.7025628089904785,
+      -1.57511305809021,
+      -1.8179688453674316,
+      -1.8571149110794067,
+      -1.4414499998092651,
+      -1.3842190504074097,
+      -1.6883375644683838,
+      -1.601452350616455,
+      -1.8336045742034912,
+      -1.284399151802063,
+      -1.288986325263977,
+      -1.5190696716308594,
+      -1.4586634635925293,
+      -1.4343172311782837,
+      -1.4818658828735352,
+      -1.3453775644302368,
+      -1.5217355489730835,
+      -1.6131032705307007,
+      -1.614372968673706,
+      -1.6065467596054077,
+      -1.6470931768417358,
+      -1.7630650997161865,
+      -1.1220555305480957,
+      -1.5090004205703735,
+      -1.3956247568130493,
+      -1.3273143768310547,
+      -1.465193510055542,
+      -1.2085615396499634,
+      -1.5898776054382324,
+      -1.4977500438690186,
+      -1.2314560413360596,
+      -1.574336051940918,
+      -1.5404393672943115,
+      -1.5331313610076904,
+      -1.3918012380599976,
+      -1.1956740617752075,
+      -1.262143850326538,
+      -1.4153560400009155,
+      -1.6031506061553955,
+      -1.4635738134384155,
+      -1.4370183944702148,
+      -1.3306372165679932,
+      -1.4228017330169678,
+      -1.3455251455307007,
+      -1.3399231433868408,
+      -1.178405523300171,
+      -1.2652480602264404,
+      -1.3037034273147583,
+      -1.2954355478286743,
+      -1.4495939016342163,
+      -1.5041433572769165,
+      -1.499807596206665,
+      -1.6011711359024048
+    ],
+    "base_loss": [
+      10.583792686462402,
+      8.066454887390137,
+      7.359574317932129,
+      6.420194149017334,
+      5.490340232849121,
+      3.802809238433838,
+      2.3162405490875244,
+      2.084392547607422,
+      1.9193041324615479,
+      1.8505445718765259,
+      1.8642131090164185,
+      1.8125228881835938,
+      1.856522560119629,
+      1.8430544137954712,
+      1.8256675004959106,
+      1.8333860635757446,
+      1.7693449258804321,
+      1.8187506198883057,
+      1.8742154836654663,
+      1.797965407371521,
+      1.7965906858444214,
+      1.8261682987213135,
+      1.917641520500183,
+      1.7743865251541138,
+      1.7797601222991943,
+      1.7174644470214844,
+      1.7597808837890625,
+      1.641912579536438,
+      1.6515651941299438,
+      1.7023732662200928,
+      1.6172168254852295,
+      1.7306687831878662,
+      1.701155424118042,
+      1.6607495546340942,
+      1.6990290880203247,
+      1.6403274536132812,
+      1.6816343069076538,
+      1.679869532585144,
+      1.6743011474609375,
+      1.6315648555755615,
+      1.5742610692977905,
+      1.6771800518035889,
+      1.6129316091537476,
+      1.6053324937820435,
+      1.569964051246643,
+      1.6778780221939087,
+      1.6199839115142822,
+      1.6674938201904297,
+      1.6263078451156616,
+      1.623223066329956,
+      1.6771548986434937,
+      1.6433137655258179,
+      1.6295852661132812,
+      1.637642741203308,
+      1.606334924697876,
+      1.5829112529754639,
+      1.5223848819732666,
+      1.59457266330719,
+      1.630057454109192,
+      1.6733797788619995,
+      1.6320444345474243,
+      1.6473969221115112,
+      1.5808079242706299,
+      1.670488953590393,
+      1.6722688674926758,
+      1.591480016708374,
+      1.616097331047058,
+      1.585060715675354,
+      1.6227563619613647,
+      1.5591118335723877,
+      1.5854694843292236,
+      1.5751146078109741,
+      1.6217409372329712,
+      1.6154325008392334,
+      1.6229082345962524,
+      1.5588018894195557,
+      1.5817945003509521,
+      1.565144658088684,
+      1.5754756927490234,
+      1.6284313201904297,
+      1.6797740459442139,
+      1.5707577466964722,
+      1.572632908821106,
+      1.577247977256775,
+      1.5987508296966553,
+      1.6152507066726685,
+      1.6013938188552856,
+      1.6258388757705688,
+      1.5390816926956177,
+      1.5352532863616943,
+      1.605579137802124,
+      1.5561282634735107,
+      1.6051266193389893,
+      1.58786940574646,
+      1.5443556308746338,
+      1.6032781600952148,
+      1.5615407228469849,
+      1.5798014402389526,
+      1.5793745517730713,
+      1.5702213048934937,
+      1.5427759885787964,
+      1.62362802028656,
+      1.5772150754928589,
+      1.557128667831421,
+      1.6311465501785278,
+      1.579343557357788,
+      1.5237773656845093,
+      1.5838626623153687,
+      1.5710700750350952,
+      1.5849696397781372,
+      1.5868189334869385,
+      1.580222487449646,
+      1.5938657522201538,
+      1.557015061378479,
+      1.5417041778564453,
+      1.5307978391647339,
+      1.6002476215362549,
+      1.4829028844833374,
+      1.5972959995269775,
+      1.5759131908416748,
+      1.5614700317382812,
+      1.548891305923462,
+      1.507192611694336,
+      1.4467004537582397,
+      1.467873454093933,
+      1.4952201843261719,
+      1.5538300275802612,
+      1.5384445190429688,
+      1.5040147304534912,
+      1.4669249057769775,
+      1.4912463426589966,
+      1.523524522781372,
+      1.4744774103164673,
+      1.4265775680541992,
+      1.4518784284591675,
+      1.3725693225860596,
+      1.3456714153289795,
+      1.1582497358322144,
+      0.9556323289871216,
+      0.8228430151939392,
+      0.7601378560066223,
+      0.6596508026123047,
+      0.5946040153503418,
+      0.5619922280311584,
+      0.5001639723777771,
+      0.49094894528388977,
+      0.5127306580543518,
+      0.5127319097518921,
+      0.46609529852867126,
+      0.4519447684288025,
+      0.42315152287483215,
+      0.4051882326602936,
+      0.4525225758552551,
+      0.39757633209228516,
+      0.3720526099205017,
+      0.41721227765083313,
+      0.4012520909309387,
+      0.377694696187973,
+      0.42150411009788513,
+      0.372927188873291,
+      0.35469961166381836,
+      0.318009614944458,
+      0.3455231189727783,
+      0.3857307434082031,
+      0.3480704724788666,
+      0.37514179944992065,
+      0.3784138560295105,
+      0.34017255902290344,
+      0.34929636120796204,
+      0.28583255410194397,
+      0.3519895076751709,
+      0.33482006192207336,
+      0.31295791268348694,
+      0.3224482834339142,
+      0.3196558654308319,
+      0.2840682864189148,
+      0.29912450909614563,
+      0.36929866671562195,
+      0.34565314650535583,
+      0.3267846703529358,
+      0.3192231059074402,
+      0.31563252210617065,
+      0.2777107059955597,
+      0.29768094420433044,
+      0.24216410517692566,
+      0.32542404532432556,
+      0.36056867241859436,
+      0.25145214796066284,
+      0.29183194041252136,
+      0.33972468972206116,
+      0.2686130702495575,
+      0.258231520652771,
+      0.2671636641025543,
+      0.2917363941669464,
+      0.25982794165611267,
+      0.23204629123210907,
+      0.28231772780418396,
+      0.27783599495887756,
+      0.2604861855506897,
+      0.2384457290172577,
+      0.255441814661026,
+      0.23715876042842865,
+      0.28448793292045593,
+      0.24126291275024414,
+      0.24047763645648956,
+      0.2671874761581421,
+      0.21945752203464508,
+      0.271971195936203,
+      0.23857174813747406,
+      0.2868176996707916,
+      0.2462073415517807,
+      0.20088985562324524,
+      0.23240797221660614,
+      0.1949901431798935,
+      0.2621559202671051,
+      0.26746684312820435,
+      0.2596105635166168,
+      0.2330942153930664,
+      0.19325841963291168,
+      0.19163039326667786,
+      0.17575077712535858,
+      0.2536766827106476,
+      0.2009972780942917,
+      0.23216930031776428,
+      0.18310262262821198,
+      0.19874657690525055,
+      0.27389222383499146,
+      0.2504861354827881,
+      0.2032155990600586,
+      0.21124167740345,
+      0.22088196873664856,
+      0.22071300446987152,
+      0.19026194512844086,
+      0.23354355990886688,
+      0.1986234188079834,
+      0.20554324984550476,
+      0.20325851440429688,
+      0.23881973326206207,
+      0.2363194078207016,
+      0.19851446151733398,
+      0.22320447862148285,
+      0.22992189228534698,
+      0.23931196331977844,
+      0.16839899122714996,
+      0.21563445031642914,
+      0.21595732867717743,
+      0.22108875215053558,
+      0.21586854755878448,
+      0.171113982796669,
+      0.18106362223625183,
+      0.19026419520378113,
+      0.17992354929447174,
+      0.19903604686260223,
+      0.17338071763515472,
+      0.18208308517932892,
+      0.15474160015583038,
+      0.16776002943515778,
+      0.19435231387615204,
+      0.180166557431221,
+      0.20833714306354523,
+      0.2148302048444748,
+      0.16802255809307098,
+      0.165029838681221,
+      0.19663789868354797,
+      0.19127193093299866,
+      0.21039649844169617,
+      0.1488167941570282,
+      0.1682043820619583,
+      0.17758512496948242,
+      0.16937068104743958,
+      0.175533264875412,
+      0.17081210017204285,
+      0.15519201755523682,
+      0.17511597275733948,
+      0.18463636934757233,
+      0.1841987818479538,
+      0.18462054431438446,
+      0.18654513359069824,
+      0.20274583995342255,
+      0.1301838606595993,
+      0.17495934665203094,
+      0.17271673679351807,
+      0.15609420835971832,
+      0.1670539379119873,
+      0.13983602821826935,
+      0.1888965219259262,
+      0.17140711843967438,
+      0.14202818274497986,
+      0.17892365157604218,
+      0.17807415127754211,
+      0.1745242327451706,
+      0.1601133644580841,
+      0.1374666690826416,
+      0.14927272498607635,
+      0.16191641986370087,
+      0.18267491459846497,
+      0.167189359664917,
+      0.16302575170993805,
+      0.1522405445575714,
+      0.16383804380893707,
+      0.1544777899980545,
+      0.15203754603862762,
+      0.1365201622247696,
+      0.1471976339817047,
+      0.14924056828022003,
+      0.1483282744884491,
+      0.16570642590522766,
+      0.17113836109638214,
+      0.17227718234062195,
+      0.1809764951467514
+    ],
+    "info_loss": [
+      -0.13822269439697266,
+      -0.23614931106567383,
+      -0.23624515533447266,
+      -0.23105287551879883,
+      -0.3242683410644531,
+      -0.12394833564758301,
+      -0.07029557228088379,
+      -0.09893393516540527,
+      -0.046825528144836426,
+      -0.09740293025970459,
+      -0.08192431926727295,
+      -0.08307325839996338,
+      -0.08164334297180176,
+      -0.16988539695739746,
+      -0.1397857666015625,
+      -0.15783727169036865,
+      -0.15407419204711914,
+      -0.17148029804229736,
+      -0.28766489028930664,
+      -0.24382555484771729,
+      -0.3434123992919922,
+      -0.6404982805252075,
+      -0.9535881876945496,
+      -0.9920961856842041,
+      -1.048412561416626,
+      -1.0944911241531372,
+      -1.1628456115722656,
+      -1.0930302143096924,
+      -1.135810136795044,
+      -1.1862694025039673,
+      -1.2093408107757568,
+      -1.3165074586868286,
+      -1.328871250152588,
+      -1.3243379592895508,
+      -1.4235337972640991,
+      -1.4003045558929443,
+      -1.391230583190918,
+      -1.4437508583068848,
+      -1.5035120248794556,
+      -1.4391214847564697,
+      -1.4072514772415161,
+      -1.494908332824707,
+      -1.504333734512329,
+      -1.4851349592208862,
+      -1.4329886436462402,
+      -1.5316672325134277,
+      -1.5204179286956787,
+      -1.5225586891174316,
+      -1.4892361164093018,
+      -1.5005210638046265,
+      -1.5909022092819214,
+      -1.5518944263458252,
+      -1.547243356704712,
+      -1.5632907152175903,
+      -1.4881101846694946,
+      -1.4804902076721191,
+      -1.4432587623596191,
+      -1.528035283088684,
+      -1.5881996154785156,
+      -1.6270760297775269,
+      -1.5966256856918335,
+      -1.6196050643920898,
+      -1.53391432762146,
+      -1.637223482131958,
+      -1.619842529296875,
+      -1.554161548614502,
+      -1.551828145980835,
+      -1.5415235757827759,
+      -1.59657883644104,
+      -1.5226147174835205,
+      -1.5548325777053833,
+      -1.4818652868270874,
+      -1.5915569067001343,
+      -1.5711668729782104,
+      -1.5963202714920044,
+      -1.510689377784729,
+      -1.5685454607009888,
+      -1.552243709564209,
+      -1.5444592237472534,
+      -1.6080268621444702,
+      -1.609379768371582,
+      -1.5412710905075073,
+      -1.5527178049087524,
+      -1.5639114379882812,
+      -1.5819191932678223,
+      -1.5679680109024048,
+      -1.5901685953140259,
+      -1.582194209098816,
+      -1.5112979412078857,
+      -1.4969711303710938,
+      -1.5462664365768433,
+      -1.5296250581741333,
+      -1.5964444875717163,
+      -1.56596040725708,
+      -1.5249685049057007,
+      -1.5649727582931519,
+      -1.5231212377548218,
+      -1.5654813051223755,
+      -1.5728145837783813,
+      -1.560842752456665,
+      -1.5312010049819946,
+      -1.6095906496047974,
+      -1.566528081893921,
+      -1.5446128845214844,
+      -1.6148988008499146,
+      -1.559473991394043,
+      -1.4984121322631836,
+      -1.5663994550704956,
+      -1.5228196382522583,
+      -1.5721852779388428,
+      -1.5657051801681519,
+      -1.573951244354248,
+      -1.5795037746429443,
+      -1.5383645296096802,
+      -1.532282829284668,
+      -1.5252710580825806,
+      -1.58170485496521,
+      -1.4770863056182861,
+      -1.5646504163742065,
+      -1.5573749542236328,
+      -1.5571423768997192,
+      -1.5411075353622437,
+      -1.4958924055099487,
+      -1.4339350461959839,
+      -1.4592657089233398,
+      -1.4667996168136597,
+      -1.5483533143997192,
+      -1.525352954864502,
+      -1.494500756263733,
+      -1.4485323429107666,
+      -1.470154881477356,
+      -1.5094600915908813,
+      -1.4534693956375122,
+      -1.4192062616348267,
+      -1.430849313735962,
+      -1.365715742111206,
+      -1.3308629989624023,
+      -1.148105502128601,
+      -0.9496700763702393,
+      -0.7906749844551086,
+      -0.7397457957267761,
+      -0.6535822749137878,
+      -0.5858854055404663,
+      -0.5533480048179626,
+      -0.48072874546051025,
+      -0.47489917278289795,
+      -0.48632532358169556,
+      -0.4680827856063843,
+      -0.4620489180088043,
+      -0.44253870844841003,
+      -0.41357550024986267,
+      -0.4023188352584839,
+      -0.44056105613708496,
+      -0.3704988360404968,
+      -0.3581610918045044,
+      -0.37899795174598694,
+      -0.38355591893196106,
+      -0.36309462785720825,
+      -0.3893692195415497,
+      -0.3648320138454437,
+      -0.34819015860557556,
+      -0.3114086389541626,
+      -0.3350915312767029,
+      -0.36147013306617737,
+      -0.31299611926078796,
+      -0.3713478147983551,
+      -0.36294907331466675,
+      -0.33462783694267273,
+      -0.34611016511917114,
+      -0.265610009431839,
+      -0.33830228447914124,
+      -0.3326902389526367,
+      -0.30470603704452515,
+      -0.31881779432296753,
+      -0.31391364336013794,
+      -0.2746647894382477,
+      -0.2890484035015106,
+      -0.3678426444530487,
+      -0.3283480405807495,
+      -0.31812164187431335,
+      -0.2873300015926361,
+      -0.3102010190486908,
+      -0.21345168352127075,
+      -0.2961680591106415,
+      -0.22915704548358917,
+      -0.3161178231239319,
+      -0.3535863757133484,
+      -0.2500080168247223,
+      -0.2886582016944885,
+      -0.3350525498390198,
+      -0.2321719527244568,
+      -0.2558571994304657,
+      -0.2626509368419647,
+      -0.2886291742324829,
+      -0.2582710087299347,
+      -0.2085762321949005,
+      -0.27760010957717896,
+      -0.2701760232448578,
+      -0.23737221956253052,
+      -0.23413917422294617,
+      -0.2549126148223877,
+      -0.23485703766345978,
+      -0.26845604181289673,
+      -0.22846893966197968,
+      -0.23943379521369934,
+      -0.2659887969493866,
+      -0.21204429864883423,
+      -0.2613718509674072,
+      -0.2017609179019928,
+      -0.27505597472190857,
+      -0.2423495203256607,
+      -0.17928309738636017,
+      -0.21955689787864685,
+      -0.1935025304555893,
+      -0.2584647536277771,
+      -0.2643345594406128,
+      -0.258984237909317,
+      -0.20908518135547638,
+      -0.1892842948436737,
+      -0.1874309927225113,
+      -0.17449191212654114,
+      -0.2502930760383606,
+      -0.19810694456100464,
+      -0.22311300039291382,
+      -0.16915571689605713,
+      -0.19783471524715424,
+      -0.27191588282585144,
+      -0.24508187174797058,
+      -0.20009776949882507,
+      -0.2026689201593399,
+      -0.20957694947719574,
+      -0.21848787367343903,
+      -0.18062527477741241,
+      -0.23321673274040222,
+      -0.19512329995632172,
+      -0.20489242672920227,
+      -0.20287422835826874,
+      -0.23402367532253265,
+      -0.22366005182266235,
+      -0.18954592943191528,
+      -0.22067011892795563,
+      -0.22672241926193237,
+      -0.2208995372056961,
+      -0.16791054606437683,
+      -0.21536563336849213,
+      -0.21331483125686646,
+      -0.20978602766990662,
+      -0.21061265468597412,
+      -0.17026440799236298,
+      -0.18054203689098358,
+      -0.18857072293758392,
+      -0.17967435717582703,
+      -0.1986488252878189,
+      -0.16699808835983276,
+      -0.18172207474708557,
+      -0.144958034157753,
+      -0.16621123254299164,
+      -0.19422952830791473,
+      -0.17906028032302856,
+      -0.20765265822410583,
+      -0.2117072194814682,
+      -0.1666542887687683,
+      -0.15931595861911774,
+      -0.19259610772132874,
+      -0.18366043269634247,
+      -0.20917025208473206,
+      -0.14747346937656403,
+      -0.14920273423194885,
+      -0.17642758786678314,
+      -0.16678817570209503,
+      -0.16640248894691467,
+      -0.16970722377300262,
+      -0.15453000366687775,
+      -0.17318691313266754,
+      -0.1839085817337036,
+      -0.1837536096572876,
+      -0.18391980230808258,
+      -0.18582303822040558,
+      -0.20264846086502075,
+      -0.12983135879039764,
+      -0.17368283867835999,
+      -0.16087041795253754,
+      -0.1521586775779724,
+      -0.16673751175403595,
+      -0.13974842429161072,
+      -0.1817723512649536,
+      -0.17127837240695953,
+      -0.14169864356517792,
+      -0.17845776677131653,
+      -0.1764056533575058,
+      -0.17412659525871277,
+      -0.15906667709350586,
+      -0.1370437890291214,
+      -0.14619038999080658,
+      -0.16133344173431396,
+      -0.18260161578655243,
+      -0.16709370911121368,
+      -0.16267499327659607,
+      -0.15166842937469482,
+      -0.16323627531528473,
+      -0.15427052974700928,
+      -0.1518806517124176,
+      -0.1349768340587616,
+      -0.1471162736415863,
+      -0.14908696711063385,
+      -0.14791764318943024,
+      -0.16540926694869995,
+      -0.17110100388526917,
+      -0.17208409309387207,
+      -0.1809041053056717
+    ],
+    "abs_loss": [
+      3.3980460166931152,
+      3.369978427886963,
+      3.3164987564086914,
+      3.2645981311798096,
+      3.1180689334869385,
+      2.889650344848633,
+      2.6817269325256348,
+      2.603313684463501,
+      2.5406126976013184,
+      2.628107786178589,
+      2.6164369583129883,
+      2.6000633239746094,
+      2.565046787261963,
+      2.538106918334961,
+      2.541606903076172,
+      2.4957196712493896,
+      2.472607135772705,
+      2.5442190170288086,
+      2.616762638092041,
+      2.3649964332580566,
+      2.2169899940490723,
+      1.9707590341567993,
+      1.5024638175964355,
+      1.4040004014968872,
+      1.2410645484924316,
+      1.2074388265609741,
+      1.1320998668670654,
+      1.1608312129974365,
+      1.1682783365249634,
+      1.1260359287261963,
+      1.0213239192962646,
+      1.0781991481781006,
+      0.9062473773956299,
+      0.8747034072875977,
+      0.8599637150764465,
+      1.0045547485351562,
+      0.8823606967926025,
+      0.812971293926239,
+      0.9398530721664429,
+      0.9983369708061218,
+      0.7522658109664917,
+      0.8004196286201477,
+      0.8944302797317505,
+      0.7996703386306763,
+      0.7197930812835693,
+      0.6556458473205566,
+      0.83084636926651,
+      0.6907240152359009,
+      0.6829873919487,
+      0.7527651786804199,
+      0.6708750128746033,
+      0.6388502717018127,
+      0.6370804905891418,
+      0.5711989402770996,
+      0.7286821007728577,
+      0.7343599200248718,
+      0.6067296862602234,
+      0.6763518452644348,
+      0.6892373561859131,
+      0.5217660665512085,
+      0.5720303058624268,
+      0.5303889513015747,
+      0.611503541469574,
+      0.521020233631134,
+      0.5607292652130127,
+      0.5533806085586548,
+      0.6655105352401733,
+      0.68523108959198,
+      0.5999661684036255,
+      0.5807480216026306,
+      0.5076874494552612,
+      0.5856181383132935,
+      0.5579119920730591,
+      0.5611492991447449,
+      0.39863288402557373,
+      0.5875321626663208,
+      0.5806396007537842,
+      0.5388836860656738,
+      0.5461868047714233,
+      0.5468869209289551,
+      0.5718828439712524,
+      0.5671204328536987,
+      0.5166946649551392,
+      0.5039520263671875,
+      0.512236475944519,
+      0.5827852487564087,
+      0.5096437335014343,
+      0.5190230011940002,
+      0.46621373295783997,
+      0.46831005811691284,
+      0.48136916756629944,
+      0.41866859793663025,
+      0.5789468288421631,
+      0.4082232713699341,
+      0.4730951488018036,
+      0.5560440421104431,
+      0.4662611484527588,
+      0.458769291639328,
+      0.4554605484008789,
+      0.35419344902038574,
+      0.4953368902206421,
+      0.2719497084617615,
+      0.39754337072372437,
+      0.3945605754852295,
+      0.36380279064178467,
+      0.4214194118976593,
+      0.33122456073760986,
+      0.2835928797721863,
+      0.4400366246700287,
+      0.3865411579608917,
+      0.39365729689598083,
+      0.3258381485939026,
+      0.4171220064163208,
+      0.3371686041355133,
+      0.34410810470581055,
+      0.38451898097991943,
+      0.3573110103607178,
+      0.39712274074554443,
+      0.3234259784221649,
+      0.37105095386505127,
+      0.3326011896133423,
+      0.34246066212654114,
+      0.3365008533000946,
+      0.3779088258743286,
+      0.34640663862228394,
+      0.24549447000026703,
+      0.27276891469955444,
+      0.3374707102775574,
+      0.2775835394859314,
+      0.20905432105064392,
+      0.30128228664398193,
+      0.24552085995674133,
+      0.3086501359939575,
+      0.2579006850719452,
+      0.26246193051338196,
+      0.23232240974903107,
+      0.2504483461380005,
+      0.2467145323753357,
+      0.2499353289604187,
+      0.3089613914489746,
+      0.32342568039894104,
+      0.29481178522109985,
+      0.34668678045272827,
+      0.2800863981246948,
+      0.21005690097808838,
+      0.29504430294036865,
+      0.3435496389865875,
+      0.2598270773887634,
+      0.3413766622543335,
+      0.27513062953948975,
+      0.2487499713897705,
+      0.28617170453071594,
+      0.33789175748825073,
+      0.3484917879104614,
+      0.294048547744751,
+      0.19054007530212402,
+      0.20470793545246124,
+      0.2774689495563507,
+      0.19391252100467682,
+      0.23970991373062134,
+      0.22812458872795105,
+      0.3208765387535095,
+      0.22734379768371582,
+      0.21200791001319885,
+      0.17434161901474,
+      0.2524522542953491,
+      0.2736799716949463,
+      0.3468610942363739,
+      0.2304113209247589,
+      0.2211047112941742,
+      0.2680600881576538,
+      0.2494480311870575,
+      0.1454586684703827,
+      0.16198983788490295,
+      0.23723238706588745,
+      0.22418081760406494,
+      0.19769242405891418,
+      0.2208307981491089,
+      0.20500163733959198,
+      0.22917988896369934,
+      0.1742870807647705,
+      0.17347998917102814,
+      0.15921762585639954,
+      0.18850788474082947,
+      0.21649175882339478,
+      0.1518469750881195,
+      0.18442171812057495,
+      0.18402068316936493,
+      0.15331679582595825,
+      0.1962621510028839,
+      0.1557868868112564,
+      0.21655847132205963,
+      0.14196164906024933,
+      0.16651952266693115,
+      0.16552630066871643,
+      0.1699790358543396,
+      0.17379239201545715,
+      0.20463721454143524,
+      0.16547945141792297,
+      0.17575129866600037,
+      0.1955820918083191,
+      0.15568506717681885,
+      0.1336551457643509,
+      0.15189820528030396,
+      0.20265045762062073,
+      0.12927722930908203,
+      0.17286217212677002,
+      0.2051667869091034,
+      0.14176535606384277,
+      0.19065766036510468,
+      0.1639125645160675,
+      0.20288512110710144,
+      0.14706236124038696,
+      0.12397182732820511,
+      0.13666926324367523,
+      0.1708441972732544,
+      0.09048313647508621,
+      0.16668900847434998,
+      0.13475078344345093,
+      0.17308390140533447,
+      0.1450437605381012,
+      0.17835193872451782,
+      0.1770174503326416,
+      0.12992995977401733,
+      0.16309191286563873,
+      0.2013746201992035,
+      0.1292719841003418,
+      0.1814294308423996,
+      0.15766873955726624,
+      0.15577255189418793,
+      0.14916115999221802,
+      0.1704961508512497,
+      0.11911983788013458,
+      0.10947217047214508,
+      0.11832094192504883,
+      0.16997061669826508,
+      0.12023496627807617,
+      0.12366921454668045,
+      0.14696502685546875,
+      0.21081238985061646,
+      0.11183337867259979,
+      0.13474304974079132,
+      0.12028975784778595,
+      0.13747942447662354,
+      0.11091765016317368,
+      0.13583603501319885,
+      0.06030312180519104,
+      0.09970827400684357,
+      0.12357888370752335,
+      0.0896293967962265,
+      0.1271551549434662,
+      0.12046898901462555,
+      0.15673765540122986,
+      0.12015324831008911,
+      0.12207504361867905,
+      0.16421657800674438,
+      0.09508495032787323,
+      0.1441260576248169,
+      0.13152356445789337,
+      0.13145820796489716,
+      0.14598149061203003,
+      0.12673842906951904,
+      0.10492148250341415,
+      0.10826437175273895,
+      0.12614858150482178,
+      0.10947737097740173,
+      0.10662246495485306,
+      0.09764628112316132,
+      0.11726167798042297,
+      0.11433490365743637,
+      0.10214796662330627,
+      0.08775028586387634,
+      0.06946165859699249,
+      0.09933647513389587,
+      0.13902157545089722,
+      0.09748313575983047,
+      0.1180696189403534,
+      0.09541209787130356,
+      0.12408377975225449,
+      0.11557712405920029,
+      0.1257520616054535,
+      0.08340153098106384,
+      0.09140900522470474,
+      0.09548808634281158,
+      0.10011038929224014,
+      0.10398255288600922,
+      0.09144933521747589,
+      0.10531884431838989,
+      0.08981457352638245,
+      0.09359326958656311,
+      0.12292107194662094,
+      0.09324193745851517,
+      0.13378912210464478,
+      0.11922477185726166,
+      0.10930902510881424,
+      0.08063478767871857,
+      0.08351168036460876,
+      0.08308646082878113,
+      0.0905536562204361,
+      0.09639933705329895,
+      0.09178976714611053,
+      0.08503931760787964,
+      0.09252388030290604,
+      0.11107531189918518,
+      0.0956282764673233,
+      0.11496362835168839,
+      0.1259007602930069,
+      0.08518417924642563,
+      0.13414232432842255,
+      0.10084954649209976
+    ],
+    "zipf_loss": [
+      6.812561988830566,
+      5.833886623382568,
+      4.236737251281738,
+      2.906264543533325,
+      1.9535157680511475,
+      1.2167904376983643,
+      0.9806873202323914,
+      0.8986221551895142,
+      0.8655501008033752,
+      0.8595107793807983,
+      0.8515556454658508,
+      0.8501031994819641,
+      0.8473920226097107,
+      0.8476628065109253,
+      0.8468807935714722,
+      0.8478845357894897,
+      0.8463108539581299,
+      0.8399034142494202,
+      0.8167651295661926,
+      0.7714415788650513,
+      0.6197283267974854,
+      0.40534600615501404,
+      0.2764799892902374,
+      0.20878319442272186,
+      0.17731904983520508,
+      0.16075995564460754,
+      0.14626535773277283,
+      0.1460246592760086,
+      0.14082393050193787,
+      0.13221590220928192,
+      0.12891355156898499,
+      0.12285328656435013,
+      0.12454423308372498,
+      0.12241454422473907,
+      0.11583086848258972,
+      0.11039087176322937,
+      0.10492554306983948,
+      0.10886098444461823,
+      0.10698096454143524,
+      0.09907151758670807,
+      0.09598459303379059,
+      0.09337620437145233,
+      0.07560954988002777,
+      0.07782987505197525,
+      0.08794909715652466,
+      0.08203543722629547,
+      0.06925708055496216,
+      0.0746869444847107,
+      0.07607772946357727,
+      0.06208964064717293,
+      0.056445829570293427,
+      0.052963413298130035,
+      0.052643515169620514,
+      0.05349712073802948,
+      0.04476241022348404,
+      0.04337061941623688,
+      0.04420023784041405,
+      0.04475340619683266,
+      0.040587861090898514,
+      0.05581096559762955,
+      0.045061901211738586,
+      0.04957014322280884,
+      0.03197324648499489,
+      0.0331258550286293,
+      0.03356559947133064,
+      0.049435731023550034,
+      0.03089987300336361,
+      0.031197452917695045,
+      0.02698611281812191,
+      0.03906417265534401,
+      0.03657431900501251,
+      0.040570054203271866,
+      0.04655037075281143,
+      0.027130626142024994,
+      0.052653223276138306,
+      0.031796298921108246,
+      0.02639320306479931,
+      0.027119144797325134,
+      0.02939813770353794,
+      0.02901133894920349,
+      0.02712780050933361,
+      0.02383788675069809,
+      0.02851581573486328,
+      0.032333821058273315,
+      0.027064111083745956,
+      0.022199369966983795,
+      0.020015262067317963,
+      0.033772777765989304,
+      0.0460648313164711,
+      0.03467245027422905,
+      0.030091946944594383,
+      0.025444069877266884,
+      0.02980213798582554,
+      0.030813321471214294,
+      0.03147619962692261,
+      0.02834276854991913,
+      0.05009395256638527,
+      0.03247137740254402,
+      0.024833565577864647,
+      0.0361790731549263,
+      0.025828974321484566,
+      0.04596130922436714,
+      0.038424380123615265,
+      0.03605397790670395,
+      0.036298178136348724,
+      0.020230259746313095,
+      0.03624265640974045,
+      0.030190177261829376,
+      0.027806095778942108,
+      0.02708495408296585,
+      0.021598979830741882,
+      0.029994435608386993,
+      0.027213560417294502,
+      0.025263624265789986,
+      0.03119480237364769,
+      0.03556299954652786,
+      0.03350179269909859,
+      0.018048599362373352,
+      0.02238570898771286,
+      0.03730948641896248,
+      0.03435484319925308,
+      0.023310724645853043,
+      0.02590544894337654,
+      0.016905849799513817,
+      0.01949998363852501,
+      0.02812476083636284,
+      0.03266098350286484,
+      0.03806779906153679,
+      0.04386970400810242,
+      0.04086627811193466,
+      0.0366770401597023,
+      0.03866710513830185,
+      0.02323247492313385,
+      0.04714139550924301,
+      0.01724546030163765,
+      0.035882800817489624,
+      0.03410448879003525,
+      0.029439300298690796,
+      0.02041974849998951,
+      0.01920677162706852,
+      0.022971099242568016,
+      0.028331564739346504,
+      0.03339792788028717,
+      0.028236214071512222,
+      0.03681683540344238,
+      0.02050747722387314,
+      0.03995659202337265,
+      0.027925927191972733,
+      0.04064967483282089,
+      0.028268780559301376,
+      0.035769566893577576,
+      0.01658254861831665,
+      0.019398927688598633,
+      0.01920575648546219,
+      0.0188615582883358,
+      0.03282739967107773,
+      0.029027318581938744,
+      0.023225843906402588,
+      0.03229328617453575,
+      0.034405145794153214,
+      0.02456742897629738,
+      0.0246428232640028,
+      0.033966295421123505,
+      0.025529980659484863,
+      0.046644628047943115,
+      0.010466357693076134,
+      0.02626398205757141,
+      0.031327031552791595,
+      0.03541884943842888,
+      0.0332460030913353,
+      0.028602520003914833,
+      0.029449403285980225,
+      0.04255424439907074,
+      0.027092909440398216,
+      0.027289755642414093,
+      0.022487711161375046,
+      0.03418818116188049,
+      0.0429987795650959,
+      0.028919201344251633,
+      0.022098271176218987,
+      0.04999305680394173,
+      0.037317924201488495,
+      0.028346505016088486,
+      0.0272943377494812,
+      0.028195345774292946,
+      0.042196765542030334,
+      0.03280748426914215,
+      0.019388850778341293,
+      0.02610059455037117,
+      0.02645121142268181,
+      0.029644761234521866,
+      0.04554266110062599,
+      0.017258264124393463,
+      0.03967483341693878,
+      0.039851561188697815,
+      0.023286346346139908,
+      0.03580312803387642,
+      0.034220561385154724,
+      0.021779578179121017,
+      0.03496241196990013,
+      0.026859907433390617,
+      0.030208542943000793,
+      0.02931339666247368,
+      0.0499223992228508,
+      0.03624880686402321,
+      0.044473424553871155,
+      0.06037009879946709,
+      0.028237389400601387,
+      0.02347707562148571,
+      0.06018330156803131,
+      0.02738623507320881,
+      0.025182217359542847,
+      0.03649861738085747,
+      0.030256854370236397,
+      0.027048002928495407,
+      0.03649982810020447,
+      0.025137431919574738,
+      0.0742945745587349,
+      0.029336459934711456,
+      0.0302302036434412,
+      0.05663887783885002,
+      0.041515450924634933,
+      0.02702762745320797,
+      0.039706043899059296,
+      0.022431524470448494,
+      0.031441450119018555,
+      0.021258242428302765,
+      0.0334407240152359,
+      0.022245360538363457,
+      0.028604380786418915,
+      0.029893646016716957,
+      0.03998545557260513,
+      0.03170153498649597,
+      0.033288709819316864,
+      0.03582096844911575,
+      0.026853401213884354,
+      0.030703924596309662,
+      0.03542246297001839,
+      0.027721714228391647,
+      0.022292520850896835,
+      0.04698874428868294,
+      0.03143282234668732,
+      0.02607708051800728,
+      0.03005710244178772,
+      0.02815207839012146,
+      0.021528279408812523,
+      0.035634540021419525,
+      0.03208673745393753,
+      0.030409611761569977,
+      0.023158103227615356,
+      0.03353625908493996,
+      0.03725823760032654,
+      0.0512077771127224,
+      0.036544814705848694,
+      0.037965454161167145,
+      0.027400074526667595,
+      0.04517415910959244,
+      0.030967533588409424,
+      0.022170791402459145,
+      0.03707485273480415,
+      0.0305289588868618,
+      0.04439655691385269,
+      0.033418457955121994,
+      0.030159130692481995,
+      0.03126520290970802,
+      0.036753639578819275,
+      0.030856451019644737,
+      0.025071844458580017,
+      0.05589497834444046,
+      0.02841416746377945,
+      0.04395944997668266,
+      0.03561931848526001,
+      0.03778424113988876,
+      0.025084055960178375,
+      0.027444057166576385,
+      0.029216008260846138,
+      0.03622381016612053,
+      0.015050912275910378,
+      0.0482652485370636,
+      0.034516554325819016,
+      0.04029349237680435,
+      0.03202264383435249,
+      0.029037408530712128,
+      0.025578860193490982,
+      0.03907560929656029,
+      0.028551045805215836,
+      0.03448163717985153,
+      0.03297030180692673,
+      0.02233646810054779,
+      0.036183737218379974,
+      0.02131832018494606,
+      0.029427960515022278,
+      0.02391825243830681,
+      0.038564980030059814,
+      0.025131095200777054,
+      0.032127249985933304,
+      0.031822703778743744,
+      0.018397251144051552,
+      0.02475113421678543,
+      0.03608299791812897,
+      0.0335233137011528,
+      0.01834193244576454,
+      0.02559027075767517,
+      0.047609567642211914,
+      0.028362805023789406,
+      0.023916248232126236,
+      0.02620236761868,
+      0.02720978669822216,
+      0.035341836512088776,
+      0.016808412969112396
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.827428936958313,
+      0.671424925327301,
+      0.4355371594429016,
+      0.2761383056640625,
+      0.19764411449432373,
+      0.1639745831489563,
+      0.13044804334640503,
+      0.09763513505458832,
+      0.08025705814361572,
+      0.06304512917995453,
+      0.052203983068466187,
+      0.04871360585093498,
+      0.047047827392816544,
+      0.045929357409477234,
+      0.046626560389995575,
+      0.051150523126125336,
+      0.04976411908864975,
+      0.04797111451625824,
+      0.04966820031404495,
+      0.0512523427605629,
+      0.05399706959724426,
+      0.06475960463285446,
+      0.0754888579249382,
+      0.08560491353273392,
+      0.0921943187713623,
+      0.09945797175168991,
+      0.10485729575157166,
+      0.1072235181927681,
+      0.11412574350833893,
+      0.11918080598115921,
+      0.12542453408241272,
+      0.13089101016521454,
+      0.13531622290611267,
+      0.13845030963420868,
+      0.142812579870224,
+      0.1476566642522812,
+      0.15259936451911926,
+      0.1554049253463745,
+      0.16178391873836517,
+      0.16461075842380524,
+      0.1640101671218872,
+      0.16973431408405304,
+      0.17018891870975494,
+      0.1709456443786621,
+      0.1720433235168457,
+      0.17485295236110687,
+      0.179266557097435,
+      0.18221420049667358,
+      0.18234266340732574,
+      0.18083101511001587,
+      0.18191446363925934,
+      0.1810181885957718,
+      0.18036189675331116,
+      0.1824311763048172,
+      0.1812783032655716,
+      0.1808960884809494,
+      0.18026043474674225,
+      0.17999278008937836,
+      0.18125073611736298,
+      0.1799735575914383,
+      0.18007639050483704,
+      0.1817018836736679,
+      0.17534105479717255,
+      0.17151685059070587,
+      0.17263628542423248,
+      0.17425446212291718,
+      0.17311222851276398,
+      0.17309057712554932,
+      0.17657621204853058,
+      0.18002821505069733,
+      0.1744808852672577,
+      0.17262759804725647,
+      0.17171679437160492,
+      0.17160746455192566,
+      0.1765274852514267,
+      0.1734113246202469,
+      0.17856542766094208,
+      0.17718186974525452,
+      0.18131829798221588,
+      0.1788182407617569,
+      0.17792543768882751,
+      0.1823016107082367,
+      0.18500618636608124,
+      0.18381965160369873,
+      0.19101843237876892,
+      0.18853366374969482,
+      0.18821357190608978,
+      0.1886235922574997,
+      0.18627484142780304,
+      0.17950628697872162,
+      0.18204718828201294,
+      0.1802850067615509,
+      0.1805403083562851,
+      0.18575280904769897,
+      0.18664105236530304,
+      0.19113248586654663,
+      0.19141356647014618,
+      0.19219623506069183,
+      0.19271692633628845,
+      0.19056299328804016,
+      0.19620807468891144,
+      0.19899681210517883,
+      0.20227837562561035,
+      0.2044687271118164,
+      0.2060144543647766,
+      0.20563283562660217,
+      0.20243804156780243,
+      0.19240102171897888,
+      0.19863666594028473,
+      0.2030983716249466,
+      0.2072049379348755,
+      0.20826730132102966,
+      0.2071978747844696,
+      0.20865215361118317,
+      0.20026527345180511,
+      0.20367886126041412,
+      0.20545694231987,
+      0.20512835681438446,
+      0.20140065252780914,
+      0.20723676681518555,
+      0.20945851504802704,
+      0.21112161874771118,
+      0.21480081975460052,
+      0.21333807706832886,
+      0.2173749953508377,
+      0.21565650403499603,
+      0.21041904389858246,
+      0.21454955637454987,
+      0.21502478420734406,
+      0.2122405767440796,
+      0.21353724598884583,
+      0.2195930927991867,
+      0.22009789943695068,
+      0.2219575196504593,
+      0.2279929369688034,
+      0.22722227871418,
+      0.23112311959266663,
+      0.23142953217029572,
+      0.2327469438314438,
+      0.2316807061433792,
+      0.2384343296289444,
+      0.2477462887763977,
+      0.2541239857673645,
+      0.2609383165836334,
+      0.26400136947631836,
+      0.27505865693092346,
+      0.2738339900970459,
+      0.2789699137210846,
+      0.2832750678062439,
+      0.28833818435668945,
+      0.29126831889152527,
+      0.29266005754470825,
+      0.29452967643737793,
+      0.2978121042251587,
+      0.2987567186355591,
+      0.30246785283088684,
+      0.307929664850235,
+      0.3047032952308655,
+      0.30671849846839905,
+      0.30758509039878845,
+      0.30814602971076965,
+      0.31224527955055237,
+      0.3103698790073395,
+      0.31271615624427795,
+      0.31407102942466736,
+      0.3142327666282654,
+      0.31299009919166565,
+      0.3166446387767792,
+      0.3147873878479004,
+      0.31458809971809387,
+      0.31245267391204834,
+      0.31644007563591003,
+      0.3156980276107788,
+      0.3200494050979614,
+      0.32049500942230225,
+      0.3200150430202484,
+      0.3233881890773773,
+      0.3243626356124878,
+      0.32112860679626465,
+      0.3201987147331238,
+      0.3216695487499237,
+      0.32685965299606323,
+      0.32346639037132263,
+      0.32615596055984497,
+      0.32929402589797974,
+      0.330515056848526,
+      0.3298525810241699,
+      0.3309860825538635,
+      0.33082884550094604,
+      0.3283136785030365,
+      0.32374125719070435,
+      0.3230162262916565,
+      0.32241517305374146,
+      0.3309914767742157,
+      0.32828187942504883,
+      0.3306802809238434,
+      0.3282453417778015,
+      0.330154150724411,
+      0.33348920941352844,
+      0.33330708742141724,
+      0.3393866717815399,
+      0.3331195116043091,
+      0.3384978175163269,
+      0.33747807145118713,
+      0.33147355914115906,
+      0.33095183968544006,
+      0.33170023560523987,
+      0.33314263820648193,
+      0.33552658557891846,
+      0.335514098405838,
+      0.3355313241481781,
+      0.3361027240753174,
+      0.33816108107566833,
+      0.33751001954078674,
+      0.333874374628067,
+      0.3335190415382385,
+      0.3304026126861572,
+      0.33213523030281067,
+      0.33157700300216675,
+      0.33280616998672485,
+      0.33208560943603516,
+      0.32953083515167236,
+      0.3310057520866394,
+      0.33369043469429016,
+      0.33322376012802124,
+      0.33351776003837585,
+      0.33360275626182556,
+      0.3334753215312958,
+      0.33686330914497375,
+      0.3349614143371582,
+      0.3361343443393707,
+      0.3360339403152466,
+      0.33757269382476807,
+      0.33795565366744995,
+      0.33851340413093567,
+      0.3372805416584015,
+      0.34057605266571045,
+      0.3405458629131317,
+      0.34144896268844604,
+      0.34201014041900635,
+      0.3403547704219818,
+      0.3434809744358063,
+      0.3442747890949249,
+      0.3466796576976776,
+      0.3471396565437317,
+      0.3460945785045624,
+      0.34654906392097473,
+      0.34674072265625,
+      0.3492203652858734,
+      0.348775714635849,
+      0.3491442799568176,
+      0.3496403396129608,
+      0.3488059639930725,
+      0.3496220111846924,
+      0.3533368408679962,
+      0.3526526987552643,
+      0.3511108458042145,
+      0.35070398449897766,
+      0.35075801610946655,
+      0.3511902689933777,
+      0.35225415229797363,
+      0.3524736166000366,
+      0.3527209162712097,
+      0.3549518585205078,
+      0.35588890314102173,
+      0.35622313618659973,
+      0.3560788929462433,
+      0.35539689660072327,
+      0.35661858320236206,
+      0.3575516939163208,
+      0.3581475019454956,
+      0.3578368127346039,
+      0.35904768109321594,
+      0.3596513867378235,
+      0.36070311069488525,
+      0.36103060841560364,
+      0.36321330070495605,
+      0.363923579454422,
+      0.3643174171447754,
+      0.36385178565979004,
+      0.3644620478153229,
+      0.3653315305709839,
+      0.3654240369796753,
+      0.3666748106479645,
+      0.36774975061416626,
+      0.3677917420864105,
+      0.367666631937027,
+      0.36842453479766846,
+      0.3692092001438141,
+      0.3693569004535675,
+      0.36985817551612854,
+      0.3703119158744812,
+      0.3706112205982208,
+      0.370940238237381,
+      0.37099459767341614,
+      0.3713039457798004,
+      0.3717826008796692,
+      0.37260204553604126,
+      0.3729584813117981,
+      0.37327811121940613,
+      0.37369832396507263,
+      0.37419503927230835,
+      0.3748515248298645,
+      0.37514984607696533,
+      0.37567055225372314,
+      0.3761422038078308,
+      0.3766128718852997,
+      0.37726834416389465,
+      0.37740060687065125,
+      0.37733811140060425
+    ],
+    "lr": [
+      4.188034188034189e-06,
+      8.461538461538462e-06,
+      1.2735042735042738e-05,
+      1.700854700854701e-05,
+      2.1282051282051282e-05,
+      2.5555555555555557e-05,
+      2.9829059829059833e-05,
+      3.4102564102564105e-05,
+      3.837606837606838e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.992995647012267e-05,
+      3.963316185199842e-05,
+      3.933636723387416e-05,
+      3.90395726157499e-05,
+      3.874277799762565e-05,
+      3.844598337950139e-05,
+      3.814918876137713e-05,
+      3.785239414325287e-05,
+      3.7555599525128613e-05,
+      3.725880490700435e-05,
+      3.696201028888009e-05,
+      3.666521567075585e-05,
+      3.6368421052631585e-05,
+      3.607162643450732e-05,
+      3.577483181638307e-05,
+      3.547803719825881e-05,
+      3.518124258013455e-05,
+      3.4884447962010287e-05,
+      3.458765334388604e-05,
+      3.4290858725761776e-05,
+      3.399406410763751e-05,
+      3.3697269489513265e-05,
+      3.3400474871389e-05,
+      3.310368025326474e-05,
+      3.280688563514048e-05,
+      3.251009101701623e-05,
+      3.2213296398891967e-05,
+      3.191650178076771e-05,
+      3.1542540561931145e-05,
+      3.124574594380688e-05,
+      3.094895132568263e-05,
+      3.065215670755838e-05,
+      3.0355362089434113e-05,
+      3.0058567471309854e-05,
+      2.97617728531856e-05,
+      2.9464978235061336e-05,
+      2.9168183616937077e-05,
+      2.8871388998812818e-05,
+      2.857459438068857e-05,
+      2.8277799762564307e-05,
+      2.798100514444005e-05,
+      2.7684210526315793e-05,
+      2.738741590819153e-05,
+      2.709062129006727e-05,
+      2.6793826671943013e-05,
+      2.6497032053818764e-05,
+      2.62002374356945e-05,
+      2.5903442817570243e-05,
+      2.5606648199445987e-05,
+      2.5309853581321725e-05,
+      2.5013058963197466e-05,
+      2.4716264345073214e-05,
+      2.441946972694896e-05,
+      2.4122675108824693e-05,
+      2.3825880490700434e-05,
+      2.352908587257618e-05,
+      2.3232291254451916e-05,
+      2.293549663632766e-05,
+      2.2638702018203408e-05,
+      2.226474079936684e-05,
+      2.196794618124258e-05,
+      2.1671151563118328e-05,
+      2.1374356944994066e-05,
+      2.1077562326869803e-05,
+      2.0780767708745554e-05,
+      2.0483973090621296e-05,
+      2.018717847249703e-05,
+      1.989038385437277e-05,
+      1.959358923624852e-05,
+      1.929679461812426e-05,
+      1.8999999999999998e-05,
+      1.8703205381875746e-05,
+      1.8406410763751487e-05,
+      1.8109616145627224e-05,
+      1.7812821527502965e-05,
+      1.7516026909378713e-05,
+      1.7219232291254454e-05,
+      1.6922437673130192e-05,
+      1.662564305500594e-05,
+      1.632884843688168e-05,
+      1.603205381875742e-05,
+      1.573525920063316e-05,
+      1.5438464582508908e-05,
+      1.5141669964384647e-05,
+      1.4844875346260385e-05,
+      1.4548080728136134e-05,
+      1.4251286110011875e-05,
+      1.3954491491887611e-05,
+      1.3657696873763363e-05,
+      1.33609022556391e-05,
+      1.298694103680253e-05,
+      1.2690146418678277e-05,
+      1.2393351800554018e-05,
+      1.2096557182429756e-05,
+      1.1799762564305506e-05,
+      1.1502967946181245e-05,
+      1.1206173328056984e-05,
+      1.0909378709932724e-05,
+      1.0612584091808472e-05,
+      1.0315789473684213e-05,
+      1.0018994855559949e-05,
+      9.7222002374357e-06,
+      9.425405619311438e-06,
+      9.128611001187177e-06,
+      8.831816383062916e-06,
+      8.535021764938666e-06,
+      8.238227146814405e-06,
+      7.941432528690143e-06,
+      7.644637910565893e-06,
+      7.347843292441632e-06,
+      7.0510486743173715e-06,
+      6.75425405619311e-06,
+      6.45745943806886e-06,
+      6.160664819944598e-06,
+      5.8638702018203375e-06,
+      5.567075583696086e-06,
+      5.270280965571826e-06,
+      4.973486347447565e-06,
+      4.676691729323314e-06,
+      4.379897111199053e-06,
+      4.083102493074792e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      1550,
+      3113,
+      4676,
+      6239,
+      7802,
+      9365,
+      10928,
+      12491,
+      14054,
+      15617
+    ],
+    "eval_accuracy": [
+      0.26,
+      0.83,
+      0.94,
+      0.95,
+      0.95,
+      0.95,
+      0.92,
+      0.98,
+      1.0,
+      1.0
+    ]
+  },
+  "final_accuracy": 0.96125,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.73,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9570247933884297,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9789473684210527,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.77,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.946078431372549,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9822485207100592,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.967741935483871,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9662162162162162,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.49,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9631901840490797,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9384615384615385,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9080459770114943,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.8078817733990148,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9829059829059829,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9834710743801653,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9834710743801653,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.9591836734693877,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7473118279569892,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.8026905829596412,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.37,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9903846153846154,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.9622641509433962,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.7,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.6579804560260586,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.36,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.545,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.54,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.65,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.67,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.79,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9821029082774049,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.975,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9527410207939508,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.8958333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.59,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9866666666666667,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8134715025906736,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.7757009345794392,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.45,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.83203125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.7708333333333334,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.48,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8464052287581699,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.8144329896907216,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8415300546448088,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.8632478632478633,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.83,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9733777038269551,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.9595959595959596,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.86,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.982078853046595,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.9862068965517241,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.9682539682539683,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.44,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9812206572769953,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9734513274336283,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9882352941176471,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.6906077348066298,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.9814814814814815,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.15,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9888268156424581,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.970873786407767,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5906040268456376,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.6572769953051644,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.15,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.54,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.45666666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.11,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.44,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.39,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.74,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9866666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9700374531835206,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9179954441913439,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.926829268292683,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.51,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9866666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7715736040609137,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.8155339805825242,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.34,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7854251012145749,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.7450980392156863,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.31,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7684563758389261,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.7772277227722773,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.515,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 4,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.95625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.9869706840390879,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.7,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.72,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.9625,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9948186528497409,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.99609375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9901960784313726,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.9948453608247423,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9972677595628415,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.9932885906040269,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.92,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.9933333333333333,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.67,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.67,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.9775,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.995,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9983333333333333,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.95,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9746192893401016,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9959514170040485,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.96125,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.96125,
+  "sft_overall_accuracy": 0.515
+}

add_sub_sorl_v1_abs30_100K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7332e8a91d9f010ea6aaaa8db3a4f5b2d0da3000748bf222741ae40b08e20106
+size 650385300

add_sub_sorl_v1_abs30_100K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 1562,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K4_100K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 100000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162519662,
+  "run_name": "add_sub_sorl_v1_abs30_100K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T12:09:53.734351+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "iffr93ko",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/iffr93ko",
+  "final_accuracy": 0.96125,
+  "sft_accuracy": 0.515,
+  "eval_method": "ArithmeticEvaluator"
+}