Upload add_sub_sorl_v1_abs30_K1_50K

add_sub_sorl_v1_abs30_K1_50K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_K1_50K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_K1_50K/metrics.json

@@ -0,0 +1,2257 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      832,
+      882,
+      932,
+      982,
+      1032,
+      1082,
+      1132,
+      1182,
+      1232,
+      1282,
+      1332,
+      1382,
+      1432,
+      1482,
+      1532,
+      1614,
+      1664,
+      1714,
+      1764,
+      1814,
+      1864,
+      1914,
+      1964,
+      2014,
+      2064,
+      2114,
+      2164,
+      2214,
+      2264,
+      2314,
+      2396,
+      2446,
+      2496,
+      2546,
+      2596,
+      2646,
+      2696,
+      2746,
+      2796,
+      2846,
+      2896,
+      2946,
+      2996,
+      3046,
+      3096,
+      3178,
+      3228,
+      3278,
+      3328,
+      3378,
+      3428,
+      3478,
+      3528,
+      3578,
+      3628,
+      3678,
+      3728,
+      3778,
+      3828,
+      3878,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4310,
+      4360,
+      4410,
+      4460,
+      4510,
+      4560,
+      4610,
+      4660,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5092,
+      5142,
+      5192,
+      5242,
+      5292,
+      5342,
+      5392,
+      5442,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5874,
+      5924,
+      5974,
+      6024,
+      6074,
+      6124,
+      6174,
+      6224,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6656,
+      6706,
+      6756,
+      6806,
+      6856,
+      6906,
+      6956,
+      7006,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7438,
+      7488,
+      7538,
+      7588,
+      7638,
+      7688,
+      7738,
+      7788
+    ],
+    "loss": [
+      -2.677070140838623,
+      4.240540504455566,
+      5.680160045623779,
+      4.387296676635742,
+      2.8017568588256836,
+      2.588024854660034,
+      2.506669044494629,
+      2.2819900512695312,
+      1.9044015407562256,
+      0.8909429907798767,
+      -0.0643576979637146,
+      -2.625436305999756,
+      -5.162726402282715,
+      -6.5062642097473145,
+      -4.659561634063721,
+      -1.8159736394882202,
+      -1.295497179031372,
+      -1.1573535203933716,
+      -1.6328198909759521,
+      -0.4736297130584717,
+      -1.0086276531219482,
+      -1.7575881481170654,
+      -1.5724313259124756,
+      -1.3880059719085693,
+      -1.958205223083496,
+      -1.473975419998169,
+      -1.1694231033325195,
+      -0.7242996096611023,
+      -0.9551407098770142,
+      -1.536046028137207,
+      -1.4602121114730835,
+      -1.0210899114608765,
+      -1.154510736465454,
+      -1.4573113918304443,
+      -0.8744689226150513,
+      -1.221395492553711,
+      -0.7826364040374756,
+      -0.9359591007232666,
+      -1.1627928018569946,
+      -0.7902859449386597,
+      -0.8775216341018677,
+      -0.50701504945755,
+      -0.35609766840934753,
+      -0.9194822311401367,
+      -0.7816537618637085,
+      -0.6689697504043579,
+      -0.5528387427330017,
+      -0.6544617414474487,
+      -0.9605201482772827,
+      -0.9643555879592896,
+      -0.6821292042732239,
+      -0.887940526008606,
+      -0.7036629319190979,
+      -0.3710128664970398,
+      -0.39701297879219055,
+      -0.5807578563690186,
+      -0.6634552478790283,
+      -0.7409839034080505,
+      -0.6898993253707886,
+      -0.506354570388794,
+      -0.6955422759056091,
+      -0.9323541522026062,
+      -0.43204134702682495,
+      -0.44987472891807556,
+      -0.676785945892334,
+      -0.6465747356414795,
+      -0.6067618131637573,
+      -0.49532854557037354,
+      -0.6834741830825806,
+      -0.732882022857666,
+      -0.40189582109451294,
+      -0.7664804458618164,
+      -0.7703322172164917,
+      -0.561592161655426,
+      -0.4559732675552368,
+      -0.992522120475769,
+      -0.48274296522140503,
+      -0.5376996397972107,
+      -0.44557809829711914,
+      -0.490071177482605,
+      -0.09410737454891205,
+      -0.28825902938842773,
+      -0.6362848877906799,
+      -0.3807905912399292,
+      -0.6656315922737122,
+      -0.32264500856399536,
+      -0.2783470153808594,
+      -0.4409356415271759,
+      -0.33182066679000854,
+      -0.3980458378791809,
+      -0.5530517101287842,
+      -0.34538453817367554,
+      -0.8654745817184448,
+      -0.5388880372047424,
+      -0.5147119760513306,
+      -0.6792829632759094,
+      -0.48428094387054443,
+      -0.14687921106815338,
+      -0.2964346408843994,
+      -0.4467563033103943,
+      -0.10742655396461487,
+      -0.348341166973114,
+      -0.2993023097515106,
+      -0.31591737270355225,
+      -0.5104386806488037,
+      -0.20582923293113708,
+      -0.11799026280641556,
+      -0.42429250478744507,
+      -0.49120867252349854,
+      -0.41805562376976013,
+      -0.22228224575519562,
+      -0.8084666132926941,
+      -0.2575153708457947,
+      -0.29879045486450195,
+      -0.15641680359840393,
+      -0.5069034695625305,
+      -0.31737959384918213,
+      -0.373086541891098,
+      -0.3146061301231384,
+      -0.18267224729061127,
+      -0.2510879337787628,
+      -0.19426313042640686,
+      -0.2531294524669647,
+      -0.16185298562049866,
+      -0.35831528902053833,
+      -0.22115467488765717,
+      -0.39162585139274597,
+      -0.3057347536087036,
+      -0.3953756093978882,
+      -0.4538670778274536,
+      -0.2203264832496643,
+      -0.046577759087085724,
+      -0.19452610611915588,
+      -0.09236270189285278,
+      -0.2539334297180176,
+      -0.2425602376461029,
+      -0.19475547969341278,
+      0.1786043345928192,
+      -0.2585909962654114,
+      -0.5148232579231262,
+      -0.3099942207336426,
+      -0.05985572189092636,
+      -0.06023649871349335,
+      -0.29088082909584045,
+      -0.08856651186943054,
+      -0.17555394768714905,
+      -0.06671421229839325,
+      -0.23185035586357117,
+      -0.0725683867931366,
+      -0.13712765276432037
+    ],
+    "base_loss": [
+      10.63121509552002,
+      7.855044841766357,
+      6.48610258102417,
+      4.184937477111816,
+      2.2254128456115723,
+      1.969483733177185,
+      1.8631941080093384,
+      1.9075061082839966,
+      1.8428534269332886,
+      1.7919542789459229,
+      1.8120174407958984,
+      1.7657603025436401,
+      1.8202065229415894,
+      1.6680501699447632,
+      1.3603564500808716,
+      0.907917320728302,
+      0.8114389181137085,
+      0.696312427520752,
+      0.7478328943252563,
+      0.5660454034805298,
+      0.5468924641609192,
+      0.5938227772712708,
+      0.5406249165534973,
+      0.5115245580673218,
+      0.5128815770149231,
+      0.4408721327781677,
+      0.38657817244529724,
+      0.37150266766548157,
+      0.32505860924720764,
+      0.3789425194263458,
+      0.35208582878112793,
+      0.25862041115760803,
+      0.24582929909229279,
+      0.29599878191947937,
+      0.24640943109989166,
+      0.22864343225955963,
+      0.211526021361351,
+      0.1996002048254013,
+      0.21388976275920868,
+      0.17000818252563477,
+      0.19602802395820618,
+      0.147139310836792,
+      0.14338411390781403,
+      0.1506159007549286,
+      0.16718284785747528,
+      0.13207398355007172,
+      0.148231640458107,
+      0.1224263533949852,
+      0.19789263606071472,
+      0.1861945539712906,
+      0.1454288512468338,
+      0.15645575523376465,
+      0.12440910190343857,
+      0.080633744597435,
+      0.09144233167171478,
+      0.10515068471431732,
+      0.11176739633083344,
+      0.13789816200733185,
+      0.14126108586788177,
+      0.09895050525665283,
+      0.10370238870382309,
+      0.1346673220396042,
+      0.07871399074792862,
+      0.08171606063842773,
+      0.09952058643102646,
+      0.10415340214967728,
+      0.11704421043395996,
+      0.11788104474544525,
+      0.10579996556043625,
+      0.10230918973684311,
+      0.08041729032993317,
+      0.11346970498561859,
+      0.11239000409841537,
+      0.08241736888885498,
+      0.08472903072834015,
+      0.1403222531080246,
+      0.08328425884246826,
+      0.08383399993181229,
+      0.07330697774887085,
+      0.07271105796098709,
+      0.041518647223711014,
+      0.0715184360742569,
+      0.09238983690738678,
+      0.06007538363337517,
+      0.08854454010725021,
+      0.06593887507915497,
+      0.04848653823137283,
+      0.0664001852273941,
+      0.06380441784858704,
+      0.08914000540971756,
+      0.07944484800100327,
+      0.055715836584568024,
+      0.13285015523433685,
+      0.09760647267103195,
+      0.07531939446926117,
+      0.1048951968550682,
+      0.06989333778619766,
+      0.05205599591135979,
+      0.04335368424654007,
+      0.07759185880422592,
+      0.03298063948750496,
+      0.04927435889840126,
+      0.044151246547698975,
+      0.055020179599523544,
+      0.06686978042125702,
+      0.033203307539224625,
+      0.02507300302386284,
+      0.059389300644397736,
+      0.06475786864757538,
+      0.07413608580827713,
+      0.03313760831952095,
+      0.09840010106563568,
+      0.03672923520207405,
+      0.04463062062859535,
+      0.02445376291871071,
+      0.0668722614645958,
+      0.04513984173536301,
+      0.050169315189123154,
+      0.04182475060224533,
+      0.028356632217764854,
+      0.03394116461277008,
+      0.03791593387722969,
+      0.036652643233537674,
+      0.03326660767197609,
+      0.04727628454566002,
+      0.030323371291160583,
+      0.05311122164130211,
+      0.040131811052560806,
+      0.05187029391527176,
+      0.0579427145421505,
+      0.03126463666558266,
+      0.022122353315353394,
+      0.0320105254650116,
+      0.016670240089297295,
+      0.03367376700043678,
+      0.03911895677447319,
+      0.027257902547717094,
+      0.014670220203697681,
+      0.03239307925105095,
+      0.06242545321583748,
+      0.0643230676651001,
+      0.013553738594055176,
+      0.012213179841637611,
+      0.03658689931035042,
+      0.01566484197974205,
+      0.02520238421857357,
+      0.01604020781815052,
+      0.030884772539138794,
+      0.01297991257160902,
+      0.018583377823233604
+    ],
+    "info_loss": [
+      -2.0281829833984375,
+      -0.8627567291259766,
+      -0.382174015045166,
+      -0.12941598892211914,
+      -0.058706045150756836,
+      -0.051110267639160156,
+      -0.04737722873687744,
+      -0.07399106025695801,
+      -0.10516953468322754,
+      -0.20122039318084717,
+      -0.29862236976623535,
+      -0.5507385730743408,
+      -0.8106637001037598,
+      -0.9297664761543274,
+      -0.71467125415802,
+      -0.384584903717041,
+      -0.32299506664276123,
+      -0.29684585332870483,
+      -0.3491358458995819,
+      -0.2144489884376526,
+      -0.26522332429885864,
+      -0.3436868190765381,
+      -0.3181033432483673,
+      -0.29408982396125793,
+      -0.3449072241783142,
+      -0.2857780158519745,
+      -0.24117933213710785,
+      -0.1892271637916565,
+      -0.20267696678638458,
+      -0.2602337896823883,
+      -0.24082936346530914,
+      -0.18451035022735596,
+      -0.19066095352172852,
+      -0.22124862670898438,
+      -0.15310901403427124,
+      -0.1832810491323471,
+      -0.1350513994693756,
+      -0.14702916145324707,
+      -0.17165809869766235,
+      -0.12784922122955322,
+      -0.1369386464357376,
+      -0.09774664044380188,
+      -0.0794207975268364,
+      -0.1336645632982254,
+      -0.1239786446094513,
+      -0.10766246169805527,
+      -0.09549392759799957,
+      -0.1017138659954071,
+      -0.13848906755447388,
+      -0.14028912782669067,
+      -0.10836859047412872,
+      -0.12526217103004456,
+      -0.10552474856376648,
+      -0.06877261400222778,
+      -0.07132845371961594,
+      -0.08903142809867859,
+      -0.09589304774999619,
+      -0.10748133063316345,
+      -0.10274599492549896,
+      -0.0769767239689827,
+      -0.0957116186618805,
+      -0.12316339462995529,
+      -0.06739163398742676,
+      -0.06931999325752258,
+      -0.09397172182798386,
+      -0.09193585067987442,
+      -0.0884222611784935,
+      -0.07503781467676163,
+      -0.09063179045915604,
+      -0.09516908973455429,
+      -0.061625346541404724,
+      -0.09922279417514801,
+      -0.09997958689928055,
+      -0.07657846063375473,
+      -0.06621500849723816,
+      -0.12527738511562347,
+      -0.06875374913215637,
+      -0.076743945479393,
+      -0.06615175306797028,
+      -0.06891793757677078,
+      -0.024998793378472328,
+      -0.045882828533649445,
+      -0.08353117108345032,
+      -0.054521091282367706,
+      -0.08604641258716583,
+      -0.04778473079204559,
+      -0.04239065572619438,
+      -0.060501810163259506,
+      -0.04899304360151291,
+      -0.05773915722966194,
+      -0.07319746911525726,
+      -0.04934433475136757,
+      -0.10862579196691513,
+      -0.07421281933784485,
+      -0.06888279318809509,
+      -0.08675345033407211,
+      -0.06313051283359528,
+      -0.029470132663846016,
+      -0.04151252284646034,
+      -0.059764422476291656,
+      -0.021982308477163315,
+      -0.04667424410581589,
+      -0.04010389745235443,
+      -0.04385942220687866,
+      -0.0653829425573349,
+      -0.029450079426169395,
+      -0.024109097197651863,
+      -0.056966543197631836,
+      -0.062335290014743805,
+      -0.056706782430410385,
+      -0.03215301036834717,
+      -0.09718271344900131,
+      -0.035244062542915344,
+      -0.03974173218011856,
+      -0.02343435399234295,
+      -0.06278496980667114,
+      -0.044510968029499054,
+      -0.047638002783060074,
+      -0.04119744151830673,
+      -0.027996622025966644,
+      -0.033130232244729996,
+      -0.028617382049560547,
+      -0.035466283559799194,
+      -0.027308251708745956,
+      -0.046674806624650955,
+      -0.02939789928495884,
+      -0.04838502034544945,
+      -0.039561882615089417,
+      -0.05009254813194275,
+      -0.05724010989069939,
+      -0.030581075698137283,
+      -0.013905520550906658,
+      -0.027479475364089012,
+      -0.016471121460199356,
+      -0.03298879414796829,
+      -0.035792917013168335,
+      -0.025578202679753304,
+      0.008877278305590153,
+      -0.032132674008607864,
+      -0.06189657002687454,
+      -0.041393741965293884,
+      -0.010720747523009777,
+      -0.012072321027517319,
+      -0.036452531814575195,
+      -0.015315920114517212,
+      -0.025097481906414032,
+      -0.013919446617364883,
+      -0.03066857159137726,
+      -0.012647179886698723,
+      -0.018482545390725136
+    ],
+    "abs_loss": [
+      3.396123170852661,
+      3.3424994945526123,
+      3.237919569015503,
+      2.879476308822632,
+      2.70353627204895,
+      2.751377820968628,
+      2.7063636779785156,
+      2.7233448028564453,
+      2.7206146717071533,
+      2.7294867038726807,
+      2.7039129734039307,
+      2.7369041442871094,
+      2.778230667114258,
+      2.7252705097198486,
+      2.7478744983673096,
+      2.7141618728637695,
+      2.72943377494812,
+      2.680093765258789,
+      2.6801583766937256,
+      2.641345739364624,
+      2.60233473777771,
+      2.562516689300537,
+      2.468102216720581,
+      2.318903684616089,
+      2.213289976119995,
+      2.0808746814727783,
+      1.912829041481018,
+      1.8291109800338745,
+      1.7464908361434937,
+      1.5849313735961914,
+      1.4459643363952637,
+      1.3509396314620972,
+      1.2760063409805298,
+      1.1705503463745117,
+      1.0894227027893066,
+      1.0287818908691406,
+      0.9590547680854797,
+      0.9078173637390137,
+      0.7793462872505188,
+      0.7693994641304016,
+      0.7726138234138489,
+      0.7485592365264893,
+      0.6382789015769958,
+      0.6104655861854553,
+      0.6006044149398804,
+      0.5588434338569641,
+      0.5816371440887451,
+      0.5126509666442871,
+      0.45538386702537537,
+      0.5014699101448059,
+      0.4625111520290375,
+      0.37617024779319763,
+      0.382155179977417,
+      0.4250859320163727,
+      0.36882543563842773,
+      0.3938884735107422,
+      0.3570753037929535,
+      0.36372554302215576,
+      0.3226584792137146,
+      0.3447456359863281,
+      0.2896538972854614,
+      0.33224985003471375,
+      0.27562522888183594,
+      0.3141898810863495,
+      0.3241802752017975,
+      0.3382759988307953,
+      0.29824408888816833,
+      0.24730698764324188,
+      0.24693258106708527,
+      0.22624905407428741,
+      0.24490737915039062,
+      0.1943686455488205,
+      0.20466960966587067,
+      0.20098577439785004,
+      0.22724151611328125,
+      0.19813261926174164,
+      0.203541561961174,
+      0.22373290359973907,
+      0.22397269308567047,
+      0.20228464901447296,
+      0.2084556370973587,
+      0.19100064039230347,
+      0.1635003238916397,
+      0.16662131249904633,
+      0.14338652789592743,
+      0.15400701761245728,
+      0.15842418372631073,
+      0.1730835884809494,
+      0.16437025368213654,
+      0.15240396559238434,
+      0.1625756472349167,
+      0.13809937238693237,
+      0.13748037815093994,
+      0.12964463233947754,
+      0.11633375287055969,
+      0.10566169023513794,
+      0.10939222574234009,
+      0.13094063103199005,
+      0.11209895461797714,
+      0.10892510414123535,
+      0.11485173553228378,
+      0.09201198816299438,
+      0.10359283536672592,
+      0.1163104996085167,
+      0.09142263978719711,
+      0.08462992310523987,
+      0.1283082216978073,
+      0.06860216706991196,
+      0.09294155985116959,
+      0.07824031263589859,
+      0.08229020982980728,
+      0.07408256083726883,
+      0.06520144641399384,
+      0.08309247344732285,
+      0.06883866339921951,
+      0.09266123920679092,
+      0.10117604583501816,
+      0.05110514163970947,
+      0.06530681252479553,
+      0.06821984052658081,
+      0.06376788020133972,
+      0.04936259612441063,
+      0.05275489762425423,
+      0.05426734685897827,
+      0.04458294436335564,
+      0.04934551194310188,
+      0.04947466775774956,
+      0.04795296862721443,
+      0.04942786693572998,
+      0.03245225176215172,
+      0.0516023151576519,
+      0.04867574945092201,
+      0.03653515875339508,
+      0.05720725655555725,
+      0.03877228870987892,
+      0.04510463774204254,
+      0.04770177602767944,
+      0.0382237583398819,
+      0.0371100977063179,
+      0.03406527638435364,
+      0.022988414391875267,
+      0.039999619126319885,
+      0.0444638729095459,
+      0.032930318266153336,
+      0.03318178281188011,
+      0.04087072238326073,
+      0.03708834946155548,
+      0.03177458047866821,
+      0.023840904235839844,
+      0.032990165054798126
+    ],
+    "zipf_loss": [
+      6.633932590484619,
+      4.6788129806518555,
+      2.6920056343078613,
+      1.208571195602417,
+      0.8930506706237793,
+      0.8545060157775879,
+      0.8466107845306396,
+      0.8420599699020386,
+      0.8411819934844971,
+      0.8382439613342285,
+      0.8394572138786316,
+      0.8424986004829407,
+      0.8458808064460754,
+      0.8508229851722717,
+      0.8520069718360901,
+      0.8505419492721558,
+      0.850071132183075,
+      0.8467832803726196,
+      0.8426898121833801,
+      0.8406801223754883,
+      0.8364794850349426,
+      0.8292058110237122,
+      0.8211670517921448,
+      0.8094773292541504,
+      0.7566565871238708,
+      0.7348451614379883,
+      0.6645092368125916,
+      0.6135583519935608,
+      0.5719212293624878,
+      0.5288562774658203,
+      0.4513993561267853,
+      0.430299311876297,
+      0.37866878509521484,
+      0.3421209454536438,
+      0.30126953125,
+      0.2798933982849121,
+      0.26044607162475586,
+      0.24395059049129486,
+      0.2619638442993164,
+      0.2412581741809845,
+      0.21857544779777527,
+      0.24845613539218903,
+      0.23089835047721863,
+      0.20550085604190826,
+      0.2308894395828247,
+      0.2196965366601944,
+      0.19570517539978027,
+      0.18898549675941467,
+      0.1809394657611847,
+      0.2021942436695099,
+      0.20987671613693237,
+      0.17060840129852295,
+      0.188960000872612,
+      0.19357092678546906,
+      0.18794670701026917,
+      0.16501688957214355,
+      0.14800027012825012,
+      0.1595587581396103,
+      0.16403374075889587,
+      0.12998756766319275,
+      0.12890611588954926,
+      0.13138753175735474,
+      0.13559846580028534,
+      0.1301901638507843,
+      0.13099274039268494,
+      0.13480272889137268,
+      0.13059218227863312,
+      0.11243785172700882,
+      0.09235051274299622,
+      0.09387483447790146,
+      0.10944969207048416,
+      0.09284092485904694,
+      0.09660668671131134,
+      0.10167653113603592,
+      0.0987236499786377,
+      0.10011626780033112,
+      0.10115612298250198,
+      0.12353251129388809,
+      0.1202351450920105,
+      0.10616868734359741,
+      0.09351634979248047,
+      0.07995074987411499,
+      0.09028695523738861,
+      0.0876828134059906,
+      0.09194929897785187,
+      0.073862724006176,
+      0.08123059570789337,
+      0.08037391304969788,
+      0.07786835730075836,
+      0.07496531307697296,
+      0.08322058618068695,
+      0.07853300869464874,
+      0.07418511807918549,
+      0.09266919642686844,
+      0.08716315031051636,
+      0.0727902203798294,
+      0.06619153916835785,
+      0.08267205953598022,
+      0.06412699073553085,
+      0.06240355595946312,
+      0.06793072819709778,
+      0.05992574244737625,
+      0.04722611978650093,
+      0.05602562054991722,
+      0.0673787072300911,
+      0.04700526222586632,
+      0.08519687503576279,
+      0.07912342995405197,
+      0.05809217691421509,
+      0.06705207377672195,
+      0.05788122117519379,
+      0.05755215883255005,
+      0.05167585238814354,
+      0.045686982572078705,
+      0.04658912867307663,
+      0.04480786994099617,
+      0.07247264683246613,
+      0.04801366850733757,
+      0.04901285469532013,
+      0.06211535260081291,
+      0.0398964062333107,
+      0.04905850440263748,
+      0.05960524082183838,
+      0.07253619283437729,
+      0.056698188185691833,
+      0.037566378712654114,
+      0.03416568040847778,
+      0.04495696723461151,
+      0.0487368181347847,
+      0.05734612047672272,
+      0.04905939847230911,
+      0.06548751890659332,
+      0.044604621827602386,
+      0.04995753988623619,
+      0.03840351849794388,
+      0.07173953205347061,
+      0.02899848483502865,
+      0.0713389590382576,
+      0.026631664484739304,
+      0.038310457020998,
+      0.037321291863918304,
+      0.029798053205013275,
+      0.04382714629173279,
+      0.033764541149139404,
+      0.04560966417193413,
+      0.046131432056427,
+      0.052731212228536606,
+      0.04077313095331192,
+      0.03853941708803177,
+      0.025815408676862717
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.8145691156387329,
+      0.44118931889533997,
+      0.22469070553779602,
+      0.16466204822063446,
+      0.15218132734298706,
+      0.13777633011341095,
+      0.13292548060417175,
+      0.1104842945933342,
+      0.10824868828058243,
+      0.11993641406297684,
+      0.12565867602825165,
+      0.13419589400291443,
+      0.14861086010932922,
+      0.1644168198108673,
+      0.17808333039283752,
+      0.18465976417064667,
+      0.19772332906723022,
+      0.20909027755260468,
+      0.2201639860868454,
+      0.23263131082057953,
+      0.24850168824195862,
+      0.26066479086875916,
+      0.2670159339904785,
+      0.2784411907196045,
+      0.28201547265052795,
+      0.28612732887268066,
+      0.29476505517959595,
+      0.30029648542404175,
+      0.30520692467689514,
+      0.3092881441116333,
+      0.30750569701194763,
+      0.3080735504627228,
+      0.3085198700428009,
+      0.30787330865859985,
+      0.30572807788848877,
+      0.30690455436706543,
+      0.3073018491268158,
+      0.30525630712509155,
+      0.311127245426178,
+      0.3117120862007141,
+      0.30561313033103943,
+      0.3098585307598114,
+      0.30772849917411804,
+      0.30842599272727966,
+      0.3059391975402832,
+      0.3027287721633911,
+      0.3016418516635895,
+      0.29902052879333496,
+      0.2971118092536926,
+      0.2960193455219269,
+      0.2868633270263672,
+      0.2864190638065338,
+      0.2834288477897644,
+      0.2859273850917816,
+      0.2813391387462616,
+      0.2839559316635132,
+      0.28302136063575745,
+      0.28215470910072327,
+      0.28377798199653625,
+      0.2831357419490814,
+      0.2788276970386505,
+      0.27627089619636536,
+      0.2742644250392914,
+      0.2668607532978058,
+      0.2634449303150177,
+      0.263126015663147,
+      0.25922033190727234,
+      0.2572340965270996,
+      0.2539440095424652,
+      0.2546662390232086,
+      0.2506442964076996,
+      0.2481406331062317,
+      0.24537904560565948,
+      0.24735184013843536,
+      0.2472449243068695,
+      0.24569280445575714,
+      0.24428997933864594,
+      0.24523788690567017,
+      0.2450263947248459,
+      0.241826131939888,
+      0.24275335669517517,
+      0.2461155652999878,
+      0.24606183171272278,
+      0.24856235086917877,
+      0.24427330493927002,
+      0.23939453065395355,
+      0.23890776932239532,
+      0.23704560101032257,
+      0.23868606984615326,
+      0.23708778619766235,
+      0.2383851855993271,
+      0.23969031870365143,
+      0.2362288534641266,
+      0.2352312058210373,
+      0.23260456323623657,
+      0.23294435441493988,
+      0.23012205958366394,
+      0.23283588886260986,
+      0.23191030323505402,
+      0.2334143966436386,
+      0.23454353213310242,
+      0.23472219705581665,
+      0.23399654030799866,
+      0.23243288695812225,
+      0.23214027285575867,
+      0.22992588579654694,
+      0.2305673211812973,
+      0.23458202183246613,
+      0.23625877499580383,
+      0.23581013083457947,
+      0.23660001158714294,
+      0.23592354357242584,
+      0.2376868575811386,
+      0.23830951750278473,
+      0.23824988305568695,
+      0.23867759108543396,
+      0.2383129745721817,
+      0.23837491869926453,
+      0.23966966569423676,
+      0.2380770742893219,
+      0.24042335152626038,
+      0.23980286717414856,
+      0.24055185914039612,
+      0.24150127172470093,
+      0.24219653010368347,
+      0.2439589947462082,
+      0.2448512315750122,
+      0.24388886988162994,
+      0.242813840508461,
+      0.2448635995388031,
+      0.24494120478630066,
+      0.24446451663970947,
+      0.24479520320892334,
+      0.24560023844242096,
+      0.24616402387619019,
+      0.2463158220052719,
+      0.24584315717220306,
+      0.24682603776454926,
+      0.24854815006256104,
+      0.24989233911037445,
+      0.2502584457397461,
+      0.25012826919555664,
+      0.251260370016098,
+      0.25164735317230225,
+      0.2518453896045685,
+      0.2522733509540558,
+      0.2523629367351532,
+      0.253236323595047,
+      0.2539100646972656,
+      0.25460585951805115
+    ],
+    "lr": [
+      8.376068376068378e-06,
+      1.6923076923076924e-05,
+      2.5470085470085475e-05,
+      3.401709401709402e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.993593461639863e-05,
+      3.934273662008964e-05,
+      3.874953862378065e-05,
+      3.815634062747166e-05,
+      3.7563142631162665e-05,
+      3.696994463485368e-05,
+      3.637674663854469e-05,
+      3.5783548642235693e-05,
+      3.519035064592671e-05,
+      3.459715264961771e-05,
+      3.400395465330873e-05,
+      3.341075665699973e-05,
+      3.281755866069075e-05,
+      3.2224360664381764e-05,
+      3.1251515950435014e-05,
+      3.065831795412602e-05,
+      3.0065119957817037e-05,
+      2.947192196150804e-05,
+      2.887872396519905e-05,
+      2.8285525968890065e-05,
+      2.769232797258107e-05,
+      2.709912997627208e-05,
+      2.6505931979963087e-05,
+      2.5912733983654104e-05,
+      2.5319535987345117e-05,
+      2.472633799103612e-05,
+      2.4133139994727132e-05,
+      2.3539941998418135e-05,
+      2.2946744002109148e-05,
+      2.1973899288162408e-05,
+      2.138070129185341e-05,
+      2.0787503295544424e-05,
+      2.019430529923544e-05,
+      1.9601107302926443e-05,
+      1.900790930661746e-05,
+      1.8414711310308462e-05,
+      1.7821513313999475e-05,
+      1.7228315317690488e-05,
+      1.663511732138149e-05,
+      1.6041919325072507e-05,
+      1.544872132876351e-05,
+      1.4855523332454524e-05,
+      1.4262325336145537e-05,
+      1.366912733983654e-05,
+      1.2696282625889797e-05,
+      1.2103084629580812e-05,
+      1.1509886633271816e-05,
+      1.0916688636962829e-05,
+      1.0323490640653833e-05,
+      9.730292644344846e-06,
+      9.137094648035861e-06,
+      8.543896651726864e-06,
+      7.950698655417878e-06,
+      7.357500659108881e-06,
+      6.764302662799895e-06,
+      6.171104666490909e-06,
+      5.577906670181913e-06,
+      4.984708673872927e-06,
+      4.391510677563931e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      750,
+      1532,
+      2314,
+      3096,
+      3878,
+      4660,
+      5442,
+      6224,
+      7006,
+      7788
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 0.9445833333333333,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9983471074380166,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9923076923076923,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9914529914529915,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.956989247311828,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.9955156950672646,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.59,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.74375,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.9413680781758957,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.25,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.29,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.83,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.47,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.766,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.985,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.996875,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.996219281663516,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9792746113989638,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.91,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.96484375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.9861111111111112,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.9,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9673202614379085,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.9896907216494846,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.994535519125683,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.98989898989899,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9953051643192489,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.81,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.87248322147651,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.2,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.44,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.8166666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.1,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.43,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.625,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.98,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9954441913439636,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.89,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.949238578680203,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.83,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9392712550607287,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.9607843137254902,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.78,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9395973154362416,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.9405940594059405,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.80875,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9901477832512315,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.9946236559139785,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.9625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.9869706840390879,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.85,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.86,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.9825,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.82,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.82,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.966,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.995,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.998109640831758,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9792746113989638,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.9813084112149533,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.99609375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9967320261437909,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.993103448275862,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.87,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.89,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.99,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.43,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.45,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.9275,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.985,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9983333333333333,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9954441913439636,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9949238578680203,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9919028340080972,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.95,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9832214765100671,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.9445833333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.9445833333333333,
+  "sft_overall_accuracy": 0.80875
+}

add_sub_sorl_v1_abs30_K1_50K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1151847aa9b351c2638dc7728c20c8a22a430e4c6963996c2455b67bba807d3
+size 650385300

add_sub_sorl_v1_abs30_K1_50K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 234,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 781,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K1_50K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 50000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162519662,
+  "run_name": "add_sub_sorl_v1_abs30_K1_50K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T14:15:29.386616+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "nil3iecw",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/nil3iecw",
+  "final_accuracy": 0.9445833333333333,
+  "sft_accuracy": 0.80875,
+  "eval_method": "ArithmeticEvaluator"
+}