Upload add_sub_sorl_v1_abs5_50K

add_sub_sorl_v1_abs5_50K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151649
+}

add_sub_sorl_v1_abs5_50K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs5_50K/metrics.json

@@ -0,0 +1,2257 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      832,
+      882,
+      932,
+      982,
+      1032,
+      1082,
+      1132,
+      1182,
+      1232,
+      1282,
+      1332,
+      1382,
+      1432,
+      1482,
+      1532,
+      1614,
+      1664,
+      1714,
+      1764,
+      1814,
+      1864,
+      1914,
+      1964,
+      2014,
+      2064,
+      2114,
+      2164,
+      2214,
+      2264,
+      2314,
+      2396,
+      2446,
+      2496,
+      2546,
+      2596,
+      2646,
+      2696,
+      2746,
+      2796,
+      2846,
+      2896,
+      2946,
+      2996,
+      3046,
+      3096,
+      3178,
+      3228,
+      3278,
+      3328,
+      3378,
+      3428,
+      3478,
+      3528,
+      3578,
+      3628,
+      3678,
+      3728,
+      3778,
+      3828,
+      3878,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4310,
+      4360,
+      4410,
+      4460,
+      4510,
+      4560,
+      4610,
+      4660,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5092,
+      5142,
+      5192,
+      5242,
+      5292,
+      5342,
+      5392,
+      5442,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5874,
+      5924,
+      5974,
+      6024,
+      6074,
+      6124,
+      6174,
+      6224,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6656,
+      6706,
+      6756,
+      6806,
+      6856,
+      6906,
+      6956,
+      7006,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7438,
+      7488,
+      7538,
+      7588,
+      7638,
+      7688,
+      7738,
+      7788
+    ],
+    "loss": [
+      18.44952964782715,
+      13.085131645202637,
+      9.236353874206543,
+      6.564369201660156,
+      4.570344924926758,
+      4.181574821472168,
+      4.083609580993652,
+      3.5401999950408936,
+      4.015557289123535,
+      3.110769748687744,
+      3.2566874027252197,
+      2.961395025253296,
+      2.5973992347717285,
+      2.910411834716797,
+      2.3630118370056152,
+      2.7860195636749268,
+      2.6816728115081787,
+      2.777552604675293,
+      0.953736424446106,
+      0.49653708934783936,
+      0.9060878753662109,
+      0.23619577288627625,
+      0.06146610155701637,
+      -2.7861270904541016,
+      -5.064986705780029,
+      -7.01597785949707,
+      -7.804261684417725,
+      -7.3292036056518555,
+      -3.4228708744049072,
+      -1.868177890777588,
+      -0.7043478488922119,
+      -1.123578429222107,
+      -0.8923968076705933,
+      -1.284299612045288,
+      -0.9496086835861206,
+      -0.6030840873718262,
+      -0.5289337635040283,
+      -0.4211568832397461,
+      -0.8843162655830383,
+      -0.9152053594589233,
+      -1.327749252319336,
+      -0.7607690691947937,
+      -0.8816430568695068,
+      -0.5288575887680054,
+      -1.1279367208480835,
+      -0.9791946411132812,
+      -0.8800588846206665,
+      -1.0997586250305176,
+      -0.677013635635376,
+      -0.8590450882911682,
+      -0.9123483300209045,
+      -0.7981971502304077,
+      -0.8279263377189636,
+      -1.1345430612564087,
+      -0.986440896987915,
+      -0.8299861550331116,
+      -1.1200369596481323,
+      -0.6735298037528992,
+      -0.7725755572319031,
+      -0.7352138161659241,
+      -1.1936038732528687,
+      -1.2745416164398193,
+      -1.1034529209136963,
+      -0.7662761807441711,
+      -1.0178569555282593,
+      -1.329269528388977,
+      -0.7702103853225708,
+      -0.9164995551109314,
+      -0.8080347776412964,
+      -0.9399026036262512,
+      -0.8073107600212097,
+      -0.9947050213813782,
+      -0.7936404943466187,
+      -0.9797720909118652,
+      -0.42384570837020874,
+      -0.703538179397583,
+      -0.9776149988174438,
+      -0.8658252358436584,
+      -0.9939393997192383,
+      -0.7609153985977173,
+      -1.029724359512329,
+      -0.7480278015136719,
+      -0.8076902031898499,
+      -1.0067354440689087,
+      -1.0216609239578247,
+      -1.0706768035888672,
+      -0.5090358257293701,
+      -0.7453817129135132,
+      -0.8285771608352661,
+      -0.636246919631958,
+      -0.8841879963874817,
+      -0.9027547240257263,
+      -0.8271254301071167,
+      -1.3286508321762085,
+      -1.0491383075714111,
+      -0.9472371935844421,
+      -0.45106542110443115,
+      -0.7443326115608215,
+      -0.9607822299003601,
+      -0.6500149369239807,
+      -0.7561839818954468,
+      -0.7872539758682251,
+      -0.5158911943435669,
+      -0.9013153910636902,
+      -0.6975657343864441,
+      -0.7518120408058167,
+      -0.8433217406272888,
+      -0.4446665346622467,
+      -0.30746784806251526,
+      -0.2535161077976227,
+      -0.37874695658683777,
+      -0.4520595669746399,
+      -0.7154723405838013,
+      -0.6318115592002869,
+      -0.6810263991355896,
+      -0.7082054615020752,
+      -0.8766549229621887,
+      -0.25121691823005676,
+      -0.5699273943901062,
+      -0.5364285707473755,
+      -0.7661551237106323,
+      -0.6259014010429382,
+      -0.48137322068214417,
+      -0.40757548809051514,
+      -0.7728077173233032,
+      -0.5705413222312927,
+      -0.5697341561317444,
+      -0.6499547958374023,
+      -0.32821428775787354,
+      -0.7727969884872437,
+      -0.5176405310630798,
+      -0.48732858896255493,
+      -0.6849607229232788,
+      -0.404767245054245,
+      -0.43089425563812256,
+      -0.8797918558120728,
+      -0.5332032442092896,
+      -0.45688626170158386,
+      -0.4607824385166168,
+      -0.16606812179088593,
+      -0.6697006225585938,
+      -0.2519899010658264,
+      -0.7106669545173645,
+      -0.4282820224761963,
+      -0.40481603145599365,
+      -0.6144987344741821,
+      -0.49886706471443176,
+      -0.3377566337585449,
+      -0.7905849814414978,
+      -0.660668134689331
+    ],
+    "base_loss": [
+      9.478815078735352,
+      7.147007465362549,
+      6.176869869232178,
+      4.685658931732178,
+      2.8318417072296143,
+      2.1071300506591797,
+      1.9056364297866821,
+      2.0133540630340576,
+      1.8907678127288818,
+      1.8147469758987427,
+      1.8525066375732422,
+      1.8073803186416626,
+      1.8407647609710693,
+      1.8241777420043945,
+      1.8264648914337158,
+      1.7623223066329956,
+      1.7605255842208862,
+      1.7310479879379272,
+      1.8446377515792847,
+      1.713182806968689,
+      1.6931337118148804,
+      1.760388970375061,
+      1.7527635097503662,
+      1.6280372142791748,
+      1.7455527782440186,
+      1.6810188293457031,
+      1.622275948524475,
+      1.5246162414550781,
+      1.0477672815322876,
+      0.853723406791687,
+      0.6793270111083984,
+      0.6314077377319336,
+      0.6169699430465698,
+      0.6420953869819641,
+      0.5424636006355286,
+      0.5155883431434631,
+      0.4695047438144684,
+      0.4409467279911041,
+      0.4563055634498596,
+      0.5218268632888794,
+      0.5143526196479797,
+      0.42720794677734375,
+      0.460726261138916,
+      0.38698291778564453,
+      0.4337417185306549,
+      0.41487598419189453,
+      0.40908247232437134,
+      0.4036952555179596,
+      0.39221641421318054,
+      0.39261919260025024,
+      0.3433583676815033,
+      0.37728267908096313,
+      0.3331398069858551,
+      0.35433879494667053,
+      0.35841792821884155,
+      0.3625955581665039,
+      0.3544054925441742,
+      0.32570600509643555,
+      0.3571837246417999,
+      0.3140494227409363,
+      0.3380233943462372,
+      0.33834314346313477,
+      0.3311188817024231,
+      0.28867587447166443,
+      0.31884241104125977,
+      0.31823158264160156,
+      0.30794399976730347,
+      0.31843724846839905,
+      0.3112233579158783,
+      0.29676029086112976,
+      0.2969057261943817,
+      0.309344619512558,
+      0.3272438645362854,
+      0.3020068109035492,
+      0.2556857466697693,
+      0.2985021471977234,
+      0.25877079367637634,
+      0.29064008593559265,
+      0.30364206433296204,
+      0.2723568081855774,
+      0.27450138330459595,
+      0.24431279301643372,
+      0.29581016302108765,
+      0.2845608592033386,
+      0.2741066515445709,
+      0.2935972809791565,
+      0.21326084434986115,
+      0.21694423258304596,
+      0.22341282665729523,
+      0.2794587314128876,
+      0.22667287290096283,
+      0.2661537528038025,
+      0.2429615557193756,
+      0.28923526406288147,
+      0.24430502951145172,
+      0.23418912291526794,
+      0.18439938127994537,
+      0.21811120212078094,
+      0.22203241288661957,
+      0.24544580280780792,
+      0.21295468509197235,
+      0.20914070308208466,
+      0.1781967133283615,
+      0.16821026802062988,
+      0.19853578507900238,
+      0.17450489103794098,
+      0.17776131629943848,
+      0.1535501480102539,
+      0.18070636689662933,
+      0.16060182452201843,
+      0.1624717265367508,
+      0.1593547910451889,
+      0.1648017019033432,
+      0.17393232882022858,
+      0.1839720904827118,
+      0.15841184556484222,
+      0.19553746283054352,
+      0.1543804109096527,
+      0.1661752611398697,
+      0.1486245095729828,
+      0.17132976651191711,
+      0.13604643940925598,
+      0.14909514784812927,
+      0.12037209421396255,
+      0.1600574105978012,
+      0.15595616400241852,
+      0.12468447536230087,
+      0.13306961953639984,
+      0.15979719161987305,
+      0.18670818209648132,
+      0.11783940345048904,
+      0.11976461857557297,
+      0.14803706109523773,
+      0.11798138171434402,
+      0.1370590478181839,
+      0.16336624324321747,
+      0.13137580454349518,
+      0.1245480626821518,
+      0.13690006732940674,
+      0.12265896052122116,
+      0.1351359784603119,
+      0.10710002481937408,
+      0.12433771789073944,
+      0.10955774784088135,
+      0.1063729077577591,
+      0.10943995416164398,
+      0.11198414862155914,
+      0.10653673857450485,
+      0.1396775096654892,
+      0.13088181614875793
+    ],
+    "info_loss": [
+      -0.24746036529541016,
+      -0.2758293151855469,
+      -0.23881292343139648,
+      -0.20038366317749023,
+      -0.15521621704101562,
+      -0.10576295852661133,
+      -0.08982551097869873,
+      -0.15279483795166016,
+      -0.09187543392181396,
+      -0.17403197288513184,
+      -0.16283965110778809,
+      -0.18757128715515137,
+      -0.22719800472259521,
+      -0.1941009759902954,
+      -0.2487088441848755,
+      -0.1989116668701172,
+      -0.2048863172531128,
+      -0.16251671314239502,
+      -0.15475523471832275,
+      -0.15556073188781738,
+      -0.10185074806213379,
+      -0.17042529582977295,
+      -0.18455612659454346,
+      -0.4623488187789917,
+      -0.7059006690979004,
+      -0.8879353404045105,
+      -0.9602308869361877,
+      -0.9024291634559631,
+      -0.4613347053527832,
+      -0.2857523560523987,
+      -0.14973372220993042,
+      -0.1866208016872406,
+      -0.16163870692253113,
+      -0.2030816376209259,
+      -0.1583549976348877,
+      -0.11972680687904358,
+      -0.1074303686618805,
+      -0.09313318133354187,
+      -0.14062458276748657,
+      -0.15033236145973206,
+      -0.1910918653011322,
+      -0.1248614490032196,
+      -0.14047962427139282,
+      -0.09757676720619202,
+      -0.16115576028823853,
+      -0.14492267370224,
+      -0.13368484377861023,
+      -0.15536251664161682,
+      -0.11111339926719666,
+      -0.12928149104118347,
+      -0.12962763011455536,
+      -0.12120935320854187,
+      -0.12030337750911713,
+      -0.15559454262256622,
+      -0.13799504935741425,
+      -0.1228315532207489,
+      -0.15123826265335083,
+      -0.10367529094219208,
+      -0.11647860705852509,
+      -0.1081794798374176,
+      -0.15632885694503784,
+      -0.16458545625209808,
+      -0.14640432596206665,
+      -0.10842107236385345,
+      -0.13644303381443024,
+      -0.16776512563228607,
+      -0.11094506084918976,
+      -0.12640875577926636,
+      -0.11487103998661041,
+      -0.1265791356563568,
+      -0.11343953013420105,
+      -0.13274838030338287,
+      -0.11428608000278473,
+      -0.13073886930942535,
+      -0.07039615511894226,
+      -0.10225088894367218,
+      -0.12548895180225372,
+      -0.11738622188568115,
+      -0.13179424405097961,
+      -0.10655610263347626,
+      -0.13193075358867645,
+      -0.100581094622612,
+      -0.11189407110214233,
+      -0.1300775110721588,
+      -0.13105560839176178,
+      -0.13781890273094177,
+      -0.07414810359477997,
+      -0.09778252989053726,
+      -0.10632943361997604,
+      -0.09285277128219604,
+      -0.11275361478328705,
+      -0.1176697313785553,
+      -0.10796576738357544,
+      -0.1628173440694809,
+      -0.13004958629608154,
+      -0.11895784735679626,
+      -0.06443554162979126,
+      -0.09720486402511597,
+      -0.11926433444023132,
+      -0.09020586311817169,
+      -0.09755898267030716,
+      -0.10018476843833923,
+      -0.07024268060922623,
+      -0.1073988527059555,
+      -0.09049692004919052,
+      -0.09413118660449982,
+      -0.10305048525333405,
+      -0.060574308037757874,
+      -0.049501851201057434,
+      -0.04185705631971359,
+      -0.055571191012859344,
+      -0.06160428375005722,
+      -0.08854448050260544,
+      -0.08096739649772644,
+      -0.08705917745828629,
+      -0.08730161935091019,
+      -0.10787373036146164,
+      -0.04118364304304123,
+      -0.07411957532167435,
+      -0.06995879113674164,
+      -0.09430643916130066,
+      -0.07795840501785278,
+      -0.06337588280439377,
+      -0.053233444690704346,
+      -0.09370310604572296,
+      -0.07298142462968826,
+      -0.07050549983978271,
+      -0.07878867536783218,
+      -0.04927454888820648,
+      -0.09804441034793854,
+      -0.06479005515575409,
+      -0.06163795664906502,
+      -0.0837118998169899,
+      -0.052562177181243896,
+      -0.05739227682352066,
+      -0.10462771356105804,
+      -0.06755431741476059,
+      -0.058615364134311676,
+      -0.06014535576105118,
+      -0.029701918363571167,
+      -0.0810285285115242,
+      -0.03681185841560364,
+      -0.08449738472700119,
+      -0.054240141063928604,
+      -0.05153011903166771,
+      -0.07323230057954788,
+      -0.06152670830488205,
+      -0.04645712301135063,
+      -0.09433066844940186,
+      -0.07951948046684265
+    ],
+    "abs_loss": [
+      1.596919059753418,
+      1.4947857856750488,
+      1.272780418395996,
+      1.1532820463180542,
+      1.0950367450714111,
+      1.0722317695617676,
+      1.0636725425720215,
+      1.072363257408142,
+      1.0710623264312744,
+      1.0716652870178223,
+      1.0752224922180176,
+      1.0683205127716064,
+      1.0763131380081177,
+      1.087066411972046,
+      1.0777089595794678,
+      1.0821614265441895,
+      1.0596169233322144,
+      0.9719851016998291,
+      0.39603376388549805,
+      0.3239653408527374,
+      0.34919559955596924,
+      0.4108819365501404,
+      0.3858979344367981,
+      0.4509623646736145,
+      0.5240193605422974,
+      0.387207955121994,
+      0.3914479613304138,
+      0.39853549003601074,
+      0.4153664708137512,
+      0.44240784645080566,
+      0.36345189809799194,
+      0.44602566957473755,
+      0.43691420555114746,
+      0.45479702949523926,
+      0.39506417512893677,
+      0.3761600852012634,
+      0.4015936851501465,
+      0.36450305581092834,
+      0.3448357582092285,
+      0.32362663745880127,
+      0.2948921322822571,
+      0.28737449645996094,
+      0.2977384924888611,
+      0.23896950483322144,
+      0.21982833743095398,
+      0.267986536026001,
+      0.1998273730278015,
+      0.17681656777858734,
+      0.17004144191741943,
+      0.18968412280082703,
+      0.2125006914138794,
+      0.18227949738502502,
+      0.18541792035102844,
+      0.18919432163238525,
+      0.12988169491291046,
+      0.14946985244750977,
+      0.19231896102428436,
+      0.19233956933021545,
+      0.15461456775665283,
+      0.21276609599590302,
+      0.21440747380256653,
+      0.20789165794849396,
+      0.17360037565231323,
+      0.1711350977420807,
+      0.1560533046722412,
+      0.16243748366832733,
+      0.12482132017612457,
+      0.1430300772190094,
+      0.17055252194404602,
+      0.15327060222625732,
+      0.15922632813453674,
+      0.1193196028470993,
+      0.13450534641742706,
+      0.12975062429904938,
+      0.08843386173248291,
+      0.12005540728569031,
+      0.09021551162004471,
+      0.10245305299758911,
+      0.07599258422851562,
+      0.06972776353359222,
+      0.05889254808425903,
+      0.050712861120700836,
+      0.047734662890434265,
+      0.04035104811191559,
+      0.051442451775074005,
+      0.057420313358306885,
+      0.05140750855207443,
+      0.051537640392780304,
+      0.037231944501399994,
+      0.035267315804958344,
+      0.027043268084526062,
+      0.02151094377040863,
+      0.03704604506492615,
+      0.02046104148030281,
+      0.015078360214829445,
+      0.023751340806484222,
+      0.03651346266269684,
+      0.03276602178812027,
+      0.0192105695605278,
+      0.037015970796346664,
+      0.024379881098866463,
+      0.018561003729701042,
+      0.020282700657844543,
+      0.016477016732096672,
+      0.020818620920181274,
+      0.02045612409710884,
+      0.025962576270103455,
+      0.01960516907274723,
+      0.017677582800388336,
+      0.011418328620493412,
+      0.01573646441102028,
+      0.010853487998247147,
+      0.012827918864786625,
+      0.01071641594171524,
+      0.014083122834563255,
+      0.012521816417574883,
+      0.02088414505124092,
+      0.013567490503191948,
+      0.014923598617315292,
+      0.01902923732995987,
+      0.0077822450548410416,
+      0.009491324424743652,
+      0.005033642053604126,
+      0.007618001662194729,
+      0.01025136187672615,
+      0.0033108433708548546,
+      0.025463657453656197,
+      0.01270243152976036,
+      0.01636495254933834,
+      0.00898685585707426,
+      0.017109215259552002,
+      0.02210104651749134,
+      0.015325679443776608,
+      0.007947909645736217,
+      0.00692419707775116,
+      0.012875733897089958,
+      0.013805515132844448,
+      0.0023547494783997536,
+      0.00827761646360159,
+      0.016240227967500687,
+      0.01196260191500187,
+      0.01752428524196148,
+      0.014476225711405277,
+      0.006404878571629524,
+      0.011613640002906322,
+      0.006845640949904919,
+      0.005130980629473925,
+      0.014368198812007904,
+      0.011586510576307774,
+      0.0030363979749381542
+    ],
+    "zipf_loss": [
+      11.285626411437988,
+      8.5469388961792,
+      5.320335388183594,
+      3.767218589782715,
+      3.181161642074585,
+      3.0248513221740723,
+      2.9698612689971924,
+      2.9475579261779785,
+      2.9364376068115234,
+      2.929176092147827,
+      2.9250550270080566,
+      2.9228954315185547,
+      2.92098331451416,
+      2.918537139892578,
+      2.9158644676208496,
+      2.904597759246826,
+      2.864048719406128,
+      2.5744731426239014,
+      0.6170476675033569,
+      0.3065650463104248,
+      0.19654211401939392,
+      0.13897156715393066,
+      0.11567406356334686,
+      0.1642279177904129,
+      0.19606496393680573,
+      0.14363624155521393,
+      0.13662658631801605,
+      0.13061854243278503,
+      0.10117242485284805,
+      0.09138116240501404,
+      0.07731714844703674,
+      0.06661941111087799,
+      0.06332895159721375,
+      0.05894177407026291,
+      0.05197131261229515,
+      0.040979593992233276,
+      0.0357058048248291,
+      0.032777901738882065,
+      0.031140435487031937,
+      0.03392868489027023,
+      0.039327800273895264,
+      0.031900085508823395,
+      0.0326530784368515,
+      0.03603024408221245,
+      0.027896452695131302,
+      0.028357498347759247,
+      0.02772437036037445,
+      0.03248963505029678,
+      0.024899842217564583,
+      0.02218230813741684,
+      0.019319508224725723,
+      0.01838580146431923,
+      0.023425905033946037,
+      0.04814408719539642,
+      0.022103402763605118,
+      0.02078687772154808,
+      0.018708189949393272,
+      0.018283190205693245,
+      0.019565414637327194,
+      0.011254880577325821,
+      0.010220523923635483,
+      0.012180733494460583,
+      0.012111258693039417,
+      0.012145175598561764,
+      0.012125654146075249,
+      0.013906529173254967,
+      0.0188140831887722,
+      0.014847748912870884,
+      0.01239692885428667,
+      0.013801433145999908,
+      0.014256242662668228,
+      0.011502091772854328,
+      0.008525846526026726,
+      0.012634696438908577,
+      0.015586694702506065,
+      0.008462974801659584,
+      0.00948210060596466,
+      0.007151526398956776,
+      0.01276172511279583,
+      0.02531604841351509,
+      0.009192601777613163,
+      0.00839912984520197,
+      0.01066692266613245,
+      0.005443607456982136,
+      0.00964430533349514,
+      0.008172955363988876,
+      0.014043688774108887,
+      0.010345567017793655,
+      0.007581113837659359,
+      0.009295312687754631,
+      0.01397094875574112,
+      0.005637701600790024,
+      0.005866094492375851,
+      0.008241323754191399,
+      0.005544657818973064,
+      0.00577706191688776,
+      0.005239288322627544,
+      0.00632823258638382,
+      0.00790767464786768,
+      0.002896323800086975,
+      0.004013112746179104,
+      0.003596983850002289,
+      0.006310561206191778,
+      0.0028152107261121273,
+      0.006785801611840725,
+      0.012949305586516857,
+      0.006825508549809456,
+      0.005565861240029335,
+      0.005076568573713303,
+      0.0033108023926615715,
+      0.012919609434902668,
+      0.0035431520082056522,
+      0.0038879462517797947,
+      0.0028584706597030163,
+      0.004184897523373365,
+      0.005146710202097893,
+      0.004456487018615007,
+      0.004882368259131908,
+      0.0036006877198815346,
+      0.012631922028958797,
+      0.004801270551979542,
+      0.016687100753188133,
+      0.002787112258374691,
+      0.0036250692792236805,
+      0.0031407540664076805,
+      0.0029856639448553324,
+      0.008090006187558174,
+      0.0035920641385018826,
+      0.003097500652074814,
+      0.020040202885866165,
+      0.010709725320339203,
+      0.007076219655573368,
+      0.0025886036455631256,
+      0.0020783632062375546,
+      0.005277036689221859,
+      0.0018314849585294724,
+      0.009583585895597935,
+      0.004483828321099281,
+      0.00294326851144433,
+      0.006668071262538433,
+      0.00425243005156517,
+      0.00727624399587512,
+      0.008521541953086853,
+      0.003921186085790396,
+      0.0029508713632822037,
+      0.007699757814407349,
+      0.0039028245955705643,
+      0.018841037526726723,
+      0.011885503306984901,
+      0.003341204486787319
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.6644464731216431,
+      0.390718936920166,
+      0.3397873342037201,
+      0.220010444521904,
+      0.17634470760822296,
+      0.11789368838071823,
+      0.0955289676785469,
+      0.07844500243663788,
+      0.07100518047809601,
+      0.061587732285261154,
+      0.062192827463150024,
+      0.07018491625785828,
+      0.06564756482839584,
+      0.0611240454018116,
+      0.05811262130737305,
+      0.059590209275484085,
+      0.05714119225740433,
+      0.05537182092666626,
+      0.05041731521487236,
+      0.05501237139105797,
+      0.05375546216964722,
+      0.05057421326637268,
+      0.05180239677429199,
+      0.06120557710528374,
+      0.07244491577148438,
+      0.08537489175796509,
+      0.09648114442825317,
+      0.10231027752161026,
+      0.11638256162405014,
+      0.11486325412988663,
+      0.1343725323677063,
+      0.14236973226070404,
+      0.15420310199260712,
+      0.1610349714756012,
+      0.2013687938451767,
+      0.2344922125339508,
+      0.20749914646148682,
+      0.22026267647743225,
+      0.21985097229480743,
+      0.24116598069667816,
+      0.2510566711425781,
+      0.27350345253944397,
+      0.25595226883888245,
+      0.25598257780075073,
+      0.27384111285209656,
+      0.2887703478336334,
+      0.2767501473426819,
+      0.28114053606987,
+      0.28092628717422485,
+      0.2867850959300995,
+      0.26383644342422485,
+      0.2907628118991852,
+      0.30944201350212097,
+      0.2962242662906647,
+      0.2884293496608734,
+      0.289986789226532,
+      0.2948214113712311,
+      0.3057153522968292,
+      0.30321916937828064,
+      0.30159616470336914,
+      0.2988344728946686,
+      0.3015638291835785,
+      0.29515600204467773,
+      0.3110824525356293,
+      0.31624576449394226,
+      0.3102674186229706,
+      0.3081153929233551,
+      0.3088230490684509,
+      0.3114872872829437,
+      0.3132418692111969,
+      0.30549758672714233,
+      0.30370408296585083,
+      0.31578776240348816,
+      0.31538280844688416,
+      0.3170136511325836,
+      0.3199930191040039,
+      0.3225075900554657,
+      0.33318015933036804,
+      0.33714455366134644,
+      0.334808349609375,
+      0.3351079225540161,
+      0.34916990995407104,
+      0.34625402092933655,
+      0.3513115346431732,
+      0.3460647165775299,
+      0.3537741005420685,
+      0.3563305735588074,
+      0.35665932297706604,
+      0.361850768327713,
+      0.3670676350593567,
+      0.3700326979160309,
+      0.3743707835674286,
+      0.37654909491539,
+      0.38266727328300476,
+      0.3887467086315155,
+      0.3925640285015106,
+      0.3913672864437103,
+      0.39072874188423157,
+      0.39183491468429565,
+      0.3888093829154968,
+      0.38142672181129456,
+      0.3814965784549713,
+      0.3843421936035156,
+      0.3847111761569977,
+      0.38761216402053833,
+      0.38767239451408386,
+      0.39530178904533386,
+      0.3994395136833191,
+      0.40793320536613464,
+      0.40820685029029846,
+      0.41256099939346313,
+      0.41478195786476135,
+      0.41378289461135864,
+      0.408303827047348,
+      0.40822821855545044,
+      0.41003018617630005,
+      0.40732336044311523,
+      0.41367655992507935,
+      0.411933571100235,
+      0.4172632396221161,
+      0.4182727336883545,
+      0.41697701811790466,
+      0.41414928436279297,
+      0.41985884308815,
+      0.42052218317985535,
+      0.41982269287109375,
+      0.41806432604789734,
+      0.41771870851516724,
+      0.42125946283340454,
+      0.4215148389339447,
+      0.4292573928833008,
+      0.4304172992706299,
+      0.4271102845668793,
+      0.425829142332077,
+      0.42774829268455505,
+      0.42603254318237305,
+      0.42814135551452637,
+      0.4269962012767792,
+      0.4269280433654785,
+      0.4310397803783417,
+      0.4302747845649719,
+      0.43275365233421326,
+      0.43321752548217773,
+      0.43411126732826233,
+      0.4353036880493164,
+      0.43637439608573914,
+      0.43796998262405396,
+      0.4378882050514221,
+      0.43787822127342224,
+      0.4387696385383606
+    ],
+    "lr": [
+      8.376068376068378e-06,
+      1.6923076923076924e-05,
+      2.5470085470085475e-05,
+      3.401709401709402e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.993593461639863e-05,
+      3.934273662008964e-05,
+      3.874953862378065e-05,
+      3.815634062747166e-05,
+      3.7563142631162665e-05,
+      3.696994463485368e-05,
+      3.637674663854469e-05,
+      3.5783548642235693e-05,
+      3.519035064592671e-05,
+      3.459715264961771e-05,
+      3.400395465330873e-05,
+      3.341075665699973e-05,
+      3.281755866069075e-05,
+      3.2224360664381764e-05,
+      3.1251515950435014e-05,
+      3.065831795412602e-05,
+      3.0065119957817037e-05,
+      2.947192196150804e-05,
+      2.887872396519905e-05,
+      2.8285525968890065e-05,
+      2.769232797258107e-05,
+      2.709912997627208e-05,
+      2.6505931979963087e-05,
+      2.5912733983654104e-05,
+      2.5319535987345117e-05,
+      2.472633799103612e-05,
+      2.4133139994727132e-05,
+      2.3539941998418135e-05,
+      2.2946744002109148e-05,
+      2.1973899288162408e-05,
+      2.138070129185341e-05,
+      2.0787503295544424e-05,
+      2.019430529923544e-05,
+      1.9601107302926443e-05,
+      1.900790930661746e-05,
+      1.8414711310308462e-05,
+      1.7821513313999475e-05,
+      1.7228315317690488e-05,
+      1.663511732138149e-05,
+      1.6041919325072507e-05,
+      1.544872132876351e-05,
+      1.4855523332454524e-05,
+      1.4262325336145537e-05,
+      1.366912733983654e-05,
+      1.2696282625889797e-05,
+      1.2103084629580812e-05,
+      1.1509886633271816e-05,
+      1.0916688636962829e-05,
+      1.0323490640653833e-05,
+      9.730292644344846e-06,
+      9.137094648035861e-06,
+      8.543896651726864e-06,
+      7.950698655417878e-06,
+      7.357500659108881e-06,
+      6.764302662799895e-06,
+      6.171104666490909e-06,
+      5.577906670181913e-06,
+      4.984708673872927e-06,
+      4.391510677563931e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      750,
+      1532,
+      2314,
+      3096,
+      3878,
+      4660,
+      5442,
+      6224,
+      7006,
+      7788
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.13,
+      0.31,
+      0.46,
+      0.61,
+      0.62,
+      0.68,
+      0.7,
+      0.7,
+      0.76
+    ]
+  },
+  "final_accuracy": 0.6954166666666667,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.88,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9818181818181818,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9894736842105263,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.84,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9656862745098039,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9822485207100592,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9763513513513513,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.72,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9754601226993865,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9461538461538461,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9655172413793104,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.916256157635468,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9829059829059829,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.46,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9669421487603306,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.9795918367346939,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7526881720430108,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.9013452914798207,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.29,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.9905660377358491,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.6625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.6644951140065146,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.23,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.41,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.5175,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.37,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.52,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.588,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.825,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9798657718120806,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.965625,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9640831758034026,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.9375,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.63,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8082901554404145,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.8878504672897196,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.55,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.7986111111111112,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.46,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8235294117647058,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.7835051546391752,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.53,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8661202185792349,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9145299145299145,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9900166389351082,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.9696969696969697,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.88,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.982078853046595,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.9724137931034482,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9583333333333334,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.9920634920634921,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.7,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9812206572769953,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9292035398230089,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9764705882352941,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.9005524861878453,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.9722222222222222,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.36,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.994413407821229,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9902912621359223,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.6510067114093959,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.8685446009389671,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.15,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.52,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.53,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.53,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.45,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.835,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9883333333333333,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9737827715355806,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9612756264236902,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.9512195121951219,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.67,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9866666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8781725888324873,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.883495145631068,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.62,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8906882591093117,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.8235294117647058,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.52,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8657718120805369,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.8564356435643564,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.5925,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 4,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.95,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9917355371900827,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.968421052631579,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9950980392156863,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9940828402366864,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9932432432432432,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.8,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9754601226993865,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9538461538461539,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9655172413793104,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9655172413793104,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9829059829059829,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.71,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.8548387096774194,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.968609865470852,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.53,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9903846153846154,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.9565217391304348,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.7875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.8566775244299675,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.47,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.59,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.7425,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.58,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.69,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.798,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.905,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9798657718120806,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.990625,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9464285714285714,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9867674858223062,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.81,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9119170984455959,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.9439252336448598,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.73,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.890625,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.9861111111111112,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.73,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9052287581699346,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.8917525773195877,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.76,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9371584699453552,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9487179487179487,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9900166389351082,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.98989898989899,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.996415770609319,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9583333333333334,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.9920634920634921,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.86,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9953051643192489,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9882352941176471,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.9337016574585635,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.9907407407407407,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.31,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9902912621359223,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5637583892617449,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.8779342723004695,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.11,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.38,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.6066666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.27,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.4775,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.935,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9983333333333333,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9925093632958801,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9811320754716981,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.979498861047836,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.74,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.99,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8934010152284264,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.8932038834951457,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.57,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8461538461538461,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.8496732026143791,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.45,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8322147651006712,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.8415841584158416,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.6954166666666667,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.6954166666666667,
+  "sft_overall_accuracy": 0.5925
+}

add_sub_sorl_v1_abs5_50K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:746f46a1162fc06ff0828d5d37320baefeb68c0ffeb8199f6f595f178fc25019
+size 650283250

add_sub_sorl_v1_abs5_50K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 234,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 781,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs5_K4_50K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 5,
+  "dataset_size": 50000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162494162,
+  "run_name": "add_sub_sorl_v1_abs5_50K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T12:51:18.803878+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "65m60ssj",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/65m60ssj",
+  "final_accuracy": 0.6954166666666667,
+  "sft_accuracy": 0.5925,
+  "eval_method": "ArithmeticEvaluator"
+}