Upload add_sub_sorl_v1_abs10_K1_10K

add_sub_sorl_v1_abs10_K1_10K/metrics.json

@@ -63,377 +63,377 @@
       3133
     ],
     "loss": [
-      9.340689659118652,
-      4.987771987915039,
-      3.1618072986602783,
-      2.856262445449829,
-      2.5268046855926514,
-      2.1091880798339844,
-      -0.44331324100494385,
-      -4.092588424682617,
-      -4.768612384796143,
-      -3.7999463081359863,
-      -1.6939505338668823,
-      -2.5403966903686523,
-      -2.0438146591186523,
-      -1.7041206359863281,
-      -1.5684587955474854,
-      -1.9102389812469482,
-      -1.880913257598877,
-      -1.9313758611679077,
-      -1.8197805881500244,
-      -1.7383019924163818,
-      -2.4177093505859375,
-      -1.3264975547790527,
-      -1.2404942512512207,
-      -1.5992217063903809,
-      -1.9741289615631104,
-      -1.0616717338562012,
-      -1.5642304420471191,
-      -1.3654898405075073,
-      -1.3151346445083618,
-      -0.7584792971611023,
-      -1.3367366790771484,
-      -1.2703979015350342,
-      -1.0298550128936768,
-      -1.6327556371688843,
-      -1.734535813331604,
-      -1.002307653427124,
-      -0.9018746018409729,
-      -2.051492214202881,
-      -0.6267629861831665,
-      -0.9340898990631104,
-      -0.7012732028961182,
-      -0.9501388669013977,
-      -0.9147610664367676,
-      -0.5015538930892944,
-      -1.2906075716018677,
-      -0.9741727113723755,
-      -0.7373249530792236,
-      -0.8238300085067749,
-      -0.48194777965545654,
-      -0.7015870213508606,
-      -0.4147356152534485,
-      -0.649371325969696,
-      -0.7445440292358398,
-      -0.7792887687683105,
-      -0.3351507782936096,
-      -0.4870011508464813,
-      -0.2853846549987793,
-      -0.4475783109664917,
-      -0.42194119095802307,
-      -0.4129505455493927
+      7.942777633666992,
+      4.879890441894531,
+      3.230745553970337,
+      3.171808958053589,
+      3.3972392082214355,
+      2.463961124420166,
+      -0.31970155239105225,
+      -5.494422912597656,
+      -6.8470611572265625,
+      -4.793292045593262,
+      -2.040498733520508,
+      -2.3940882682800293,
+      -2.170124053955078,
+      -1.7399487495422363,
+      -2.3086109161376953,
+      -2.7756190299987793,
+      -2.891526699066162,
+      -2.4666450023651123,
+      -2.295619010925293,
+      -2.048872470855713,
+      -1.5986868143081665,
+      -2.349621057510376,
+      -1.2452442646026611,
+      -2.901057004928589,
+      -1.3245774507522583,
+      -0.9554328918457031,
+      -0.9006512761116028,
+      -0.7657713890075684,
+      -0.7965363264083862,
+      -1.0751973390579224,
+      -0.8257421255111694,
+      -0.7674775123596191,
+      -0.6108952164649963,
+      -0.39728718996047974,
+      -1.0187804698944092,
+      -0.31591135263442993,
+      -0.39469149708747864,
+      -0.2729785442352295,
+      -0.10930931568145752,
+      -0.24461984634399414,
+      -0.08862721920013428,
+      -0.2117801159620285,
+      -0.16590479016304016,
+      -0.38362351059913635,
+      -0.4167384207248688,
+      -0.40334945917129517,
+      -0.19316883385181427,
+      -0.14879119396209717,
+      -0.19406011700630188,
+      -0.17914623022079468,
+      -0.2789972126483917,
+      -0.05913069099187851,
+      -0.29538771510124207,
+      -0.36722680926322937,
+      -0.35507896542549133,
+      -0.05119173973798752,
+      -0.11240553855895996,
+      -0.19777432084083557,
+      -0.007915813475847244,
+      -0.033937860280275345
     ],
     "base_loss": [
-      7.390318870544434,
-      3.826241970062256,
-      1.9951651096343994,
-      1.8394466638565063,
-      1.8613300323486328,
-      1.835070252418518,
-      1.8304622173309326,
-      1.7413225173950195,
-      1.6582175493240356,
-      1.3251668214797974,
-      0.9685406684875488,
-      0.9228308796882629,
-      0.7192235589027405,
-      0.5874363780021667,
-      0.5402608513832092,
-      0.5179902911186218,
-      0.47875410318374634,
-      0.4619639217853546,
-      0.39008235931396484,
-      0.43836668133735657,
-      0.48746830224990845,
-      0.2996414601802826,
-      0.26576855778694153,
-      0.3262496590614319,
-      0.33699464797973633,
-      0.24052418768405914,
-      0.2867306172847748,
-      0.23591551184654236,
-      0.233742356300354,
-      0.15799184143543243,
-      0.2535308301448822,
-      0.22049279510974884,
-      0.1870928704738617,
-      0.2405395358800888,
-      0.25070902705192566,
-      0.16667716205120087,
-      0.15809106826782227,
-      0.31137070059776306,
-      0.11368274688720703,
-      0.14004917442798615,
-      0.13582302629947662,
-      0.14286470413208008,
-      0.13543015718460083,
-      0.08498942852020264,
-      0.18833109736442566,
-      0.13422875106334686,
-      0.10775727778673172,
-      0.12300104647874832,
-      0.07167976349592209,
-      0.11587230116128922,
-      0.0645364299416542,
-      0.09650751203298569,
-      0.09376537799835205,
-      0.09722624719142914,
-      0.045721422880887985,
-      0.06196039915084839,
-      0.05605100467801094,
-      0.06696300953626633,
-      0.053617555648088455,
-      0.052670858800411224
+      7.671196937561035,
+      4.010580062866211,
+      1.8913097381591797,
+      1.9108556509017944,
+      1.8557425737380981,
+      1.833831548690796,
+      1.82988440990448,
+      1.787339448928833,
+      1.7512098550796509,
+      1.2430487871170044,
+      0.9070897698402405,
+      0.9038087725639343,
+      0.7104665637016296,
+      0.6114657521247864,
+      0.6148020625114441,
+      0.5949119329452515,
+      0.5328211188316345,
+      0.44292375445365906,
+      0.41273045539855957,
+      0.3532269299030304,
+      0.2902839481830597,
+      0.36093834042549133,
+      0.24082647264003754,
+      0.40677782893180847,
+      0.2435729205608368,
+      0.17918619513511658,
+      0.16348111629486084,
+      0.14683635532855988,
+      0.1624801903963089,
+      0.17330823838710785,
+      0.14925424754619598,
+      0.13458098471164703,
+      0.11640487611293793,
+      0.09549444168806076,
+      0.16984343528747559,
+      0.0791398137807846,
+      0.08832497149705887,
+      0.061722755432128906,
+      0.04690014570951462,
+      0.0503598190844059,
+      0.04861803725361824,
+      0.04905056953430176,
+      0.04119657352566719,
+      0.06299275904893875,
+      0.0651763305068016,
+      0.05840466544032097,
+      0.03202011436223984,
+      0.029487838968634605,
+      0.035392142832279205,
+      0.03208036348223686,
+      0.04259788244962692,
+      0.020311372354626656,
+      0.041775498539209366,
+      0.050391655415296555,
+      0.05066566914319992,
+      0.008960098959505558,
+      0.018476324155926704,
+      0.027055857703089714,
+      0.005176883656531572,
+      0.008925193920731544
     ],
     "info_loss": [
-      -0.4859275817871094,
-      -0.1129457950592041,
-      -0.07287442684173584,
-      -0.0863027572631836,
-      -0.12096810340881348,
-      -0.1600182056427002,
-      -0.41505467891693115,
-      -0.7713239789009094,
-      -0.8302392959594727,
-      -0.6981891989707947,
-      -0.4436832070350647,
-      -0.5114461183547974,
-      -0.42171621322631836,
-      -0.35522836446762085,
-      -0.32737311720848083,
-      -0.3521292805671692,
-      -0.33850210905075073,
-      -0.33436745405197144,
-      -0.3098219931125641,
-      -0.3004074692726135,
-      -0.3650834560394287,
-      -0.2377147525548935,
-      -0.21807271242141724,
-      -0.2605586349964142,
-      -0.2963896691799164,
-      -0.19201940298080444,
-      -0.2403343915939331,
-      -0.2125036120414734,
-      -0.2005012035369873,
-      -0.14176680147647858,
-      -0.20001220703125,
-      -0.18811163306236267,
-      -0.16207079589366913,
-      -0.22838185727596283,
-      -0.23002460598945618,
-      -0.1475900262594223,
-      -0.1343834400177002,
-      -0.26728555560112,
-      -0.10240201652050018,
-      -0.1339702010154724,
-      -0.11256629973649979,
-      -0.13482710719108582,
-      -0.12813962996006012,
-      -0.08094710111618042,
-      -0.1647377759218216,
-      -0.13163283467292786,
-      -0.1023663803935051,
-      -0.11182866990566254,
-      -0.06797388941049576,
-      -0.09571453928947449,
-      -0.06312839686870575,
-      -0.0817064419388771,
-      -0.09321703016757965,
-      -0.09572642296552658,
-      -0.04448504000902176,
-      -0.06147210672497749,
-      -0.040348708629608154,
-      -0.05947120487689972,
-      -0.05293218418955803,
-      -0.05189599096775055
+      -0.6543827056884766,
+      -0.1371753215789795,
+      -0.05590975284576416,
+      -0.062001824378967285,
+      -0.032317399978637695,
+      -0.11670446395874023,
+      -0.3843038082122803,
+      -0.8943668007850647,
+      -1.0231645107269287,
+      -0.7599178552627563,
+      -0.449214369058609,
+      -0.482033371925354,
+      -0.43131735920906067,
+      -0.3736041486263275,
+      -0.4215145707130432,
+      -0.4602014720439911,
+      -0.4593234658241272,
+      -0.3962600529193878,
+      -0.36444565653800964,
+      -0.3282434344291687,
+      -0.27206987142562866,
+      -0.3457140326499939,
+      -0.21850009262561798,
+      -0.3961044251918793,
+      -0.22760552167892456,
+      -0.16711612045764923,
+      -0.1588182896375656,
+      -0.14292679727077484,
+      -0.14594416320323944,
+      -0.16852590441703796,
+      -0.14467473328113556,
+      -0.13348457217216492,
+      -0.11509168893098831,
+      -0.09416339546442032,
+      -0.15572039783000946,
+      -0.07744807004928589,
+      -0.08698518574237823,
+      -0.06068040058016777,
+      -0.0375378355383873,
+      -0.04975220188498497,
+      -0.04101487994194031,
+      -0.04862057790160179,
+      -0.0394514799118042,
+      -0.05956646800041199,
+      -0.06477057933807373,
+      -0.05815031751990318,
+      -0.03168302774429321,
+      -0.029237886890769005,
+      -0.03526085987687111,
+      -0.03186190128326416,
+      -0.04240777716040611,
+      -0.020102767273783684,
+      -0.0416855663061142,
+      -0.0503324493765831,
+      -0.05060190334916115,
+      -0.008832032792270184,
+      -0.01841319166123867,
+      -0.026989510282874107,
+      -0.0051260534673929214,
+      -0.008811188861727715
     ],
     "abs_loss": [
-      2.2262747287750244,
-      1.8807449340820312,
-      1.833791732788086,
-      1.8724111318588257,
-      1.8476206064224243,
-      1.8559261560440063,
-      1.87335205078125,
-      1.8458162546157837,
-      1.813267707824707,
-      1.6594558954238892,
-      1.4701725244522095,
-      1.2008872032165527,
-      0.8842588067054749,
-      0.7724447250366211,
-      0.6772302985191345,
-      0.6123369336128235,
-      0.49044322967529297,
-      0.4312553107738495,
-      0.38433346152305603,
-      0.3575459420681,
-      0.2791888415813446,
-      0.3738294541835785,
-      0.2938316762447357,
-      0.2500404715538025,
-      0.24418945610523224,
-      0.2506295442581177,
-      0.1918429583311081,
-      0.21762271225452423,
-      0.19322265684604645,
-      0.174521803855896,
-      0.1660517454147339,
-      0.1862434595823288,
-      0.14573685824871063,
-      0.13860176503658295,
-      0.13301999866962433,
-      0.12413140386343002,
-      0.0981183648109436,
-      0.1176028847694397,
-      0.12784387171268463,
-      0.08845017105340958,
-      0.10072199255228043,
-      0.08419985324144363,
-      0.10068891197443008,
-      0.08726515620946884,
-      0.08775246143341064,
-      0.06938570737838745,
-      0.07202962785959244,
-      0.07672454416751862,
-      0.06882075220346451,
-      0.0632840245962143,
-      0.06827760487794876,
-      0.05087992176413536,
-      0.04930369183421135,
-      0.05453367158770561,
-      0.04361484572291374,
-      0.05198704078793526,
-      0.042146146297454834,
-      0.0432199127972126,
-      0.042553484439849854,
-      0.054908428341150284
+      2.223989725112915,
+      1.871252179145813,
+      1.8544975519180298,
+      1.8529828786849976,
+      1.7808314561843872,
+      1.5479220151901245,
+      1.4092364311218262,
+      1.4009379148483276,
+      1.320881962776184,
+      1.1338235139846802,
+      1.0362251996994019,
+      0.9345109462738037,
+      0.8529377579689026,
+      0.7628306746482849,
+      0.690753161907196,
+      0.6221196055412292,
+      0.5280961394309998,
+      0.46778860688209534,
+      0.3726791441440582,
+      0.31298816204071045,
+      0.2620382010936737,
+      0.18087559938430786,
+      0.21465563774108887,
+      0.16152803599834442,
+      0.15743668377399445,
+      0.14526526629924774,
+      0.14475920796394348,
+      0.12059714645147324,
+      0.1169460192322731,
+      0.09149991720914841,
+      0.09585744142532349,
+      0.0799269825220108,
+      0.08726473897695541,
+      0.06842993944883347,
+      0.0803660899400711,
+      0.07159644365310669,
+      0.07437893003225327,
+      0.06340949982404709,
+      0.06646980345249176,
+      0.07242506742477417,
+      0.05180174112319946,
+      0.06437482684850693,
+      0.07705993205308914,
+      0.05421462655067444,
+      0.04291941598057747,
+      0.03475573658943176,
+      0.03258237987756729,
+      0.04605608060956001,
+      0.029467642307281494,
+      0.02811291068792343,
+      0.016363779082894325,
+      0.02535133622586727,
+      0.01502001192420721,
+      0.016885360702872276,
+      0.029067935422062874,
+      0.019427979364991188,
+      0.028035461902618408,
+      0.012935961596667767,
+      0.03265028074383736,
+      0.02496322989463806
     ],
     "zipf_loss": [
-      6.587018966674805,
-      2.1029131412506104,
-      1.7120072841644287,
-      1.692602276802063,
-      1.6903936862945557,
-      1.6887073516845703,
-      1.6894363164901733,
-      1.6947468519210815,
-      1.694236397743225,
-      1.690833568572998,
-      1.6273235082626343,
-      1.531144618988037,
-      1.3656980991363525,
-      1.1834821701049805,
-      1.0972886085510254,
-      1.031829595565796,
-      0.9763095378875732,
-      0.9072092771530151,
-      0.8499235510826111,
-      0.7916512489318848,
-      0.7177382707595825,
-      0.7136257290840149,
-      0.6450810432434082,
-      0.655110776424408,
-      0.6283543109893799,
-      0.5929351449012756,
-      0.5331985950469971,
-      0.5018686056137085,
-      0.43681272864341736,
-      0.48374468088150024,
-      0.39324942231178284,
-      0.37160131335258484,
-      0.3891863524913788,
-      0.3966630697250366,
-      0.3016990125179291,
-      0.2945023775100708,
-      0.2740568518638611,
-      0.29823243618011475,
-      0.27079012989997864,
-      0.2567179501056671,
-      0.278494656085968,
-      0.24684756994247437,
-      0.22113612294197083,
-      0.21420112252235413,
-      0.1596638262271881,
-      0.2009882926940918,
-      0.17137852311134338,
-      0.1637832671403885,
-      0.1192292720079422,
-      0.13335762917995453,
-      0.14518415927886963,
-      0.06609760969877243,
-      0.08893045783042908,
-      0.07529586553573608,
-      0.05961669981479645,
-      0.06056078150868416,
-      0.05783679336309433,
-      0.07584869861602783,
-      0.0495077446103096,
-      0.04784762114286423
+      6.593008518218994,
+      2.053938388824463,
+      1.7130836248397827,
+      1.6956732273101807,
+      1.6865873336791992,
+      1.6423821449279785,
+      1.5525283813476562,
+      1.5218125581741333,
+      1.5012855529785156,
+      1.4494550228118896,
+      1.4409327507019043,
+      1.4289852380752563,
+      1.3472892045974731,
+      1.3083441257476807,
+      1.222657322883606,
+      1.169271469116211,
+      1.1160770654678345,
+      1.0062527656555176,
+      0.8988392949104309,
+      0.8490362763404846,
+      0.8055242300033569,
+      0.7284933924674988,
+      0.6774646043777466,
+      0.6370565295219421,
+      0.6921612024307251,
+      0.5220155715942383,
+      0.5095745921134949,
+      0.5046005249023438,
+      0.4887305498123169,
+      0.42760345339775085,
+      0.4621652364730835,
+      0.42479458451271057,
+      0.4148903489112854,
+      0.4420093297958374,
+      0.36054351925849915,
+      0.372269868850708,
+      0.3793974816799164,
+      0.2657617926597595,
+      0.21252191066741943,
+      0.19529986381530762,
+      0.26772335171699524,
+      0.21893762052059174,
+      0.17970743775367737,
+      0.14362695813179016,
+      0.16149911284446716,
+      0.11627347767353058,
+      0.08838309347629547,
+      0.10949420928955078,
+      0.12020957469940186,
+      0.10458115488290787,
+      0.1008462905883789,
+      0.11905048042535782,
+      0.07819044589996338,
+      0.08401751518249512,
+      0.09736760705709457,
+      0.02622569352388382,
+      0.050446510314941406,
+      0.04377133026719093,
+      0.03490281105041504,
+      0.04275251552462578
     ],
     "denoise_loss": [],
     "ortho_loss": [
-      0.3918377161026001,
-      0.20118887722492218,
-      0.12065954506397247,
-      0.0842815414071083,
-      0.07851361483335495,
-      0.08886810392141342,
-      0.1118798479437828,
-      0.16310755908489227,
-      0.21554774045944214,
-      0.2462739199399948,
-      0.2753961980342865,
-      0.30527833104133606,
-      0.33816447854042053,
-      0.3525509536266327,
-      0.36425891518592834,
-      0.3624407649040222,
-      0.36967605352401733,
-      0.3742343485355377,
-      0.3982963562011719,
-      0.3951874375343323,
-      0.3906378149986267,
-      0.3858691453933716,
-      0.390056312084198,
-      0.3822699785232544,
-      0.3838372230529785,
-      0.3822362422943115,
-      0.375870019197464,
-      0.38709157705307007,
-      0.3905230164527893,
-      0.39558085799217224,
-      0.3952634334564209,
-      0.39085277915000916,
-      0.39921504259109497,
-      0.39666083455085754,
-      0.3942829966545105,
-      0.3991093337535858,
-      0.39491263031959534,
-      0.3742454946041107,
-      0.3708445429801941,
-      0.37462660670280457,
-      0.3777347207069397,
-      0.36790767312049866,
-      0.36087462306022644,
-      0.3528926968574524,
-      0.3517568111419678,
-      0.3517394959926605,
-      0.3531302809715271,
-      0.35252636671066284,
-      0.35542812943458557,
-      0.35618269443511963,
-      0.3573307693004608,
-      0.3604697287082672,
-      0.36059707403182983,
-      0.36255255341529846,
-      0.362486869096756,
-      0.3620503544807434,
-      0.363882839679718,
-      0.36408552527427673,
-      0.3636517822742462,
-      0.3643637001514435
+      0.3635007441043854,
+      0.18963280320167542,
+      0.1485888510942459,
+      0.08869273215532303,
+      0.11493418365716934,
+      0.1571045219898224,
+      0.1805325299501419,
+      0.18971112370491028,
+      0.2055690884590149,
+      0.22591449320316315,
+      0.24307440221309662,
+      0.23882627487182617,
+      0.2520371079444885,
+      0.25790753960609436,
+      0.2630678117275238,
+      0.30761033296585083,
+      0.32258620858192444,
+      0.32426348328590393,
+      0.32995936274528503,
+      0.33315232396125793,
+      0.3359712064266205,
+      0.3444741368293762,
+      0.33952945470809937,
+      0.34313759207725525,
+      0.3325212001800537,
+      0.3572663366794586,
+      0.3538215458393097,
+      0.35331991314888,
+      0.34806913137435913,
+      0.35233867168426514,
+      0.3563789427280426,
+      0.35214924812316895,
+      0.3493984043598175,
+      0.35396456718444824,
+      0.33881986141204834,
+      0.34869933128356934,
+      0.3384784758090973,
+      0.3430968225002289,
+      0.33994701504707336,
+      0.3194200396537781,
+      0.3237259089946747,
+      0.3362569808959961,
+      0.3448117971420288,
+      0.35400205850601196,
+      0.3509945273399353,
+      0.3481496572494507,
+      0.34772804379463196,
+      0.34519320726394653,
+      0.3447076976299286,
+      0.3428516387939453,
+      0.34590229392051697,
+      0.3494682312011719,
+      0.3497217297554016,
+      0.34933948516845703,
+      0.35108673572540283,
+      0.35249459743499756,
+      0.35031837224960327,
+      0.3520292043685913,
+      0.35233885049819946,
+      0.3538017272949219
     ],
     "lr": [
       3.9200000000000004e-05,
@@ -521,8 +521,8 @@
       3083
     ],
     "eval_accuracy": [
-      0.01,
-      0.0,
+      0.02,
+      0.02,
       0.0,
       0.0,
       0.0,
@@ -543,573 +543,573 @@
       0.0
     ]
   },
-  "final_accuracy": 0.9470833333333334,
+  "final_accuracy": 0.9561538461538461,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.9,
-        "digit_accuracy": 0.9857142857142858,
-        "n_examples": 50,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9864406779661017,
-            "count": 295
+            "accuracy": 0.996694214876033,
+            "count": 605
           },
           "SS": {
-            "accuracy": 0.9818181818181818,
-            "count": 55
+            "accuracy": 1.0,
+            "count": 95
           }
         }
       },
       "add_S1": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 126
+            "count": 204
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 79
+            "count": 169
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 21
+            "count": 31
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 124
+            "count": 296
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.88,
-        "digit_accuracy": 0.9771428571428571,
-        "n_examples": 50,
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9733333333333334,
-            "count": 75
+            "accuracy": 1.0,
+            "count": 163
           },
           "SC": {
-            "accuracy": 0.967741935483871,
-            "count": 62
+            "accuracy": 1.0,
+            "count": 130
           },
           "SS": {
-            "accuracy": 0.9230769230769231,
-            "count": 39
+            "accuracy": 1.0,
+            "count": 87
           },
           "UC": {
-            "accuracy": 0.990990990990991,
-            "count": 111
+            "accuracy": 1.0,
+            "count": 203
           },
           "US": {
             "accuracy": 1.0,
-            "count": 63
+            "count": 117
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.72,
-        "digit_accuracy": 0.96,
-        "n_examples": 50,
+        "full_accuracy": 0.93,
+        "digit_accuracy": 0.99,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 60
+            "count": 121
           },
           "SC": {
-            "accuracy": 0.9824561403508771,
-            "count": 57
+            "accuracy": 1.0,
+            "count": 121
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 19
+            "count": 49
           },
           "UC": {
-            "accuracy": 0.875,
-            "count": 104
+            "accuracy": 0.9623655913978495,
+            "count": 186
           },
           "US": {
             "accuracy": 1.0,
-            "count": 110
+            "count": 223
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.62,
-        "digit_accuracy": 0.9228571428571428,
-        "n_examples": 50,
+        "full_accuracy": 0.71,
+        "digit_accuracy": 0.9585714285714285,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 104
           },
           "SC": {
-            "accuracy": 0.9807692307692307,
-            "count": 52
+            "accuracy": 1.0,
+            "count": 106
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 7
+            "count": 23
           },
           "UC": {
-            "accuracy": 0.8314606741573034,
-            "count": 89
+            "accuracy": 0.825,
+            "count": 160
           },
           "US": {
-            "accuracy": 0.9285714285714286,
-            "count": 154
+            "accuracy": 0.996742671009772,
+            "count": 307
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.42,
-        "digit_accuracy": 0.7971428571428572,
-        "n_examples": 50,
+        "full_accuracy": 0.61,
+        "digit_accuracy": 0.91,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.48,
-            "count": 50
+            "accuracy": 0.71,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.775,
-            "count": 200
+            "accuracy": 0.915,
+            "count": 400
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.9,
-        "digit_accuracy": 0.9457142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.83,
+        "digit_accuracy": 0.9528571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9,
-            "count": 50
+            "accuracy": 0.93,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.944,
-            "count": 250
+            "accuracy": 0.948,
+            "count": 500
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.98,
-        "digit_accuracy": 0.9971428571428571,
+        "full_accuracy": 0.985,
+        "digit_accuracy": 0.9978571428571429,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 431
+            "accuracy": 0.9977628635346756,
+            "count": 447
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 316
+            "count": 320
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 39
+            "count": 56
           },
           "UC": {
-            "accuracy": 0.9928571428571429,
-            "count": 560
+            "accuracy": 0.996219281663516,
+            "count": 529
           },
           "US": {
             "accuracy": 1.0,
-            "count": 54
+            "count": 48
           }
         }
       },
       "add_C1": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 250
+            "count": 500
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.96,
-        "digit_accuracy": 0.9914285714285714,
-        "n_examples": 50,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 400
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9759036144578314,
-            "count": 83
+            "accuracy": 0.9871794871794872,
+            "count": 156
           },
           "US": {
-            "accuracy": 0.9411764705882353,
-            "count": 17
+            "accuracy": 1.0,
+            "count": 44
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.88,
-        "digit_accuracy": 0.98,
-        "n_examples": 50,
+        "full_accuracy": 0.95,
+        "digit_accuracy": 0.9928571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 150
+            "count": 300
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.94,
-            "count": 100
+            "accuracy": 0.9748743718592965,
+            "count": 199
           },
           "US": {
-            "accuracy": 0.98,
-            "count": 50
+            "accuracy": 1.0,
+            "count": 101
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.82,
-        "digit_accuracy": 0.9714285714285714,
-        "n_examples": 50,
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9914285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9393939393939394,
-            "count": 132
+            "accuracy": 0.9772727272727273,
+            "count": 264
           },
           "US": {
-            "accuracy": 0.9705882352941176,
-            "count": 68
+            "accuracy": 1.0,
+            "count": 136
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.68,
-        "digit_accuracy": 0.9457142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.92,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.8904109589041096,
-            "count": 146
+            "accuracy": 0.9741935483870968,
+            "count": 310
           },
           "US": {
-            "accuracy": 0.9711538461538461,
-            "count": 104
+            "accuracy": 1.0,
+            "count": 190
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.88,
-        "digit_accuracy": 0.9742857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.87,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9682539682539683,
-            "count": 189
+            "accuracy": 0.9675675675675676,
+            "count": 370
           },
           "US": {
-            "accuracy": 0.972972972972973,
-            "count": 111
+            "accuracy": 0.9826086956521739,
+            "count": 230
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 303
+            "count": 615
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 47
+            "count": 85
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.96,
-        "digit_accuracy": 0.9942857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.97,
+        "digit_accuracy": 0.9957142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 141
+            "accuracy": 0.9931506849315068,
+            "count": 292
           },
           "MB": {
-            "accuracy": 0.9861111111111112,
-            "count": 72
+            "accuracy": 1.0,
+            "count": 144
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 18
+            "count": 25
           },
           "UB": {
-            "accuracy": 0.9915966386554622,
-            "count": 119
+            "accuracy": 0.99581589958159,
+            "count": 239
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.92,
-        "digit_accuracy": 0.9885714285714285,
-        "n_examples": 50,
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9910714285714286,
-            "count": 112
+            "accuracy": 1.0,
+            "count": 211
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 53
+            "count": 115
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 47
+            "count": 85
           },
           "UB": {
-            "accuracy": 0.9764705882352941,
-            "count": 85
+            "accuracy": 1.0,
+            "count": 181
           },
           "UD": {
-            "accuracy": 0.9811320754716981,
-            "count": 53
+            "accuracy": 1.0,
+            "count": 108
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.22,
-        "digit_accuracy": 0.8857142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.92,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 97
+            "count": 179
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 51
+            "count": 103
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 27
+            "count": 56
           },
           "UB": {
-            "accuracy": 0.5,
-            "count": 74
+            "accuracy": 0.9463087248322147,
+            "count": 149
           },
           "UD": {
-            "accuracy": 0.9702970297029703,
-            "count": 101
+            "accuracy": 1.0,
+            "count": 213
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.7,
-        "n_examples": 50,
+        "full_accuracy": 0.3,
+        "digit_accuracy": 0.8814285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.14,
-            "count": 50
+            "accuracy": 0.33,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.5866666666666667,
-            "count": 150
+            "accuracy": 0.9466666666666667,
+            "count": 300
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5428571428571428,
-        "n_examples": 50,
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.7242857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.14,
-            "count": 50
+            "accuracy": 0.43,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.415,
-            "count": 200
+            "accuracy": 0.66,
+            "count": 400
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.98,
-        "digit_accuracy": 0.9971428571428571,
+        "full_accuracy": 0.985,
+        "digit_accuracy": 0.9978571428571429,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9982456140350877,
-            "count": 570
+            "accuracy": 0.9966666666666667,
+            "count": 600
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 277
+            "count": 267
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9957537154989384,
-            "count": 471
+            "accuracy": 0.9977220956719818,
+            "count": 439
           },
           "UD": {
-            "accuracy": 0.9655172413793104,
-            "count": 29
+            "accuracy": 1.0,
+            "count": 41
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.68,
-        "digit_accuracy": 0.9542857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9942857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 150
+            "count": 300
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.8415841584158416,
-            "count": 101
+            "accuracy": 0.9796954314720813,
+            "count": 197
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 49
+            "count": 103
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.58,
-        "digit_accuracy": 0.9171428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.87,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.8264462809917356,
-            "count": 121
+            "accuracy": 0.951417004048583,
+            "count": 247
           },
           "UD": {
-            "accuracy": 0.8987341772151899,
-            "count": 79
+            "accuracy": 0.9738562091503268,
+            "count": 153
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.58,
-        "digit_accuracy": 0.9228571428571428,
-        "n_examples": 50,
+        "full_accuracy": 0.9,
+        "digit_accuracy": 0.9757142857142858,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.881578947368421,
-            "count": 152
+            "accuracy": 0.9731543624161074,
+            "count": 298
           },
           "UD": {
-            "accuracy": 0.9081632653061225,
-            "count": 98
+            "accuracy": 0.9554455445544554,
+            "count": 202
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.7806666666666666,
-      "digit_accuracy": 0.9444761904761905,
-      "total_examples": 1500,
+      "overall_accuracy": 0.87,
+      "digit_accuracy": 0.9724175824175825,
+      "total_examples": 2600,
       "n_splits": 24
     }
   },
@@ -1119,167 +1119,167 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 50
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 295
+            "count": 605
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 55
+            "count": 95
           }
         }
       },
       "add_S1": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 126
+            "count": 204
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 79
+            "count": 169
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 21
+            "count": 31
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 124
+            "count": 296
           }
         }
       },
       "add_S2": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 75
+            "count": 163
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 62
+            "count": 130
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 39
+            "count": 87
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 111
+            "count": 203
           },
           "US": {
             "accuracy": 1.0,
-            "count": 63
+            "count": 117
           }
         }
       },
       "add_S3": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 60
+            "count": 121
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 57
+            "count": 121
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 19
+            "count": 49
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 186
           },
           "US": {
             "accuracy": 1.0,
-            "count": 110
+            "count": 223
           }
         }
       },
       "add_S4": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 104
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 52
+            "count": 106
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 7
+            "count": 23
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 89
+            "count": 160
           },
           "US": {
             "accuracy": 1.0,
-            "count": 154
+            "count": 307
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.54,
-        "digit_accuracy": 0.9342857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.61,
+        "digit_accuracy": 0.9442857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.54,
-            "count": 50
+            "accuracy": 0.61,
+            "count": 100
           },
           "US": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 400
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 1.0,
-        "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9928571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 1.0,
-            "count": 50
+            "accuracy": 0.99,
+            "count": 100
           },
           "US": {
-            "accuracy": 1.0,
-            "count": 250
+            "accuracy": 0.992,
+            "count": 500
           }
         }
       },
@@ -1290,291 +1290,291 @@
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 431
+            "count": 447
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 316
+            "count": 320
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 39
+            "count": 56
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 560
+            "count": 529
           },
           "US": {
             "accuracy": 1.0,
-            "count": 54
+            "count": 48
           }
         }
       },
       "add_C1": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 250
+            "count": 500
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           }
         }
       },
       "add_C2": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 400
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 83
+            "count": 156
           },
           "US": {
             "accuracy": 1.0,
-            "count": 17
+            "count": 44
           }
         }
       },
       "add_C3": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 150
+            "count": 300
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 199
           },
           "US": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 101
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 1.0,
-        "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.99,
+        "digit_accuracy": 0.9985714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 1.0,
-            "count": 132
+            "accuracy": 0.9962121212121212,
+            "count": 264
           },
           "US": {
             "accuracy": 1.0,
-            "count": 68
+            "count": 136
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.94,
-        "digit_accuracy": 0.9914285714285714,
-        "n_examples": 50,
+        "full_accuracy": 0.99,
+        "digit_accuracy": 0.9985714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9794520547945206,
-            "count": 146
+            "accuracy": 0.9967741935483871,
+            "count": 310
           },
           "US": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 190
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 1.0,
-        "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9942857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 1.0,
-            "count": 189
+            "accuracy": 0.9891891891891892,
+            "count": 370
           },
           "US": {
             "accuracy": 1.0,
-            "count": 111
+            "count": 230
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 303
+            "count": 615
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 47
+            "count": 85
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 141
+            "count": 292
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 72
+            "count": 144
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 18
+            "count": 25
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 119
+            "count": 239
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 112
+            "count": 211
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 53
+            "count": 115
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 47
+            "count": 85
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 85
+            "count": 181
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 53
+            "count": 108
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 97
+            "count": 179
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 51
+            "count": 103
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 27
+            "count": 56
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 74
+            "count": 149
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 101
+            "count": 213
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 150
+            "count": 300
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.34,
-        "digit_accuracy": 0.9057142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.39,
+        "digit_accuracy": 0.9128571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.34,
-            "count": 50
+            "accuracy": 0.39,
+            "count": 100
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 400
           }
         }
       },
@@ -1585,11 +1585,11 @@
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 570
+            "count": 600
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 277
+            "count": 267
           },
           "ME": {
             "accuracy": 1.0,
@@ -1597,91 +1597,91 @@
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 471
+            "count": 439
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 29
+            "count": 41
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 150
+            "count": 300
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 101
+            "count": 197
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 49
+            "count": 103
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 1.0,
         "digit_accuracy": 1.0,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 247
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 79
+            "count": 153
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.98,
-        "digit_accuracy": 0.9971428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9942857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.993421052631579,
-            "count": 152
+            "accuracy": 0.9865771812080537,
+            "count": 298
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 98
+            "count": 202
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.96,
-      "digit_accuracy": 0.9942857142857143,
-      "total_examples": 1500,
+      "overall_accuracy": 0.9561538461538461,
+      "digit_accuracy": 0.9936813186813187,
+      "total_examples": 2600,
       "n_splits": 24
     }
   },
-  "sorl_overall_accuracy": 0.9470833333333334,
-  "sft_overall_accuracy": 0.7258333333333333
+  "sorl_overall_accuracy": 0.9561538461538461,
+  "sft_overall_accuracy": 0.87
 }

add_sub_sorl_v1_abs10_K1_10K/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82f05b0b5afcd6ea5dc437d2363128885fcfd080a9b715032b42b76a2c61eb2d
+oid sha256:47ed228ea959d64128647e608261362d573f06e45dfd07c02259100cb8257aed
 size 650303660

add_sub_sorl_v1_abs10_K1_10K/train_config.json

@@ -36,7 +36,7 @@
   "eval_every": 156,
   "save_every": 999999,
   "eval_samples": 100,
-  "output_dir": "ckpt/sweep/as_sorl_abs10_K1_10K",
+  "output_dir": "ckpt/sweep/as_sorl_abs10_K1_10K_2L3H510d",
   "eval_K": 4,
   "alpha_traj": 0.0,
   "corrupt_method": "shuffle",
@@ -69,16 +69,21 @@
   "no_wandb": false,
   "n_params": 162499262,
   "run_name": "add_sub_sorl_v1_abs10_K1_10K",
-  "git_commit": "57deaa28d9c21e39ddac5ef448d6e1be992fba91",
-  "timestamp": "2026-04-13T10:55:46.960375+00:00",
+  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
+  "timestamp": "2026-04-16T04:12:01.606714+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_10K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "v1",
-  "wandb_run_id": "0r6snjo4",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/0r6snjo4",
-  "final_accuracy": 0.9470833333333334,
-  "sft_accuracy": 0.7258333333333333,
+  "wandb_run_id": "lb17t8m0",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/lb17t8m0",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "config_hash": "5637ae880e02",
+  "final_accuracy": 0.9561538461538461,
+  "sft_accuracy": 0.87,
   "eval_method": "ArithmeticEvaluator"
 }