Upload add_sub_sorl_v1_abs10_50K

add_sub_sorl_v1_abs10_50K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151654
+}

add_sub_sorl_v1_abs10_50K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs10_50K/metrics.json

@@ -0,0 +1,2257 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      832,
+      882,
+      932,
+      982,
+      1032,
+      1082,
+      1132,
+      1182,
+      1232,
+      1282,
+      1332,
+      1382,
+      1432,
+      1482,
+      1532,
+      1614,
+      1664,
+      1714,
+      1764,
+      1814,
+      1864,
+      1914,
+      1964,
+      2014,
+      2064,
+      2114,
+      2164,
+      2214,
+      2264,
+      2314,
+      2396,
+      2446,
+      2496,
+      2546,
+      2596,
+      2646,
+      2696,
+      2746,
+      2796,
+      2846,
+      2896,
+      2946,
+      2996,
+      3046,
+      3096,
+      3178,
+      3228,
+      3278,
+      3328,
+      3378,
+      3428,
+      3478,
+      3528,
+      3578,
+      3628,
+      3678,
+      3728,
+      3778,
+      3828,
+      3878,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4310,
+      4360,
+      4410,
+      4460,
+      4510,
+      4560,
+      4610,
+      4660,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5092,
+      5142,
+      5192,
+      5242,
+      5292,
+      5342,
+      5392,
+      5442,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5874,
+      5924,
+      5974,
+      6024,
+      6074,
+      6124,
+      6174,
+      6224,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6656,
+      6706,
+      6756,
+      6806,
+      6856,
+      6906,
+      6956,
+      7006,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7438,
+      7488,
+      7538,
+      7588,
+      7638,
+      7688,
+      7738,
+      7788
+    ],
+    "loss": [
+      16.378482818603516,
+      11.21066951751709,
+      7.495655059814453,
+      4.388947486877441,
+      3.196079730987549,
+      2.9686319828033447,
+      2.8419575691223145,
+      2.5052895545959473,
+      2.5802419185638428,
+      2.3319308757781982,
+      2.3821346759796143,
+      2.090925693511963,
+      2.018442392349243,
+      1.0070109367370605,
+      1.1892414093017578,
+      1.2099348306655884,
+      1.1913542747497559,
+      1.5283111333847046,
+      1.493798017501831,
+      1.1172080039978027,
+      1.102172613143921,
+      0.8972475528717041,
+      0.4773584008216858,
+      -0.12886226177215576,
+      -0.2539653778076172,
+      -0.7412328124046326,
+      -0.963010311126709,
+      -1.3086572885513306,
+      -2.896714448928833,
+      -4.599146366119385,
+      -7.537700176239014,
+      -9.402212142944336,
+      -9.60422134399414,
+      -10.469030380249023,
+      -10.379827499389648,
+      -10.23600959777832,
+      -11.531538009643555,
+      -11.200029373168945,
+      -10.917357444763184,
+      -12.113883972167969,
+      -11.413084983825684,
+      -11.835735321044922,
+      -12.362313270568848,
+      -12.001949310302734,
+      -11.52999210357666,
+      -12.262264251708984,
+      -12.913427352905273,
+      -12.869568824768066,
+      -12.031846046447754,
+      -12.037793159484863,
+      -12.129172325134277,
+      -11.788479804992676,
+      -11.700554847717285,
+      -13.019804954528809,
+      -12.643450736999512,
+      -12.31265926361084,
+      -12.17251968383789,
+      -12.044828414916992,
+      -12.354959487915039,
+      -13.130356788635254,
+      -12.746546745300293,
+      -12.852694511413574,
+      -12.775166511535645,
+      -13.146623611450195,
+      -13.022551536560059,
+      -13.103012084960938,
+      -12.287353515625,
+      -12.436395645141602,
+      -12.934889793395996,
+      -13.008482933044434,
+      -12.999844551086426,
+      -12.155572891235352,
+      -12.309308052062988,
+      -12.694921493530273,
+      -12.600488662719727,
+      -11.684075355529785,
+      -13.753292083740234,
+      -12.823161125183105,
+      -12.862237930297852,
+      -12.892120361328125,
+      -13.087822914123535,
+      -13.00316047668457,
+      -12.494159698486328,
+      -13.454633712768555,
+      -12.264759063720703,
+      -11.953563690185547,
+      -13.270965576171875,
+      -13.304547309875488,
+      -13.360864639282227,
+      -12.359524726867676,
+      -12.799259185791016,
+      -12.921852111816406,
+      -12.180639266967773,
+      -12.332677841186523,
+      -11.831616401672363,
+      -12.805928230285645,
+      -12.48134994506836,
+      -12.441422462463379,
+      -12.271108627319336,
+      -12.153956413269043,
+      -12.179333686828613,
+      -12.23478889465332,
+      -11.327901840209961,
+      -12.195158004760742,
+      -12.080638885498047,
+      -11.152657508850098,
+      -11.550673484802246,
+      -10.325593948364258,
+      -9.875710487365723,
+      -8.455825805664062,
+      -8.968307495117188,
+      -8.065058708190918,
+      -6.983817100524902,
+      -7.070670127868652,
+      -6.46987771987915,
+      -6.365849018096924,
+      -6.478631496429443,
+      -5.721439838409424,
+      -4.883893013000488,
+      -5.037868022918701,
+      -4.996427059173584,
+      -5.036664009094238,
+      -5.050143718719482,
+      -4.510994911193848,
+      -4.2226057052612305,
+      -5.094910144805908,
+      -4.5359344482421875,
+      -4.703138828277588,
+      -4.337795257568359,
+      -3.8652913570404053,
+      -3.8839306831359863,
+      -3.3045520782470703,
+      -4.1146416664123535,
+      -3.769718647003174,
+      -3.929957628250122,
+      -3.768799066543579,
+      -3.972208261489868,
+      -3.634816884994507,
+      -3.598145008087158,
+      -3.5497827529907227,
+      -4.0864362716674805,
+      -3.5880346298217773,
+      -4.015559673309326,
+      -3.432640790939331,
+      -3.608142614364624,
+      -3.474095106124878,
+      -3.7869486808776855,
+      -3.5873405933380127,
+      -3.4235167503356934,
+      -3.184415578842163
+    ],
+    "base_loss": [
+      9.474967002868652,
+      7.123448371887207,
+      6.168188571929932,
+      4.6793622970581055,
+      2.6044986248016357,
+      2.013458728790283,
+      1.9103567600250244,
+      2.0000832080841064,
+      1.8899085521697998,
+      1.8412806987762451,
+      1.8483163118362427,
+      1.8022738695144653,
+      1.847564458847046,
+      1.8393151760101318,
+      1.850179672241211,
+      1.8068408966064453,
+      1.824257731437683,
+      1.7777620553970337,
+      1.8727468252182007,
+      1.7854875326156616,
+      1.8274955749511719,
+      1.8095120191574097,
+      1.8368505239486694,
+      1.684551477432251,
+      1.8621140718460083,
+      1.7836496829986572,
+      1.767014741897583,
+      1.770763635635376,
+      1.8025929927825928,
+      1.887298345565796,
+      1.8839874267578125,
+      1.8471672534942627,
+      1.812026858329773,
+      1.891275405883789,
+      1.7659833431243896,
+      1.7723257541656494,
+      1.8089889287948608,
+      1.76226806640625,
+      1.7039188146591187,
+      1.7891939878463745,
+      1.719329833984375,
+      1.7772554159164429,
+      1.7795614004135132,
+      1.6842670440673828,
+      1.6844215393066406,
+      1.7464462518692017,
+      1.7909437417984009,
+      1.7481502294540405,
+      1.661051869392395,
+      1.6622161865234375,
+      1.6147782802581787,
+      1.6070369482040405,
+      1.657061219215393,
+      1.714040756225586,
+      1.6753875017166138,
+      1.6686259508132935,
+      1.6674834489822388,
+      1.664836049079895,
+      1.6701329946517944,
+      1.7454251050949097,
+      1.6487113237380981,
+      1.6333528757095337,
+      1.657004714012146,
+      1.6842231750488281,
+      1.6459062099456787,
+      1.638092279434204,
+      1.6258095502853394,
+      1.6174780130386353,
+      1.6530956029891968,
+      1.6498746871948242,
+      1.6689900159835815,
+      1.57684326171875,
+      1.6067333221435547,
+      1.6402298212051392,
+      1.5982598066329956,
+      1.5155116319656372,
+      1.692285180091858,
+      1.6260746717453003,
+      1.6221132278442383,
+      1.6246081590652466,
+      1.6117967367172241,
+      1.6194757223129272,
+      1.598102331161499,
+      1.6332045793533325,
+      1.5825088024139404,
+      1.5107394456863403,
+      1.602547526359558,
+      1.6012418270111084,
+      1.6358078718185425,
+      1.532452940940857,
+      1.5894438028335571,
+      1.553748369216919,
+      1.488343596458435,
+      1.5165965557098389,
+      1.4477319717407227,
+      1.536694884300232,
+      1.5032269954681396,
+      1.5219676494598389,
+      1.5278981924057007,
+      1.496490716934204,
+      1.4675737619400024,
+      1.4852674007415771,
+      1.3362356424331665,
+      1.4154537916183472,
+      1.444419026374817,
+      1.3438326120376587,
+      1.3477458953857422,
+      1.24502432346344,
+      1.1895085573196411,
+      1.0301271677017212,
+      1.0721133947372437,
+      0.9904470443725586,
+      0.8669577836990356,
+      0.8616147637367249,
+      0.8164688944816589,
+      0.7701811194419861,
+      0.7990650534629822,
+      0.7189332246780396,
+      0.6271408796310425,
+      0.658858597278595,
+      0.6362829208374023,
+      0.6195059418678284,
+      0.6116113662719727,
+      0.5777509808540344,
+      0.5383127331733704,
+      0.6213598251342773,
+      0.5587041974067688,
+      0.5645622611045837,
+      0.5378245115280151,
+      0.5126824378967285,
+      0.48708105087280273,
+      0.45352664589881897,
+      0.4964592754840851,
+      0.4707101881504059,
+      0.4800238609313965,
+      0.48176077008247375,
+      0.49011462926864624,
+      0.4257812798023224,
+      0.4405454993247986,
+      0.45625683665275574,
+      0.5016933083534241,
+      0.4303135275840759,
+      0.46569007635116577,
+      0.4130376875400543,
+      0.4513709545135498,
+      0.42867225408554077,
+      0.44723838567733765,
+      0.4232305884361267,
+      0.41526561975479126,
+      0.3808412551879883
+    ],
+    "info_loss": [
+      -0.20977783203125,
+      -0.2442469596862793,
+      -0.26015424728393555,
+      -0.29062795639038086,
+      -0.148298978805542,
+      -0.09887826442718506,
+      -0.09743881225585938,
+      -0.13865768909454346,
+      -0.11879169940948486,
+      -0.13874804973602295,
+      -0.13430380821228027,
+      -0.15891695022583008,
+      -0.17007505893707275,
+      -0.27056288719177246,
+      -0.2535163164138794,
+      -0.24606919288635254,
+      -0.2501150369644165,
+      -0.21087312698364258,
+      -0.22212731838226318,
+      -0.2465611696243286,
+      -0.24265611171722412,
+      -0.23396706581115723,
+      -0.21633541584014893,
+      -0.22790491580963135,
+      -0.23808276653289795,
+      -0.27501964569091797,
+      -0.2901705503463745,
+      -0.32492220401763916,
+      -0.4848982095718384,
+      -0.6613947153091431,
+      -0.9592947363853455,
+      -1.1417292356491089,
+      -1.1531314849853516,
+      -1.2465033531188965,
+      -1.2249419689178467,
+      -1.2093983888626099,
+      -1.342968225479126,
+      -1.3036762475967407,
+      -1.2695143222808838,
+      -1.3967969417572021,
+      -1.3202941417694092,
+      -1.3677679300308228,
+      -1.4199516773223877,
+      -1.3747591972351074,
+      -1.327619194984436,
+      -1.4063856601715088,
+      -1.4757684469223022,
+      -1.4670329093933105,
+      -1.3743075132369995,
+      -1.3745038509368896,
+      -1.3793561458587646,
+      -1.3434416055679321,
+      -1.3401577472686768,
+      -1.4780397415161133,
+      -1.4364835023880005,
+      -1.403468132019043,
+      -1.3874809741973877,
+      -1.3744466304779053,
+      -1.406492829322815,
+      -1.4912915229797363,
+      -1.4426169395446777,
+      -1.4530000686645508,
+      -1.4468318223953247,
+      -1.4860813617706299,
+      -1.470852017402649,
+      -1.4778376817703247,
+      -1.3942036628723145,
+      -1.4087891578674316,
+      -1.461976408958435,
+      -1.4693336486816406,
+      -1.470320701599121,
+      -1.376815676689148,
+      -1.3944391012191772,
+      -1.4354804754257202,
+      -1.4221135377883911,
+      -1.3215875625610352,
+      -1.5468448400497437,
+      -1.4476895332336426,
+      -1.453835368156433,
+      -1.4539093971252441,
+      -1.4728425741195679,
+      -1.4641965627670288,
+      -1.4120572805404663,
+      -1.5112266540527344,
+      -1.3862770795822144,
+      -1.348540186882019,
+      -1.4900604486465454,
+      -1.4927648305892944,
+      -1.5019117593765259,
+      -1.3912949562072754,
+      -1.4404125213623047,
+      -1.4496182203292847,
+      -1.3697378635406494,
+      -1.3870716094970703,
+      -1.3299174308776855,
+      -1.4373135566711426,
+      -1.4011181592941284,
+      -1.3986619710922241,
+      -1.3821871280670166,
+      -1.3670028448104858,
+      -1.3672733306884766,
+      -1.3753691911697388,
+      -1.2679178714752197,
+      -1.3628404140472412,
+      -1.3553755283355713,
+      -1.2510477304458618,
+      -1.292446494102478,
+      -1.1589775085449219,
+      -1.1088510751724243,
+      -0.9512673616409302,
+      -1.0063751935958862,
+      -0.9080485701560974,
+      -0.7877959609031677,
+      -0.7959626913070679,
+      -0.7321491837501526,
+      -0.7169923782348633,
+      -0.7309204936027527,
+      -0.6458167433738708,
+      -0.5527870655059814,
+      -0.5714083909988403,
+      -0.5652658343315125,
+      -0.5687893033027649,
+      -0.5684319138526917,
+      -0.5115469694137573,
+      -0.47789427638053894,
+      -0.5751711130142212,
+      -0.510647177696228,
+      -0.5287784337997437,
+      -0.4918251633644104,
+      -0.4407775402069092,
+      -0.4399414360523224,
+      -0.3773854374885559,
+      -0.4628640413284302,
+      -0.4261557459831238,
+      -0.4435183107852936,
+      -0.4269271194934845,
+      -0.4480868875980377,
+      -0.4070889949798584,
+      -0.4065954089164734,
+      -0.4023192524909973,
+      -0.4620068371295929,
+      -0.40317302942276,
+      -0.4504946768283844,
+      -0.3861498236656189,
+      -0.4086449146270752,
+      -0.39321067929267883,
+      -0.4254463016986847,
+      -0.40254276990890503,
+      -0.3855791985988617,
+      -0.35843029618263245
+    ],
+    "abs_loss": [
+      2.294851779937744,
+      2.217968463897705,
+      2.085430383682251,
+      1.933550477027893,
+      1.8507790565490723,
+      1.8573724031448364,
+      1.8490761518478394,
+      1.8258271217346191,
+      1.7650401592254639,
+      1.8163961172103882,
+      1.7979645729064941,
+      1.81875479221344,
+      1.8053371906280518,
+      1.8457976579666138,
+      1.8368562459945679,
+      1.7651417255401611,
+      1.7947784662246704,
+      1.8123019933700562,
+      1.75529146194458,
+      1.7103999853134155,
+      1.665234923362732,
+      1.4871249198913574,
+      1.1473619937896729,
+      0.9188952445983887,
+      0.835640549659729,
+      0.7401660680770874,
+      0.6749497652053833,
+      0.623252272605896,
+      0.6291333436965942,
+      0.5610553026199341,
+      0.6342823505401611,
+      0.5864875316619873,
+      0.5124613642692566,
+      0.45354872941970825,
+      0.505251407623291,
+      0.3990432024002075,
+      0.3562731146812439,
+      0.3264189660549164,
+      0.34121859073638916,
+      0.29397594928741455,
+      0.2657366991043091,
+      0.28916534781455994,
+      0.28594478964805603,
+      0.2538009285926819,
+      0.23509228229522705,
+      0.18661481142044067,
+      0.24677905440330505,
+      0.17556226253509521,
+      0.23916086554527283,
+      0.15728092193603516,
+      0.18109017610549927,
+      0.1437540054321289,
+      0.1634223461151123,
+      0.19285456836223602,
+      0.13049659132957458,
+      0.14706766605377197,
+      0.11795340478420258,
+      0.12607064843177795,
+      0.15534041821956635,
+      0.16451779007911682,
+      0.1086958646774292,
+      0.14760926365852356,
+      0.11080951988697052,
+      0.1084408164024353,
+      0.1662750542163849,
+      0.09962759166955948,
+      0.0982552170753479,
+      0.16848281025886536,
+      0.13386185467243195,
+      0.06918559968471527,
+      0.06314872950315475,
+      0.07399706542491913,
+      0.08023437857627869,
+      0.06503905355930328,
+      0.06501073390245438,
+      0.0373649075627327,
+      0.07079947739839554,
+      0.06553753465414047,
+      0.04528520256280899,
+      0.05383242666721344,
+      0.06707749515771866,
+      0.06644883751869202,
+      0.05952143669128418,
+      0.04839048162102699,
+      0.05015701800584793,
+      0.035811927169561386,
+      0.03779171034693718,
+      0.04513540118932724,
+      0.06980851292610168,
+      0.053471483290195465,
+      0.027496855705976486,
+      0.025432242080569267,
+      0.046835899353027344,
+      0.04602156579494476,
+      0.049780599772930145,
+      0.04161358252167702,
+      0.0522635355591774,
+      0.043539658188819885,
+      0.02877829782664776,
+      0.03934263810515404,
+      0.03931758925318718,
+      0.03406631201505661,
+      0.03732343763113022,
+      0.03033389523625374,
+      0.02612752467393875,
+      0.017829855903983116,
+      0.051261644810438156,
+      0.017979038879275322,
+      0.020870964974164963,
+      0.03181803226470947,
+      0.027377821505069733,
+      0.04135163500905037,
+      0.04830572381615639,
+      0.025115353986620903,
+      0.05764007568359375,
+      0.024219660088419914,
+      0.016297604888677597,
+      0.04372173175215721,
+      0.020716790109872818,
+      0.04647689312696457,
+      0.017766781151294708,
+      0.035948872566223145,
+      0.03585080802440643,
+      0.02679501473903656,
+      0.020538482815027237,
+      0.023500392213463783,
+      0.020186439156532288,
+      0.019431833177804947,
+      0.03605831041932106,
+      0.03780654817819595,
+      0.038680173456668854,
+      0.03160756081342697,
+      0.026255251839756966,
+      0.017248544842004776,
+      0.026952730491757393,
+      0.02061101794242859,
+      0.03785064071416855,
+      0.023413699120283127,
+      0.022951096296310425,
+      0.023105671629309654,
+      0.020348209887742996,
+      0.01708866097033024,
+      0.04392319917678833,
+      0.03585944324731827,
+      0.02257593162357807,
+      0.02412750944495201,
+      0.022317443042993546,
+      0.013668009079992771,
+      0.031480975449085236,
+      0.025552885606884956
+    ],
+    "zipf_loss": [
+      8.771808624267578,
+      6.307893753051758,
+      3.720466136932373,
+      2.4225094318389893,
+      1.8894931077957153,
+      1.7582186460494995,
+      1.7210813760757446,
+      1.709200382232666,
+      1.7017463445663452,
+      1.6964911222457886,
+      1.697059988975525,
+      1.6959459781646729,
+      1.6910948753356934,
+      1.6887447834014893,
+      1.6905392408370972,
+      1.6872717142105103,
+      1.688768982887268,
+      1.67805016040802,
+      1.666795253753662,
+      1.6262922286987305,
+      1.5347145795822144,
+      1.278693675994873,
+      0.6891258358955383,
+      0.3737459182739258,
+      0.18118418753147125,
+      0.15129734575748444,
+      0.10418544709682465,
+      0.1074758768081665,
+      0.08676093816757202,
+      0.07139623165130615,
+      0.10783088207244873,
+      0.10926471650600433,
+      0.06382100284099579,
+      0.0593724325299263,
+      0.05308477208018303,
+      0.045744385570287704,
+      0.05352798104286194,
+      0.041822511702775955,
+      0.039745353162288666,
+      0.03549409657716751,
+      0.04395335540175438,
+      0.035772405564785004,
+      0.02904796227812767,
+      0.035995081067085266,
+      0.038269124925136566,
+      0.03648350387811661,
+      0.02863462083041668,
+      0.035054247826337814,
+      0.026261046528816223,
+      0.02930118702352047,
+      0.03150211647152901,
+      0.024524109438061714,
+      0.02761816792190075,
+      0.02726607210934162,
+      0.03294641897082329,
+      0.03868941217660904,
+      0.023010730743408203,
+      0.022195344790816307,
+      0.024301059544086456,
+      0.02068120427429676,
+      0.020041795447468758,
+      0.029191507026553154,
+      0.025066332891583443,
+      0.019122403115034103,
+      0.023434175178408623,
+      0.027309784665703773,
+      0.019048001617193222,
+      0.017169244587421417,
+      0.018393322825431824,
+      0.028059540316462517,
+      0.028056636452674866,
+      0.028341447934508324,
+      0.02032645046710968,
+      0.013149574398994446,
+      0.01588503085076809,
+      0.012552021071314812,
+      0.01579051837325096,
+      0.021106014028191566,
+      0.04947477951645851,
+      0.01698216237127781,
+      0.022098727524280548,
+      0.01268475130200386,
+      0.022358544170856476,
+      0.01958904229104519,
+      0.010487602092325687,
+      0.01751772314310074,
+      0.023311246186494827,
+      0.01734558492898941,
+      0.015464305877685547,
+      0.01562531292438507,
+      0.012672880664467812,
+      0.018038703128695488,
+      0.023711949586868286,
+      0.01683882251381874,
+      0.014847498387098312,
+      0.026351360604166985,
+      0.021378500387072563,
+      0.018875695765018463,
+      0.019985806196928024,
+      0.01564783975481987,
+      0.02189333736896515,
+      0.030228938907384872,
+      0.01130809634923935,
+      0.014759158715605736,
+      0.026083514094352722,
+      0.012203274294734001,
+      0.020920155569911003,
+      0.017358364537358284,
+      0.021204819902777672,
+      0.02353861555457115,
+      0.02059362642467022,
+      0.02084406465291977,
+      0.022354595363140106,
+      0.024830598384141922,
+      0.02938147634267807,
+      0.03147149458527565,
+      0.029878588393330574,
+      0.013422621414065361,
+      0.014764735475182533,
+      0.012709204107522964,
+      0.01817169226706028,
+      0.028127998113632202,
+      0.018979191780090332,
+      0.024044431746006012,
+      0.015970000997185707,
+      0.03309129178524017,
+      0.009814741089940071,
+      0.018140148371458054,
+      0.03902601823210716,
+      0.026020988821983337,
+      0.024534741416573524,
+      0.012614620849490166,
+      0.014913609251379967,
+      0.019403688609600067,
+      0.022506240755319595,
+      0.016650553792715073,
+      0.014760621823370457,
+      0.007950497791171074,
+      0.02496860735118389,
+      0.014842228963971138,
+      0.02990444004535675,
+      0.011673047207295895,
+      0.0193045511841774,
+      0.012233665212988853,
+      0.02467804029583931,
+      0.026926595717668533,
+      0.018044285476207733,
+      0.013489766046404839,
+      0.013861648738384247,
+      0.016490790992975235
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.6935886740684509,
+      0.48234307765960693,
+      0.3088524043560028,
+      0.20281840860843658,
+      0.15621885657310486,
+      0.10378851741552353,
+      0.07804737240076065,
+      0.06271085143089294,
+      0.05669274926185608,
+      0.05394528806209564,
+      0.05465354025363922,
+      0.05041395500302315,
+      0.051717281341552734,
+      0.049015142023563385,
+      0.049808476120233536,
+      0.05488650128245354,
+      0.05550508573651314,
+      0.05550394207239151,
+      0.05379695072770119,
+      0.0546729750931263,
+      0.05207265913486481,
+      0.05092643201351166,
+      0.05123722553253174,
+      0.051657721400260925,
+      0.047756366431713104,
+      0.0484192855656147,
+      0.048870690166950226,
+      0.047017455101013184,
+      0.047359365969896317,
+      0.05155592039227486,
+      0.05629339441657066,
+      0.059633731842041016,
+      0.06442190706729889,
+      0.06989780068397522,
+      0.07373283058404922,
+      0.07644601911306381,
+      0.08206634968519211,
+      0.08509311079978943,
+      0.08663709461688995,
+      0.08705299347639084,
+      0.08799070119857788,
+      0.0882645770907402,
+      0.08959797769784927,
+      0.09180758148431778,
+      0.09077784419059753,
+      0.09113264083862305,
+      0.08888813853263855,
+      0.09106779098510742,
+      0.09238111227750778,
+      0.09107761830091476,
+      0.08944051712751389,
+      0.08877777308225632,
+      0.08691896498203278,
+      0.09018930047750473,
+      0.08876080811023712,
+      0.08874350786209106,
+      0.09306778758764267,
+      0.09424176067113876,
+      0.09464000165462494,
+      0.0924631878733635,
+      0.09248515218496323,
+      0.0956466943025589,
+      0.09548068791627884,
+      0.09362413734197617,
+      0.09517014771699905,
+      0.09995341300964355,
+      0.10262968391180038,
+      0.10635028779506683,
+      0.10774730145931244,
+      0.11003488302230835,
+      0.11236182600259781,
+      0.11327866464853287,
+      0.11391086876392365,
+      0.11230804026126862,
+      0.11398068070411682,
+      0.11666402220726013,
+      0.12519533932209015,
+      0.12921561300754547,
+      0.1298656314611435,
+      0.13479764759540558,
+      0.14023935794830322,
+      0.14109951257705688,
+      0.14126062393188477,
+      0.143479585647583,
+      0.14300858974456787,
+      0.1485244631767273,
+      0.1472393125295639,
+      0.14510539174079895,
+      0.14728957414627075,
+      0.15059445798397064,
+      0.15292133390903473,
+      0.15796807408332825,
+      0.15788158774375916,
+      0.15734679996967316,
+      0.15908081829547882,
+      0.16669081151485443,
+      0.17414961755275726,
+      0.18038775026798248,
+      0.17948652803897858,
+      0.1796618551015854,
+      0.17886602878570557,
+      0.17792661488056183,
+      0.18056391179561615,
+      0.1810716837644577,
+      0.19002854824066162,
+      0.1949450820684433,
+      0.20069141685962677,
+      0.1973830759525299,
+      0.2024090737104416,
+      0.20541264116764069,
+      0.2075474113225937,
+      0.21202749013900757,
+      0.21167878806591034,
+      0.2112017720937729,
+      0.21490934491157532,
+      0.21238234639167786,
+      0.21160206198692322,
+      0.21397870779037476,
+      0.21555015444755554,
+      0.2155567705631256,
+      0.2197999805212021,
+      0.223252072930336,
+      0.2264471799135208,
+      0.22776155173778534,
+      0.2280261218547821,
+      0.2283269762992859,
+      0.2301325798034668,
+      0.22769907116889954,
+      0.22877106070518494,
+      0.23092709481716156,
+      0.22949814796447754,
+      0.22870399057865143,
+      0.2302009016275406,
+      0.23301368951797485,
+      0.23054614663124084,
+      0.23147828876972198,
+      0.2322801649570465,
+      0.23291891813278198,
+      0.23302015662193298,
+      0.23411573469638824,
+      0.23496660590171814,
+      0.2342650145292282,
+      0.23444189131259918,
+      0.23619861900806427,
+      0.23654569685459137,
+      0.23704758286476135,
+      0.2376752644777298,
+      0.23932912945747375,
+      0.23906366527080536,
+      0.23852844536304474
+    ],
+    "lr": [
+      8.376068376068378e-06,
+      1.6923076923076924e-05,
+      2.5470085470085475e-05,
+      3.401709401709402e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.993593461639863e-05,
+      3.934273662008964e-05,
+      3.874953862378065e-05,
+      3.815634062747166e-05,
+      3.7563142631162665e-05,
+      3.696994463485368e-05,
+      3.637674663854469e-05,
+      3.5783548642235693e-05,
+      3.519035064592671e-05,
+      3.459715264961771e-05,
+      3.400395465330873e-05,
+      3.341075665699973e-05,
+      3.281755866069075e-05,
+      3.2224360664381764e-05,
+      3.1251515950435014e-05,
+      3.065831795412602e-05,
+      3.0065119957817037e-05,
+      2.947192196150804e-05,
+      2.887872396519905e-05,
+      2.8285525968890065e-05,
+      2.769232797258107e-05,
+      2.709912997627208e-05,
+      2.6505931979963087e-05,
+      2.5912733983654104e-05,
+      2.5319535987345117e-05,
+      2.472633799103612e-05,
+      2.4133139994727132e-05,
+      2.3539941998418135e-05,
+      2.2946744002109148e-05,
+      2.1973899288162408e-05,
+      2.138070129185341e-05,
+      2.0787503295544424e-05,
+      2.019430529923544e-05,
+      1.9601107302926443e-05,
+      1.900790930661746e-05,
+      1.8414711310308462e-05,
+      1.7821513313999475e-05,
+      1.7228315317690488e-05,
+      1.663511732138149e-05,
+      1.6041919325072507e-05,
+      1.544872132876351e-05,
+      1.4855523332454524e-05,
+      1.4262325336145537e-05,
+      1.366912733983654e-05,
+      1.2696282625889797e-05,
+      1.2103084629580812e-05,
+      1.1509886633271816e-05,
+      1.0916688636962829e-05,
+      1.0323490640653833e-05,
+      9.730292644344846e-06,
+      9.137094648035861e-06,
+      8.543896651726864e-06,
+      7.950698655417878e-06,
+      7.357500659108881e-06,
+      6.764302662799895e-06,
+      6.171104666490909e-06,
+      5.577906670181913e-06,
+      4.984708673872927e-06,
+      4.391510677563931e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      750,
+      1532,
+      2314,
+      3096,
+      3878,
+      4660,
+      5442,
+      6224,
+      7006,
+      7788
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.01,
+      0.2,
+      0.5,
+      0.57,
+      0.69,
+      0.74,
+      0.83,
+      0.86,
+      0.9
+    ]
+  },
+  "final_accuracy": 0.8245833333333333,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.18,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8066115702479338,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.8947368421052632,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.22,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8529411764705882,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.8816568047337278,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.9032258064516129,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.7466216216216216,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.17,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8404907975460123,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.735632183908046,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.6798029556650246,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.7948717948717948,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.19,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9090909090909091,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.859504132231405,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.7346938775510204,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7043010752688172,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.7354260089686099,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.32,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9326923076923077,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.8867924528301887,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.7391304347826086,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.7625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.6742671009771987,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.3,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.86,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.93,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.66,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.5175,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.54,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.77,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.718,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.205,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.843400447427293,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.8875,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.8035714285714286,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.7580340264650284,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.6458333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.14,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.88,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.93,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.5958549222797928,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.6261682242990654,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.16,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.92,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.95,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.59765625,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.6805555555555556,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.08,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.95,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.93,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6045751633986928,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.7268041237113402,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.18,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.94,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.73224043715847,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.8034188034188035,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.48,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9068219633943427,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.898989898989899,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.23,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9032258064516129,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.903448275862069,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9166666666666666,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.7182539682539683,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.11,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9436619718309859,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.831858407079646,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9058823529411765,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.5524861878453039,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.6944444444444444,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.09,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9720670391061452,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.8543689320388349,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9464285714285714,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.4697986577181208,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.48826291079812206,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.04,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.955,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.87,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.48,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.33666666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.13,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.59,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.3675,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.29,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.915,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.8801498127340824,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9622641509433962,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.6993166287015945,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.8048780487804879,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.11,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.92,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.95,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.5380710659898477,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.5242718446601942,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.11,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.975,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.93,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.5546558704453441,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.5359477124183006,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.92,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.5234899328859061,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.504950495049505,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.19958333333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 4,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9950980392156863,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9940828402366864,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9938650306748467,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9923076923076923,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.9,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.9516129032258065,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.9955156950672646,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.77,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.86875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.993485342019544,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.68,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.96,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.89,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.89,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.97,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.99,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.996219281663516,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9637305699481865,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.91,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.96484375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9771241830065359,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.9948453608247423,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9808743169398907,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966722129783694,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.996415770609319,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.993103448275862,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.94,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9953051643192489,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.9723756906077348,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.58,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.7315436241610739,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.9765258215962441,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.27,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.56,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.7933333333333333,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.53,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.57,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.965,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9933333333333333,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9931662870159453,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.8,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.99,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9187817258883249,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.9805825242718447,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.73,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9230769230769231,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.9281045751633987,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.65,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9026845637583892,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.9108910891089109,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.8245833333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.8245833333333333,
+  "sft_overall_accuracy": 0.19958333333333333
+}

add_sub_sorl_v1_abs10_50K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb609285c5151c0d0634d5d2b7a3d5ff7ec57d8acf496116f999081eba9e3749
+size 650303660

add_sub_sorl_v1_abs10_50K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 234,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 781,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs10_K4_50K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 10,
+  "dataset_size": 50000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162499262,
+  "run_name": "add_sub_sorl_v1_abs10_50K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T19:31:11.043883+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "ajnrconh",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/ajnrconh",
+  "final_accuracy": 0.8245833333333333,
+  "sft_accuracy": 0.19958333333333333,
+  "eval_method": "ArithmeticEvaluator"
+}