Upload add_sub_sorl_v1_abs30_50K

add_sub_sorl_v1_abs30_50K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151674
+}

add_sub_sorl_v1_abs30_50K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs30_50K/metrics.json

@@ -0,0 +1,2257 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      832,
+      882,
+      932,
+      982,
+      1032,
+      1082,
+      1132,
+      1182,
+      1232,
+      1282,
+      1332,
+      1382,
+      1432,
+      1482,
+      1532,
+      1614,
+      1664,
+      1714,
+      1764,
+      1814,
+      1864,
+      1914,
+      1964,
+      2014,
+      2064,
+      2114,
+      2164,
+      2214,
+      2264,
+      2314,
+      2396,
+      2446,
+      2496,
+      2546,
+      2596,
+      2646,
+      2696,
+      2746,
+      2796,
+      2846,
+      2896,
+      2946,
+      2996,
+      3046,
+      3096,
+      3178,
+      3228,
+      3278,
+      3328,
+      3378,
+      3428,
+      3478,
+      3528,
+      3578,
+      3628,
+      3678,
+      3728,
+      3778,
+      3828,
+      3878,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4310,
+      4360,
+      4410,
+      4460,
+      4510,
+      4560,
+      4610,
+      4660,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5092,
+      5142,
+      5192,
+      5242,
+      5292,
+      5342,
+      5392,
+      5442,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5874,
+      5924,
+      5974,
+      6024,
+      6074,
+      6124,
+      6174,
+      6224,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6656,
+      6706,
+      6756,
+      6806,
+      6856,
+      6906,
+      6956,
+      7006,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7438,
+      7488,
+      7538,
+      7588,
+      7638,
+      7688,
+      7738,
+      7788
+    ],
+    "loss": [
+      14.925691604614258,
+      10.310871124267578,
+      7.15638542175293,
+      4.645656108856201,
+      2.783916711807251,
+      2.4809165000915527,
+      2.7336008548736572,
+      2.3534011840820312,
+      2.153804302215576,
+      1.4827725887298584,
+      1.6645079851150513,
+      1.3439991474151611,
+      1.3138333559036255,
+      0.969010055065155,
+      0.8032979965209961,
+      0.020356476306915283,
+      -1.0327482223510742,
+      -2.2422666549682617,
+      -4.954840660095215,
+      -6.268737316131592,
+      -7.6543707847595215,
+      -8.756476402282715,
+      -10.073406219482422,
+      -9.491241455078125,
+      -10.407691955566406,
+      -10.537861824035645,
+      -11.763997077941895,
+      -11.57911205291748,
+      -12.794604301452637,
+      -12.53265380859375,
+      -13.207358360290527,
+      -13.6425142288208,
+      -12.904062271118164,
+      -14.24144458770752,
+      -13.814043998718262,
+      -13.221015930175781,
+      -13.979741096496582,
+      -13.792145729064941,
+      -13.058844566345215,
+      -14.35586166381836,
+      -14.468194007873535,
+      -14.658733367919922,
+      -14.305359840393066,
+      -13.779980659484863,
+      -13.826972007751465,
+      -14.544889450073242,
+      -14.795659065246582,
+      -14.45028018951416,
+      -13.597357749938965,
+      -13.89336109161377,
+      -13.701704025268555,
+      -13.72293472290039,
+      -13.997247695922852,
+      -14.357726097106934,
+      -14.675873756408691,
+      -14.496241569519043,
+      -13.631208419799805,
+      -14.704141616821289,
+      -14.385679244995117,
+      -14.757608413696289,
+      -14.189804077148438,
+      -14.076844215393066,
+      -14.364145278930664,
+      -14.118315696716309,
+      -14.320837020874023,
+      -14.189194679260254,
+      -13.755496978759766,
+      -14.0785493850708,
+      -14.141383171081543,
+      -13.994441986083984,
+      -14.23997974395752,
+      -13.254907608032227,
+      -13.587363243103027,
+      -13.777580261230469,
+      -13.173924446105957,
+      -11.22693157196045,
+      -10.27431869506836,
+      -7.386316776275635,
+      -6.493227481842041,
+      -5.810792922973633,
+      -5.425526142120361,
+      -5.403774738311768,
+      -5.274623870849609,
+      -5.1203083992004395,
+      -5.120668411254883,
+      -4.660671234130859,
+      -4.481213092803955,
+      -4.476528644561768,
+      -4.4041571617126465,
+      -4.332465648651123,
+      -4.374258995056152,
+      -4.8543548583984375,
+      -3.9383552074432373,
+      -3.6458797454833984,
+      -3.5874781608581543,
+      -4.045778274536133,
+      -3.5161209106445312,
+      -3.6460752487182617,
+      -3.4689207077026367,
+      -3.659513473510742,
+      -3.5809338092803955,
+      -3.4906952381134033,
+      -3.3742854595184326,
+      -3.003173589706421,
+      -3.4980430603027344,
+      -3.279160499572754,
+      -2.6153182983398438,
+      -3.0391597747802734,
+      -2.8614397048950195,
+      -3.2860047817230225,
+      -3.176285743713379,
+      -2.8159425258636475,
+      -2.7563352584838867,
+      -2.9784791469573975,
+      -2.917531728744507,
+      -2.713247537612915,
+      -2.519273281097412,
+      -2.386897325515747,
+      -2.3819081783294678,
+      -2.354832649230957,
+      -2.129584550857544,
+      -2.350475788116455,
+      -2.5259969234466553,
+      -2.2565855979919434,
+      -2.326547384262085,
+      -2.5057482719421387,
+      -1.9806913137435913,
+      -1.929569125175476,
+      -2.4524779319763184,
+      -2.075390338897705,
+      -1.8700034618377686,
+      -1.989833116531372,
+      -2.270037889480591,
+      -1.87515127658844,
+      -1.9000284671783447,
+      -1.84731924533844,
+      -1.946153163909912,
+      -1.4869354963302612,
+      -1.890562891960144,
+      -1.7415295839309692,
+      -1.9608829021453857,
+      -1.5101802349090576,
+      -1.5716921091079712,
+      -1.6069391965866089,
+      -1.429286241531372,
+      -1.6014431715011597,
+      -1.6325364112854004,
+      -1.3494127988815308,
+      -1.460780382156372,
+      -1.367804765701294
+    ],
+    "base_loss": [
+      9.473811149597168,
+      7.116661071777344,
+      6.11044979095459,
+      4.431495189666748,
+      2.3752102851867676,
+      2.003913640975952,
+      1.8888157606124878,
+      1.960724115371704,
+      1.8655742406845093,
+      1.85293710231781,
+      1.8393819332122803,
+      1.7841354608535767,
+      1.8464338779449463,
+      1.8436554670333862,
+      1.8355497121810913,
+      1.8096510171890259,
+      1.8457410335540771,
+      1.8437222242355347,
+      2.0343832969665527,
+      1.8724702596664429,
+      1.8979225158691406,
+      1.8813482522964478,
+      1.9130842685699463,
+      1.6835806369781494,
+      1.7973169088363647,
+      1.8007093667984009,
+      1.7375422716140747,
+      1.7017974853515625,
+      1.7410304546356201,
+      1.7181750535964966,
+      1.7001968622207642,
+      1.7098767757415771,
+      1.6393336057662964,
+      1.7414904832839966,
+      1.6889947652816772,
+      1.6481058597564697,
+      1.6936885118484497,
+      1.6631056070327759,
+      1.587760329246521,
+      1.6887438297271729,
+      1.6885937452316284,
+      1.7139508724212646,
+      1.6895262002944946,
+      1.6100366115570068,
+      1.6152337789535522,
+      1.6766698360443115,
+      1.7149112224578857,
+      1.6795676946640015,
+      1.6007461547851562,
+      1.6156295537948608,
+      1.5643459558486938,
+      1.551654577255249,
+      1.5922502279281616,
+      1.6294152736663818,
+      1.663206934928894,
+      1.6547516584396362,
+      1.560704231262207,
+      1.6592508554458618,
+      1.6300275325775146,
+      1.6693183183670044,
+      1.593511700630188,
+      1.5905581712722778,
+      1.6223725080490112,
+      1.6157256364822388,
+      1.6122468709945679,
+      1.6028401851654053,
+      1.56356942653656,
+      1.5920432806015015,
+      1.5996407270431519,
+      1.5716848373413086,
+      1.5974481105804443,
+      1.5151593685150146,
+      1.5354450941085815,
+      1.5449351072311401,
+      1.4808731079101562,
+      1.261785626411438,
+      1.1568143367767334,
+      0.8340187072753906,
+      0.7368689775466919,
+      0.6682016253471375,
+      0.6186455488204956,
+      0.6124207377433777,
+      0.5980902910232544,
+      0.59001624584198,
+      0.5819413065910339,
+      0.5578277707099915,
+      0.5163649320602417,
+      0.5072833299636841,
+      0.49900832772254944,
+      0.4926478862762451,
+      0.4992688000202179,
+      0.5497917532920837,
+      0.46581941843032837,
+      0.441849023103714,
+      0.42151254415512085,
+      0.476866751909256,
+      0.40012720227241516,
+      0.42673566937446594,
+      0.4007093906402588,
+      0.4211159646511078,
+      0.4143979847431183,
+      0.3965285122394562,
+      0.3838675916194916,
+      0.3422652781009674,
+      0.39702606201171875,
+      0.3723902702331543,
+      0.30194923281669617,
+      0.34926849603652954,
+      0.3268876075744629,
+      0.37237080931663513,
+      0.36258235573768616,
+      0.32173871994018555,
+      0.3123743236064911,
+      0.3381679952144623,
+      0.33103036880493164,
+      0.30924150347709656,
+      0.28604206442832947,
+      0.27141666412353516,
+      0.26933255791664124,
+      0.2693643271923065,
+      0.25377893447875977,
+      0.2691524624824524,
+      0.2878199517726898,
+      0.25663235783576965,
+      0.27092984318733215,
+      0.28533241152763367,
+      0.22697682678699493,
+      0.21974924206733704,
+      0.2805941104888916,
+      0.24086768925189972,
+      0.2150326520204544,
+      0.22676725685596466,
+      0.2578849494457245,
+      0.21510757505893707,
+      0.21743905544281006,
+      0.2111271172761917,
+      0.22255614399909973,
+      0.17357924580574036,
+      0.2218475341796875,
+      0.20064163208007812,
+      0.22746801376342773,
+      0.1737966537475586,
+      0.1820410192012787,
+      0.183633491396904,
+      0.16445359587669373,
+      0.18439149856567383,
+      0.1871500015258789,
+      0.15578648447990417,
+      0.17073509097099304,
+      0.15720374882221222
+    ],
+    "info_loss": [
+      -0.14559650421142578,
+      -0.17780828475952148,
+      -0.1878204345703125,
+      -0.14971208572387695,
+      -0.08238768577575684,
+      -0.06698238849639893,
+      -0.02822732925415039,
+      -0.07236886024475098,
+      -0.08202946186065674,
+      -0.14824891090393066,
+      -0.12821662425994873,
+      -0.15460705757141113,
+      -0.16380441188812256,
+      -0.19755160808563232,
+      -0.21291589736938477,
+      -0.28692495822906494,
+      -0.3872305154800415,
+      -0.48517298698425293,
+      -0.7449564933776855,
+      -0.8471189737319946,
+      -0.9838017225265503,
+      -1.090334177017212,
+      -1.2246140241622925,
+      -1.1417529582977295,
+      -1.2453727722167969,
+      -1.256455421447754,
+      -1.3754072189331055,
+      -1.350670576095581,
+      -1.4751745462417603,
+      -1.444676399230957,
+      -1.5119096040725708,
+      -1.5553081035614014,
+      -1.4727306365966797,
+      -1.6164841651916504,
+      -1.5695509910583496,
+      -1.505563735961914,
+      -1.5860234498977661,
+      -1.563294768333435,
+      -1.4825890064239502,
+      -1.6203558444976807,
+      -1.6318106651306152,
+      -1.6535658836364746,
+      -1.6152052879333496,
+      -1.554687738418579,
+      -1.558127760887146,
+      -1.6352657079696655,
+      -1.6648855209350586,
+      -1.6253372430801392,
+      -1.5322500467300415,
+      -1.5633134841918945,
+      -1.5416780710220337,
+      -1.5396777391433716,
+      -1.5700269937515259,
+      -1.609108805656433,
+      -1.6445403099060059,
+      -1.6247903108596802,
+      -1.5285240411758423,
+      -1.647214651107788,
+      -1.6125141382217407,
+      -1.653419017791748,
+      -1.5891343355178833,
+      -1.5781067609786987,
+      -1.6078630685806274,
+      -1.5827549695968628,
+      -1.6012533903121948,
+      -1.5873873233795166,
+      -1.539661169052124,
+      -1.5761311054229736,
+      -1.580564260482788,
+      -1.5653799772262573,
+      -1.5943796634674072,
+      -1.4857620000839233,
+      -1.5196541547775269,
+      -1.5394915342330933,
+      -1.472989797592163,
+      -1.257468342781067,
+      -1.1512645483016968,
+      -0.8297007083892822,
+      -0.7310910820960999,
+      -0.656235933303833,
+      -0.6113075017929077,
+      -0.607907772064209,
+      -0.5933750867843628,
+      -0.5776615142822266,
+      -0.5772484540939331,
+      -0.5271150469779968,
+      -0.5060456991195679,
+      -0.5048168897628784,
+      -0.49672001600265503,
+      -0.4906327426433563,
+      -0.49443888664245605,
+      -0.5481942892074585,
+      -0.4475238025188446,
+      -0.4167567789554596,
+      -0.40784752368927,
+      -0.46132850646972656,
+      -0.39877843856811523,
+      -0.41653740406036377,
+      -0.39473432302474976,
+      -0.41418391466140747,
+      -0.4068829417228699,
+      -0.39585503935813904,
+      -0.3816283345222473,
+      -0.3418673276901245,
+      -0.3965541422367096,
+      -0.3707251250743866,
+      -0.29932451248168945,
+      -0.3433356285095215,
+      -0.3257765471935272,
+      -0.37057408690452576,
+      -0.3593398630619049,
+      -0.31936100125312805,
+      -0.31205815076828003,
+      -0.3377631604671478,
+      -0.3308146595954895,
+      -0.30861154198646545,
+      -0.28568384051322937,
+      -0.27113717794418335,
+      -0.2691645920276642,
+      -0.26842546463012695,
+      -0.24299602210521698,
+      -0.26901406049728394,
+      -0.28628090023994446,
+      -0.25644367933273315,
+      -0.2672256827354431,
+      -0.2849532663822174,
+      -0.22515301406383514,
+      -0.2194351702928543,
+      -0.2802630364894867,
+      -0.23897716403007507,
+      -0.2148922085762024,
+      -0.22648395597934723,
+      -0.2577228546142578,
+      -0.21499043703079224,
+      -0.21737247705459595,
+      -0.21103811264038086,
+      -0.2220824509859085,
+      -0.1734149008989334,
+      -0.218211829662323,
+      -0.20041964948177338,
+      -0.2263200283050537,
+      -0.17371030151844025,
+      -0.18098947405815125,
+      -0.18344050645828247,
+      -0.16407114267349243,
+      -0.18422402441501617,
+      -0.18706662952899933,
+      -0.15566548705101013,
+      -0.1706431806087494,
+      -0.15713831782341003
+    ],
+    "abs_loss": [
+      3.39304256439209,
+      3.3440451622009277,
+      3.2221078872680664,
+      2.962374210357666,
+      2.6000261306762695,
+      2.6146888732910156,
+      2.6088027954101562,
+      2.613365411758423,
+      2.5813589096069336,
+      2.5816922187805176,
+      2.5931968688964844,
+      2.561342716217041,
+      2.560413360595703,
+      2.4853343963623047,
+      2.5109357833862305,
+      2.5940842628479004,
+      2.4048895835876465,
+      2.3294265270233154,
+      1.660865068435669,
+      1.3182013034820557,
+      1.1554183959960938,
+      1.1588599681854248,
+      1.1115765571594238,
+      1.0685930252075195,
+      1.1162045001983643,
+      0.9794039130210876,
+      1.186870813369751,
+      1.0021696090698242,
+      0.9881029725074768,
+      0.7798445224761963,
+      1.0104799270629883,
+      0.9486855268478394,
+      0.876146674156189,
+      0.7982321381568909,
+      0.9442781209945679,
+      0.9166160225868225,
+      1.0005611181259155,
+      0.8304280042648315,
+      0.8332905173301697,
+      0.7923551201820374,
+      0.6501508951187134,
+      0.7929084897041321,
+      0.9291647672653198,
+      0.7792672514915466,
+      0.7903759479522705,
+      0.7803529500961304,
+      0.9942322373390198,
+      0.6684384346008301,
+      0.7799667119979858,
+      0.6845371127128601,
+      0.8857107162475586,
+      0.9151805639266968,
+      0.6339882612228394,
+      0.6938873529434204,
+      0.5363549590110779,
+      0.5950627326965332,
+      0.5747663974761963,
+      0.6981031894683838,
+      0.5535870790481567,
+      0.5422572493553162,
+      0.5722933411598206,
+      0.5791301131248474,
+      0.6247014999389648,
+      0.688178539276123,
+      0.4211939573287964,
+      0.4434513449668884,
+      0.4255868196487427,
+      0.5411297082901001,
+      0.41299617290496826,
+      0.4775952696800232,
+      0.3815616965293884,
+      0.4609728157520294,
+      0.5534173250198364,
+      0.40973585844039917,
+      0.5403074026107788,
+      0.34058454632759094,
+      0.4934236407279968,
+      0.43036895990371704,
+      0.4531347453594208,
+      0.5358093976974487,
+      0.39890944957733154,
+      0.3504709303379059,
+      0.4234974682331085,
+      0.33043530583381653,
+      0.3883510231971741,
+      0.3900966942310333,
+      0.3034008741378784,
+      0.3844878077507019,
+      0.3556361198425293,
+      0.32722166180610657,
+      0.2960817515850067,
+      0.3175443112850189,
+      0.36087870597839355,
+      0.44915831089019775,
+      0.3406302332878113,
+      0.27062511444091797,
+      0.4819590449333191,
+      0.2790176272392273,
+      0.3151218295097351,
+      0.3895567059516907,
+      0.30522969365119934,
+      0.3638022840023041,
+      0.3443005084991455,
+      0.30038052797317505,
+      0.3221493363380432,
+      0.27090904116630554,
+      0.3231700658798218,
+      0.2101253718137741,
+      0.2681146264076233,
+      0.363462895154953,
+      0.2364412546157837,
+      0.2628980576992035,
+      0.29717278480529785,
+      0.19711089134216309,
+      0.23752276599407196,
+      0.30123162269592285,
+      0.21607771515846252,
+      0.24906513094902039,
+      0.2326553612947464,
+      0.2548292279243469,
+      0.19183772802352905,
+      0.33658137917518616,
+      0.2623642385005951,
+      0.2028590589761734,
+      0.17227476835250854,
+      0.30687063932418823,
+      0.2223254144191742,
+      0.28637945652008057,
+      0.30192694067955017,
+      0.31335651874542236,
+      0.2542653977870941,
+      0.24489624798297882,
+      0.2581295967102051,
+      0.2891785502433777,
+      0.22562867403030396,
+      0.2942545413970947,
+      0.2457028329372406,
+      0.23869061470031738,
+      0.23281575739383698,
+      0.2988502085208893,
+      0.2741028070449829,
+      0.25956836342811584,
+      0.2576809525489807,
+      0.21786843240261078,
+      0.21318396925926208,
+      0.27870291471481323,
+      0.26582416892051697,
+      0.20977771282196045,
+      0.2978009581565857,
+      0.24936990439891815
+    ],
+    "zipf_loss": [
+      6.568541526794434,
+      4.6378889083862305,
+      2.601929187774658,
+      1.4150443077087402,
+      0.97258061170578,
+      0.8853577375411987,
+      0.866178035736084,
+      0.8550291061401367,
+      0.850388765335083,
+      0.8541553020477295,
+      0.8479726314544678,
+      0.8498000502586365,
+      0.8494022488594055,
+      0.8523372411727905,
+      0.8458136916160583,
+      0.820546567440033,
+      0.7533268928527832,
+      0.5327982902526855,
+      0.2942545413970947,
+      0.19816140830516815,
+      0.17018242180347443,
+      0.14963090419769287,
+      0.14849303662776947,
+      0.13584934175014496,
+      0.1370982527732849,
+      0.12804265320301056,
+      0.13384655117988586,
+      0.1255798637866974,
+      0.11729998141527176,
+      0.11795015633106232,
+      0.11049142479896545,
+      0.10582149028778076,
+      0.09629538655281067,
+      0.10208211839199066,
+      0.09804347902536392,
+      0.09485457837581635,
+      0.08674925565719604,
+      0.09465356171131134,
+      0.09595625847578049,
+      0.07971762865781784,
+      0.09630455076694489,
+      0.08368515968322754,
+      0.06425083428621292,
+      0.07893300801515579,
+      0.060034651309251785,
+      0.05306267365813255,
+      0.0388617068529129,
+      0.05668090283870697,
+      0.046399977058172226,
+      0.055690523236989975,
+      0.06215830147266388,
+      0.030669555068016052,
+      0.04737374186515808,
+      0.03455729782581329,
+      0.052687350660562515,
+      0.037404127418994904,
+      0.035850390791893005,
+      0.03894411772489548,
+      0.05407535284757614,
+      0.053038209676742554,
+      0.0507982037961483,
+      0.05575310066342354,
+      0.029641835018992424,
+      0.02469017170369625,
+      0.037330590188503265,
+      0.03749319911003113,
+      0.034987181425094604,
+      0.03660454601049423,
+      0.02331935241818428,
+      0.0399140790104866,
+      0.06821182370185852,
+      0.04145489260554314,
+      0.018391745164990425,
+      0.031426701694726944,
+      0.021070394665002823,
+      0.05190783739089966,
+      0.03217053785920143,
+      0.0336339958012104,
+      0.035501234233379364,
+      0.029784096404910088,
+      0.029012762010097504,
+      0.027835464105010033,
+      0.018687181174755096,
+      0.03324688971042633,
+      0.03104039654135704,
+      0.013641919940710068,
+      0.03253910318017006,
+      0.02590865269303322,
+      0.028471380472183228,
+      0.048491280525922775,
+      0.041252508759498596,
+      0.04604184627532959,
+      0.03497536480426788,
+      0.03492319583892822,
+      0.03542107716202736,
+      0.06357758492231369,
+      0.023340530693531036,
+      0.06466128677129745,
+      0.04620100557804108,
+      0.022254012525081635,
+      0.04297468811273575,
+      0.03494641184806824,
+      0.023700159043073654,
+      0.04319623485207558,
+      0.038257330656051636,
+      0.028609748929739,
+      0.04366075620055199,
+      0.02391563169658184,
+      0.04262681305408478,
+      0.011019270867109299,
+      0.030886277556419373,
+      0.02963891625404358,
+      0.022154763340950012,
+      0.0412735678255558,
+      0.035832080990076065,
+      0.03350311517715454,
+      0.029915394261479378,
+      0.028151482343673706,
+      0.017139362171292305,
+      0.03457484766840935,
+      0.02741282619535923,
+      0.03685455769300461,
+      0.022755559533834457,
+      0.030933089554309845,
+      0.05755234882235527,
+      0.027764728292822838,
+      0.02162948064506054,
+      0.016395458951592445,
+      0.039365850389003754,
+      0.0421781912446022,
+      0.038459327071905136,
+      0.023749586194753647,
+      0.02349282242357731,
+      0.030727511271834373,
+      0.0336943119764328,
+      0.022509198635816574,
+      0.027544958516955376,
+      0.04976517707109451,
+      0.04642646759748459,
+      0.03214017301797867,
+      0.04743911325931549,
+      0.02716921642422676,
+      0.030393626540899277,
+      0.02204555831849575,
+      0.025653110817074776,
+      0.028535209596157074,
+      0.024397503584623337,
+      0.030477900058031082,
+      0.04513617232441902,
+      0.021437697112560272
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.7070927023887634,
+      0.4337051808834076,
+      0.20525239408016205,
+      0.12833012640476227,
+      0.08922113478183746,
+      0.07138099521398544,
+      0.06153824180364609,
+      0.050383489578962326,
+      0.048941195011138916,
+      0.05425061658024788,
+      0.056658461689949036,
+      0.05490320920944214,
+      0.058780405670404434,
+      0.06422601640224457,
+      0.0640580803155899,
+      0.07662343978881836,
+      0.07740550488233566,
+      0.07985741645097733,
+      0.08211264759302139,
+      0.08479849249124527,
+      0.08962702006101608,
+      0.09255073219537735,
+      0.09895103424787521,
+      0.10785157978534698,
+      0.11439661681652069,
+      0.11830822378396988,
+      0.12574751675128937,
+      0.13142701983451843,
+      0.13731370866298676,
+      0.14256031811237335,
+      0.14993664622306824,
+      0.15445290505886078,
+      0.1589001566171646,
+      0.1592540740966797,
+      0.16055789589881897,
+      0.16371439397335052,
+      0.16041849553585052,
+      0.166905015707016,
+      0.16415290534496307,
+      0.16529402136802673,
+      0.16753415763378143,
+      0.1604532152414322,
+      0.16251112520694733,
+      0.16150222718715668,
+      0.1627689003944397,
+      0.1577976644039154,
+      0.15638048946857452,
+      0.15296657383441925,
+      0.15357930958271027,
+      0.15450166165828705,
+      0.15814656019210815,
+      0.15624772012233734,
+      0.15832668542861938,
+      0.16225354373455048,
+      0.15898476541042328,
+      0.16120833158493042,
+      0.16103416681289673,
+      0.15684235095977783,
+      0.15743117034435272,
+      0.15934884548187256,
+      0.1600343883037567,
+      0.15916313230991364,
+      0.15786658227443695,
+      0.15931351482868195,
+      0.15841044485569,
+      0.15795432031154633,
+      0.15867257118225098,
+      0.15787459909915924,
+      0.1565372198820114,
+      0.1533852070569992,
+      0.15143153071403503,
+      0.1523621827363968,
+      0.14922958612442017,
+      0.15004687011241913,
+      0.14945951104164124,
+      0.14668743312358856,
+      0.14628423750400543,
+      0.1476394087076187,
+      0.15357595682144165,
+      0.1595272421836853,
+      0.16347573697566986,
+      0.17226624488830566,
+      0.17014096677303314,
+      0.1705310344696045,
+      0.17292410135269165,
+      0.17185914516448975,
+      0.1698039323091507,
+      0.17396710813045502,
+      0.1786484271287918,
+      0.1821245551109314,
+      0.17920179665088654,
+      0.18061326444149017,
+      0.1835414171218872,
+      0.18515288829803467,
+      0.1845807284116745,
+      0.1872483342885971,
+      0.18795664608478546,
+      0.18821750581264496,
+      0.19346866011619568,
+      0.19681136310100555,
+      0.20208954811096191,
+      0.20705866813659668,
+      0.20783445239067078,
+      0.20989425480365753,
+      0.2125101536512375,
+      0.21912641823291779,
+      0.21778839826583862,
+      0.21937686204910278,
+      0.2197670340538025,
+      0.22236476838588715,
+      0.22307045757770538,
+      0.22510498762130737,
+      0.2264418601989746,
+      0.2309156209230423,
+      0.23448912799358368,
+      0.23486173152923584,
+      0.23727869987487793,
+      0.23963871598243713,
+      0.24129126965999603,
+      0.24589307606220245,
+      0.24669644236564636,
+      0.24793024361133575,
+      0.2496657818555832,
+      0.25194722414016724,
+      0.25236889719963074,
+      0.2532859444618225,
+      0.25499001145362854,
+      0.25745660066604614,
+      0.25832274556159973,
+      0.2611595094203949,
+      0.2630329132080078,
+      0.2670155167579651,
+      0.2685263454914093,
+      0.2688233554363251,
+      0.2699948251247406,
+      0.2728136479854584,
+      0.27377867698669434,
+      0.27549049258232117,
+      0.27645307779312134,
+      0.2766518294811249,
+      0.27672335505485535,
+      0.2774313986301422,
+      0.2787529528141022,
+      0.27945172786712646,
+      0.2794017493724823,
+      0.28105705976486206,
+      0.2820282578468323,
+      0.2829187214374542,
+      0.2841479480266571,
+      0.2846400737762451
+    ],
+    "lr": [
+      8.376068376068378e-06,
+      1.6923076923076924e-05,
+      2.5470085470085475e-05,
+      3.401709401709402e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.993593461639863e-05,
+      3.934273662008964e-05,
+      3.874953862378065e-05,
+      3.815634062747166e-05,
+      3.7563142631162665e-05,
+      3.696994463485368e-05,
+      3.637674663854469e-05,
+      3.5783548642235693e-05,
+      3.519035064592671e-05,
+      3.459715264961771e-05,
+      3.400395465330873e-05,
+      3.341075665699973e-05,
+      3.281755866069075e-05,
+      3.2224360664381764e-05,
+      3.1251515950435014e-05,
+      3.065831795412602e-05,
+      3.0065119957817037e-05,
+      2.947192196150804e-05,
+      2.887872396519905e-05,
+      2.8285525968890065e-05,
+      2.769232797258107e-05,
+      2.709912997627208e-05,
+      2.6505931979963087e-05,
+      2.5912733983654104e-05,
+      2.5319535987345117e-05,
+      2.472633799103612e-05,
+      2.4133139994727132e-05,
+      2.3539941998418135e-05,
+      2.2946744002109148e-05,
+      2.1973899288162408e-05,
+      2.138070129185341e-05,
+      2.0787503295544424e-05,
+      2.019430529923544e-05,
+      1.9601107302926443e-05,
+      1.900790930661746e-05,
+      1.8414711310308462e-05,
+      1.7821513313999475e-05,
+      1.7228315317690488e-05,
+      1.663511732138149e-05,
+      1.6041919325072507e-05,
+      1.544872132876351e-05,
+      1.4855523332454524e-05,
+      1.4262325336145537e-05,
+      1.366912733983654e-05,
+      1.2696282625889797e-05,
+      1.2103084629580812e-05,
+      1.1509886633271816e-05,
+      1.0916688636962829e-05,
+      1.0323490640653833e-05,
+      9.730292644344846e-06,
+      9.137094648035861e-06,
+      8.543896651726864e-06,
+      7.950698655417878e-06,
+      7.357500659108881e-06,
+      6.764302662799895e-06,
+      6.171104666490909e-06,
+      5.577906670181913e-06,
+      4.984708673872927e-06,
+      4.391510677563931e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      750,
+      1532,
+      2314,
+      3096,
+      3878,
+      4660,
+      5442,
+      6224,
+      7006,
+      7788
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.62,
+      0.84,
+      0.99,
+      0.97,
+      0.98,
+      0.99,
+      0.98,
+      0.99,
+      0.99
+    ]
+  },
+  "final_accuracy": 0.995,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9851239669421488,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.72,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9705882352941176,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9822485207100592,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9222972972972973,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.64,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9815950920245399,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9307692307692308,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9425287356321839,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.8768472906403941,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9401709401709402,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.45,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9586776859504132,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.9795918367346939,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7634408602150538,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.8654708520179372,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.34,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9903846153846154,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.9622641509433962,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.9565217391304348,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.70625,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.7068403908794788,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.39,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.66,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.585,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.59,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.62,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.7,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.715,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9552572706935123,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.978125,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9464285714285714,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.943289224952741,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.8958333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.55,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7823834196891192,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.7757009345794392,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.55,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8203125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.8472222222222222,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.46,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8333333333333334,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.8298969072164949,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.45,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8415300546448088,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9273504273504274,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9883527454242929,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.9797979797979798,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.78,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.974910394265233,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.993103448275862,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9166666666666666,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.9444444444444444,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.57,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9906103286384976,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9911504424778761,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.7624309392265194,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.9814814814814815,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.21,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.994413407821229,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.970873786407767,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5302013422818792,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.7887323943661971,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.47,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.39666666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.04,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.43,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.3375,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.77,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.98,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9850187265917603,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9811320754716981,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9202733485193622,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.5,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9833333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7766497461928934,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.7961165048543689,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.41,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7854251012145749,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.7712418300653595,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.25,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7315436241610739,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.7574257425742574,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.5304166666666666,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 4,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.99375,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.93,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.99,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9949238578680203,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.995,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.995,
+  "sft_overall_accuracy": 0.5304166666666666
+}

add_sub_sorl_v1_abs30_50K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4acff836b99d599e5d5478cb5262bcf1b97b1d27955ef86f39ee768aeca36b25
+size 650385300

add_sub_sorl_v1_abs30_50K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 234,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 781,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs30_K4_50K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 30,
+  "dataset_size": 50000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162519662,
+  "run_name": "add_sub_sorl_v1_abs30_50K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T17:01:04.959212+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "8ug8xyio",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/8ug8xyio",
+  "final_accuracy": 0.995,
+  "sft_accuracy": 0.5304166666666666,
+  "eval_method": "ArithmeticEvaluator"
+}