Upload add_sub_sorl_v1_abs10_K1_100K_1L2H256d

add_sub_sorl_v1_abs10_K1_100K_1L2H256d/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "layer_types": [
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 2,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151654
+}

add_sub_sorl_v1_abs10_K1_100K_1L2H256d/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs10_K1_100K_1L2H256d/metrics.json

@@ -0,0 +1,3537 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1613,
+      1663,
+      1713,
+      1763,
+      1813,
+      1863,
+      1913,
+      1963,
+      2013,
+      2063,
+      2113,
+      2163,
+      2213,
+      2263,
+      2313,
+      2363,
+      2413,
+      2463,
+      2513,
+      2563,
+      2613,
+      2663,
+      2713,
+      2763,
+      2813,
+      2863,
+      2913,
+      2963,
+      3013,
+      3063,
+      3113,
+      3176,
+      3226,
+      3276,
+      3326,
+      3376,
+      3426,
+      3476,
+      3526,
+      3576,
+      3626,
+      3676,
+      3726,
+      3776,
+      3826,
+      3876,
+      3926,
+      3976,
+      4026,
+      4076,
+      4126,
+      4176,
+      4226,
+      4276,
+      4326,
+      4376,
+      4426,
+      4476,
+      4526,
+      4576,
+      4626,
+      4676,
+      4739,
+      4789,
+      4839,
+      4889,
+      4939,
+      4989,
+      5039,
+      5089,
+      5139,
+      5189,
+      5239,
+      5289,
+      5339,
+      5389,
+      5439,
+      5489,
+      5539,
+      5589,
+      5639,
+      5689,
+      5739,
+      5789,
+      5839,
+      5889,
+      5939,
+      5989,
+      6039,
+      6089,
+      6139,
+      6189,
+      6239,
+      6302,
+      6352,
+      6402,
+      6452,
+      6502,
+      6552,
+      6602,
+      6652,
+      6702,
+      6752,
+      6802,
+      6852,
+      6902,
+      6952,
+      7002,
+      7052,
+      7102,
+      7152,
+      7202,
+      7252,
+      7302,
+      7352,
+      7402,
+      7452,
+      7502,
+      7552,
+      7602,
+      7652,
+      7702,
+      7752,
+      7802,
+      7865,
+      7915,
+      7965,
+      8015,
+      8065,
+      8115,
+      8165,
+      8215,
+      8265,
+      8315,
+      8365,
+      8415,
+      8465,
+      8515,
+      8565,
+      8615,
+      8665,
+      8715,
+      8765,
+      8815,
+      8865,
+      8915,
+      8965,
+      9015,
+      9065,
+      9115,
+      9165,
+      9215,
+      9265,
+      9315,
+      9365,
+      9428,
+      9478,
+      9528,
+      9578,
+      9628,
+      9678,
+      9728,
+      9778,
+      9828,
+      9878,
+      9928,
+      9978,
+      10028,
+      10078,
+      10128,
+      10178,
+      10228,
+      10278,
+      10328,
+      10378,
+      10428,
+      10478,
+      10528,
+      10578,
+      10628,
+      10678,
+      10728,
+      10778,
+      10828,
+      10878,
+      10928,
+      10991,
+      11041,
+      11091,
+      11141,
+      11191,
+      11241,
+      11291,
+      11341,
+      11391,
+      11441,
+      11491,
+      11541,
+      11591,
+      11641,
+      11691,
+      11741,
+      11791,
+      11841,
+      11891,
+      11941,
+      11991,
+      12041,
+      12091,
+      12141,
+      12191,
+      12241,
+      12291,
+      12341,
+      12391,
+      12441,
+      12491,
+      12554,
+      12604,
+      12654,
+      12704,
+      12754,
+      12804,
+      12854,
+      12904,
+      12954,
+      13004,
+      13054,
+      13104,
+      13154,
+      13204,
+      13254,
+      13304,
+      13354,
+      13404,
+      13454,
+      13504,
+      13554,
+      13604,
+      13654,
+      13704,
+      13754,
+      13804,
+      13854,
+      13904,
+      13954,
+      14004,
+      14054,
+      14117,
+      14167,
+      14217,
+      14267,
+      14317,
+      14367,
+      14417,
+      14467,
+      14517,
+      14567,
+      14617,
+      14667,
+      14717,
+      14767,
+      14817,
+      14867,
+      14917,
+      14967,
+      15017,
+      15067,
+      15117,
+      15167,
+      15217,
+      15267,
+      15317,
+      15367,
+      15417,
+      15467,
+      15517,
+      15567,
+      15617
+    ],
+    "loss": [
+      16.28274154663086,
+      5.540170669555664,
+      6.775415420532227,
+      7.807186603546143,
+      9.381738662719727,
+      10.969490051269531,
+      10.444869995117188,
+      10.054375648498535,
+      8.876975059509277,
+      7.385862350463867,
+      6.720154762268066,
+      5.888507843017578,
+      5.069924831390381,
+      4.413970470428467,
+      4.091370582580566,
+      3.9275927543640137,
+      3.597487449645996,
+      3.644043445587158,
+      3.5632052421569824,
+      3.4460277557373047,
+      3.1855978965759277,
+      3.4230589866638184,
+      3.1606740951538086,
+      3.182441234588623,
+      3.3141112327575684,
+      2.986382484436035,
+      3.221864700317383,
+      2.7190232276916504,
+      2.959211587905884,
+      3.123387098312378,
+      2.808103084564209,
+      3.1065189838409424,
+      2.8977174758911133,
+      3.1317670345306396,
+      2.892608165740967,
+      2.915052890777588,
+      2.815518379211426,
+      2.557642936706543,
+      2.8139472007751465,
+      2.9387216567993164,
+      2.443659543991089,
+      2.685591220855713,
+      2.305596113204956,
+      2.809103012084961,
+      2.673182964324951,
+      2.780465602874756,
+      2.5513243675231934,
+      2.5324931144714355,
+      2.0034539699554443,
+      2.344390869140625,
+      2.150602102279663,
+      2.395678997039795,
+      2.310455322265625,
+      2.3398852348327637,
+      1.8025025129318237,
+      2.0364208221435547,
+      2.1480724811553955,
+      1.9397914409637451,
+      2.0974535942077637,
+      2.267531156539917,
+      2.0740933418273926,
+      2.1916496753692627,
+      0.9210824966430664,
+      0.5328047275543213,
+      -0.03742372989654541,
+      -1.2958544492721558,
+      -1.9911248683929443,
+      -3.7487802505493164,
+      -4.078698635101318,
+      -4.745406150817871,
+      -5.1121978759765625,
+      -5.62625789642334,
+      -6.281767845153809,
+      -5.8938517570495605,
+      -5.597670555114746,
+      -5.640196323394775,
+      -5.883167743682861,
+      -5.6108574867248535,
+      -4.988748550415039,
+      -5.896568775177002,
+      -5.29911994934082,
+      -5.248095989227295,
+      -4.921140193939209,
+      -4.775120258331299,
+      -5.096627235412598,
+      -4.684965133666992,
+      -4.368593215942383,
+      -3.99047589302063,
+      -4.204165458679199,
+      -3.5383524894714355,
+      -3.862689733505249,
+      -3.3033714294433594,
+      -2.654770851135254,
+      -4.525285720825195,
+      -3.0260891914367676,
+      -2.738969326019287,
+      -2.811995267868042,
+      -3.2562503814697266,
+      -2.3418192863464355,
+      -3.116446018218994,
+      -2.609781265258789,
+      -3.0971579551696777,
+      -0.9029642343521118,
+      -2.7395613193511963,
+      -2.838641405105591,
+      -2.255490303039551,
+      -3.0321860313415527,
+      -1.458106517791748,
+      -1.9517803192138672,
+      -1.5932023525238037,
+      -1.8326020240783691,
+      -2.6090221405029297,
+      -2.3446717262268066,
+      -1.0041512250900269,
+      -1.532463788986206,
+      -1.2158831357955933,
+      -2.256141185760498,
+      -2.2466983795166016,
+      -1.8169989585876465,
+      -2.3416221141815186,
+      -1.766740083694458,
+      -1.8386216163635254,
+      -1.825034737586975,
+      -1.8314135074615479,
+      -1.1143181324005127,
+      -2.257542371749878,
+      -1.9196970462799072,
+      -1.6531178951263428,
+      -1.3443089723587036,
+      -1.216113805770874,
+      -1.194965124130249,
+      -1.7019418478012085,
+      -1.354518175125122,
+      -1.4456787109375,
+      -0.8984273076057434,
+      -1.4400514364242554,
+      -1.3528547286987305,
+      -0.9314453601837158,
+      -1.4572619199752808,
+      -1.126159906387329,
+      -1.3588547706604004,
+      -1.5057815313339233,
+      -1.2514081001281738,
+      -0.48061734437942505,
+      -1.2874188423156738,
+      -1.2596217393875122,
+      -1.026874303817749,
+      -1.3001481294631958,
+      -0.7930020093917847,
+      -1.502524733543396,
+      -1.2518855333328247,
+      -0.963351845741272,
+      -1.2755764722824097,
+      -0.9888055920600891,
+      -0.8977932929992676,
+      -1.0401407480239868,
+      -1.0860685110092163,
+      -0.9494705200195312,
+      -1.436676263809204,
+      -0.980256199836731,
+      -0.9127078056335449,
+      -0.9295126795768738,
+      -1.1240662336349487,
+      -1.308225393295288,
+      -1.0526622533798218,
+      -0.9832852482795715,
+      -0.6777130961418152,
+      -0.5491603016853333,
+      -1.2792612314224243,
+      -0.7567460536956787,
+      -0.7345216274261475,
+      -0.931378960609436,
+      -1.0858885049819946,
+      -0.9426130056381226,
+      -1.132487177848816,
+      -1.269047737121582,
+      -0.9801985025405884,
+      -0.9577396512031555,
+      -0.9174380302429199,
+      -0.9476673603057861,
+      -0.9074362516403198,
+      -1.01382577419281,
+      -0.6530601382255554,
+      -1.0053731203079224,
+      -0.85746169090271,
+      -0.9764938950538635,
+      -0.8527376055717468,
+      -1.1913673877716064,
+      -1.0628403425216675,
+      -1.010689616203308,
+      -1.0420571565628052,
+      -1.2670626640319824,
+      -1.0489301681518555,
+      -1.476209044456482,
+      -0.7085262537002563,
+      -1.0096030235290527,
+      -0.9678630828857422,
+      -0.8825294375419617,
+      -0.8680270910263062,
+      -1.4723944664001465,
+      -0.5348420739173889,
+      -0.5407968163490295,
+      -1.1955978870391846,
+      -0.8696709275245667,
+      -0.998271644115448,
+      -0.8704187870025635,
+      -1.025557279586792,
+      -0.5137739181518555,
+      -1.0028613805770874,
+      -0.7887760400772095,
+      -0.8884354829788208,
+      -1.544988751411438,
+      -1.0478770732879639,
+      -1.0589046478271484,
+      -1.423649787902832,
+      -1.346470594406128,
+      -0.7388043403625488,
+      -0.950998842716217,
+      -1.0140622854232788,
+      -0.913789689540863,
+      -1.0420395135879517,
+      -1.023274540901184,
+      -1.045817255973816,
+      -1.155444860458374,
+      -1.1306260824203491,
+      -0.6776385307312012,
+      -0.7011786699295044,
+      -1.334304690361023,
+      -1.3645905256271362,
+      -1.1866199970245361,
+      -1.0871729850769043,
+      -0.9968757033348083,
+      -1.0070263147354126,
+      -1.3208439350128174,
+      -1.0825363397598267,
+      -1.2156729698181152,
+      -1.5994991064071655,
+      -1.1699426174163818,
+      -1.2325425148010254,
+      -1.551932692527771,
+      -0.8055230975151062,
+      -0.9251660704612732,
+      -1.285549283027649,
+      -1.6701040267944336,
+      -1.0110410451889038,
+      -1.5398916006088257,
+      -0.8088244199752808,
+      -1.0506784915924072,
+      -1.3399046659469604,
+      -1.2484153509140015,
+      -1.1578189134597778,
+      -1.1194970607757568,
+      -1.094886302947998,
+      -1.690836787223816,
+      -0.8785786628723145,
+      -1.3069560527801514,
+      -1.1893820762634277,
+      -1.1490076780319214,
+      -1.384873628616333,
+      -0.9505579471588135,
+      -1.1507246494293213,
+      -1.1267321109771729,
+      -1.1957576274871826,
+      -1.3316500186920166,
+      -1.088510274887085,
+      -1.4421019554138184,
+      -1.0149816274642944,
+      -1.0990639925003052,
+      -1.0213133096694946,
+      -1.3298611640930176,
+      -1.2966575622558594,
+      -1.4341070652008057,
+      -1.242478847503662,
+      -1.3080660104751587,
+      -1.0835931301116943,
+      -0.9934505820274353,
+      -1.5340348482131958,
+      -1.549197793006897,
+      -1.4818686246871948,
+      -1.4660003185272217,
+      -1.0767110586166382,
+      -1.234635591506958,
+      -1.1799765825271606,
+      -1.2203795909881592,
+      -1.4335767030715942,
+      -1.0706619024276733,
+      -1.1721309423446655,
+      -1.5047404766082764,
+      -1.1540157794952393,
+      -1.3545101881027222,
+      -0.8991901874542236,
+      -1.2084336280822754,
+      -0.9482430219650269,
+      -1.0421302318572998,
+      -1.5079543590545654,
+      -1.1981526613235474,
+      -1.2378123998641968,
+      -1.5786161422729492,
+      -1.2765175104141235,
+      -1.5619491338729858,
+      -1.1734968423843384,
+      -1.5895291566848755,
+      -1.4394630193710327,
+      -1.1113046407699585,
+      -1.4897830486297607,
+      -1.5032153129577637,
+      -1.0263656377792358,
+      -0.968646764755249,
+      -1.7087695598602295,
+      -0.9967091083526611
+    ],
+    "base_loss": [
+      11.919148445129395,
+      11.712471008300781,
+      11.401796340942383,
+      10.933218955993652,
+      10.320596694946289,
+      9.511163711547852,
+      8.804636001586914,
+      8.224196434020996,
+      7.224529266357422,
+      6.05776834487915,
+      5.158497333526611,
+      4.23229455947876,
+      3.593045473098755,
+      2.9647161960601807,
+      2.6388473510742188,
+      2.4015965461730957,
+      2.2090916633605957,
+      2.208362102508545,
+      2.0944769382476807,
+      2.0746800899505615,
+      2.00931715965271,
+      2.021466016769409,
+      2.0564398765563965,
+      1.939684510231018,
+      1.9386273622512817,
+      1.888741135597229,
+      1.9201011657714844,
+      1.8141651153564453,
+      1.8398933410644531,
+      1.8827736377716064,
+      1.8586024045944214,
+      1.9333986043930054,
+      1.897300124168396,
+      1.8588038682937622,
+      1.9004508256912231,
+      1.8095324039459229,
+      1.893649935722351,
+      1.9098756313323975,
+      1.9035409688949585,
+      1.823283076286316,
+      1.8070026636123657,
+      1.9187535047531128,
+      1.8444538116455078,
+      1.8281522989273071,
+      1.7837480306625366,
+      1.899887204170227,
+      1.8322093486785889,
+      1.8817052841186523,
+      1.846428632736206,
+      1.8482989072799683,
+      1.920029640197754,
+      1.8435605764389038,
+      1.8243188858032227,
+      1.832368016242981,
+      1.851176381111145,
+      1.802765965461731,
+      1.7245272397994995,
+      1.7926665544509888,
+      1.8278177976608276,
+      1.8894437551498413,
+      1.8149478435516357,
+      1.8408777713775635,
+      1.7879098653793335,
+      1.8937746286392212,
+      1.8526757955551147,
+      1.82782781124115,
+      1.8439241647720337,
+      1.8362096548080444,
+      1.844258427619934,
+      1.7638307809829712,
+      1.7990596294403076,
+      1.7790606021881104,
+      1.7944040298461914,
+      1.7255467176437378,
+      1.6491506099700928,
+      1.6426500082015991,
+      1.6298809051513672,
+      1.5767576694488525,
+      1.5220197439193726,
+      1.6187695264816284,
+      1.5979760885238647,
+      1.4917420148849487,
+      1.485482931137085,
+      1.4504286050796509,
+      1.4805594682693481,
+      1.4392255544662476,
+      1.3809515237808228,
+      1.367879867553711,
+      1.3364511728286743,
+      1.2636011838912964,
+      1.30233633518219,
+      1.2523545026779175,
+      1.158416509628296,
+      1.3363336324691772,
+      1.1729052066802979,
+      1.1867791414260864,
+      1.1456400156021118,
+      1.1837338209152222,
+      1.0774470567703247,
+      1.177153468132019,
+      1.0712212324142456,
+      1.1321731805801392,
+      0.8989397883415222,
+      1.0842927694320679,
+      1.1380772590637207,
+      1.0632250308990479,
+      1.1286226511001587,
+      0.9986239671707153,
+      0.9940967559814453,
+      0.9673513770103455,
+      0.9583966135978699,
+      1.0217516422271729,
+      1.043201208114624,
+      0.856368899345398,
+      0.920617938041687,
+      0.872459352016449,
+      0.9523282051086426,
+      0.9214135408401489,
+      0.8750053644180298,
+      0.894688069820404,
+      0.8571006059646606,
+      0.8104948401451111,
+      0.8283589482307434,
+      0.8048704266548157,
+      0.6931681632995605,
+      0.8597663044929504,
+      0.8563704490661621,
+      0.7519769668579102,
+      0.7527551054954529,
+      0.7053692936897278,
+      0.7427781820297241,
+      0.7440962791442871,
+      0.7218002676963806,
+      0.7397311329841614,
+      0.6547626256942749,
+      0.7235930562019348,
+      0.7188329696655273,
+      0.6374866366386414,
+      0.695144534111023,
+      0.6721176505088806,
+      0.6877056360244751,
+      0.7002540230751038,
+      0.6525031924247742,
+      0.5708634257316589,
+      0.6753590703010559,
+      0.6607192158699036,
+      0.6464635729789734,
+      0.653294026851654,
+      0.6221825480461121,
+      0.6701773405075073,
+      0.6161959767341614,
+      0.6116393208503723,
+      0.6640933752059937,
+      0.6056033372879028,
+      0.5704834461212158,
+      0.6159607768058777,
+      0.6038890480995178,
+      0.6191163063049316,
+      0.6200740933418274,
+      0.5866245627403259,
+      0.6012955904006958,
+      0.5644311308860779,
+      0.6018303632736206,
+      0.6254978775978088,
+      0.6019884943962097,
+      0.6134956479072571,
+      0.5368608832359314,
+      0.5497142672538757,
+      0.6003592610359192,
+      0.5710440874099731,
+      0.5597032308578491,
+      0.5748957395553589,
+      0.5628146529197693,
+      0.5685555338859558,
+      0.5940120816230774,
+      0.574928343296051,
+      0.5729753375053406,
+      0.5881057977676392,
+      0.5648398995399475,
+      0.5493632555007935,
+      0.5613466501235962,
+      0.5757318735122681,
+      0.5404321551322937,
+      0.5661589503288269,
+      0.5393258929252625,
+      0.5560475587844849,
+      0.5709224939346313,
+      0.5478270649909973,
+      0.5310960412025452,
+      0.510890007019043,
+      0.5301312804222107,
+      0.5633818507194519,
+      0.5569104552268982,
+      0.5721749067306519,
+      0.5167989730834961,
+      0.5295454859733582,
+      0.5481741428375244,
+      0.4657522737979889,
+      0.513382077217102,
+      0.5558506846427917,
+      0.4801826775074005,
+      0.4515341818332672,
+      0.5078088641166687,
+      0.5079418420791626,
+      0.4744904041290283,
+      0.5114633440971375,
+      0.48083963990211487,
+      0.4864089787006378,
+      0.5075017213821411,
+      0.4670749008655548,
+      0.5124925971031189,
+      0.5444943308830261,
+      0.48026639223098755,
+      0.5228786468505859,
+      0.5363203883171082,
+      0.49465709924697876,
+      0.5207586288452148,
+      0.4996943175792694,
+      0.5098710656166077,
+      0.5180677771568298,
+      0.44347116351127625,
+      0.5016202330589294,
+      0.46924081444740295,
+      0.4860851466655731,
+      0.49279502034187317,
+      0.44735997915267944,
+      0.4221813380718231,
+      0.4918827712535858,
+      0.49178367853164673,
+      0.48495641350746155,
+      0.48869290947914124,
+      0.4592805504798889,
+      0.46563562750816345,
+      0.5303858518600464,
+      0.47391048073768616,
+      0.4904748499393463,
+      0.5385576486587524,
+      0.49158334732055664,
+      0.4710582196712494,
+      0.5327602028846741,
+      0.458906888961792,
+      0.4368543028831482,
+      0.4965294301509857,
+      0.5137410759925842,
+      0.46738097071647644,
+      0.4764968752861023,
+      0.4679408669471741,
+      0.4773694574832916,
+      0.4808627665042877,
+      0.5116496682167053,
+      0.49025028944015503,
+      0.4609079957008362,
+      0.46923965215682983,
+      0.4925435185432434,
+      0.4423721730709076,
+      0.4723752439022064,
+      0.4821006953716278,
+      0.4392525851726532,
+      0.4597508907318115,
+      0.4585941433906555,
+      0.453828364610672,
+      0.43628570437431335,
+      0.4857936501502991,
+      0.4496951699256897,
+      0.47560811042785645,
+      0.5053812861442566,
+      0.41853663325309753,
+      0.45380905270576477,
+      0.4250417649745941,
+      0.4768609404563904,
+      0.4845709502696991,
+      0.5037023425102234,
+      0.4682858884334564,
+      0.44493526220321655,
+      0.4778483211994171,
+      0.4533351957798004,
+      0.47540754079818726,
+      0.5056682825088501,
+      0.5049258470535278,
+      0.4170120656490326,
+      0.4253399968147278,
+      0.4467671811580658,
+      0.435434490442276,
+      0.47085121273994446,
+      0.4235445559024811,
+      0.4327235817909241,
+      0.4594845771789551,
+      0.48848652839660645,
+      0.4533780515193939,
+      0.46535059809684753,
+      0.3864735960960388,
+      0.47652027010917664,
+      0.43212202191352844,
+      0.46275147795677185,
+      0.4903310239315033,
+      0.48137861490249634,
+      0.47230079770088196,
+      0.4315291941165924,
+      0.4502604603767395,
+      0.47259607911109924,
+      0.43375158309936523,
+      0.47321611642837524,
+      0.4843279719352722,
+      0.40902772545814514,
+      0.43668094277381897,
+      0.48824581503868103,
+      0.4492872655391693,
+      0.4369087815284729,
+      0.48378390073776245,
+      0.4564586281776428
+    ],
+    "info_loss": [
+      -0.5122928619384766,
+      -1.5516386032104492,
+      -1.3631782531738281,
+      -1.153665542602539,
+      -0.8245782852172852,
+      -0.4422922134399414,
+      -0.2909421920776367,
+      -0.16642475128173828,
+      -0.1158132553100586,
+      -0.10533332824707031,
+      -0.061872005462646484,
+      -0.042049407958984375,
+      -0.05414772033691406,
+      -0.05299186706542969,
+      -0.05021357536315918,
+      -0.040880441665649414,
+      -0.053603410720825195,
+      -0.04790902137756348,
+      -0.0438995361328125,
+      -0.053289175033569336,
+      -0.07236289978027344,
+      -0.04931437969207764,
+      -0.07886946201324463,
+      -0.06484067440032959,
+      -0.05133175849914551,
+      -0.07906532287597656,
+      -0.05845069885253906,
+      -0.09797036647796631,
+      -0.07637512683868408,
+      -0.0641258955001831,
+      -0.09331917762756348,
+      -0.07079529762268066,
+      -0.0879737138748169,
+      -0.06091618537902832,
+      -0.08890426158905029,
+      -0.07740592956542969,
+      -0.09558463096618652,
+      -0.12296092510223389,
+      -0.09653604030609131,
+      -0.07610416412353516,
+      -0.12391853332519531,
+      -0.11074912548065186,
+      -0.14138221740722656,
+      -0.08941447734832764,
+      -0.09848332405090332,
+      -0.09921717643737793,
+      -0.11567354202270508,
+      -0.12235105037689209,
+      -0.17184650897979736,
+      -0.1377655267715454,
+      -0.16433262825012207,
+      -0.13210749626159668,
+      -0.13878023624420166,
+      -0.13659119606018066,
+      -0.192283034324646,
+      -0.16389799118041992,
+      -0.1449146270751953,
+      -0.1726670265197754,
+      -0.16052234172821045,
+      -0.1495753526687622,
+      -0.1610504388809204,
+      -0.15235579013824463,
+      -0.27394044399261475,
+      -0.32341551780700684,
+      -0.3764849901199341,
+      -0.4997999668121338,
+      -0.5710752010345459,
+      -0.7459827661514282,
+      -0.7797424793243408,
+      -0.8384594321250916,
+      -0.8787723183631897,
+      -0.9284245371818542,
+      -0.9954951405525208,
+      -0.9495471119880676,
+      -0.9122193455696106,
+      -0.9159963726997375,
+      -0.9387544989585876,
+      -0.9063917398452759,
+      -0.8386374115943909,
+      -0.9387176036834717,
+      -0.877072811126709,
+      -0.861398458480835,
+      -0.8278194665908813,
+      -0.8095294833183289,
+      -0.8443809151649475,
+      -0.7982234954833984,
+      -0.7608359456062317,
+      -0.7209914922714233,
+      -0.73811274766922,
+      -0.6635478138923645,
+      -0.7001921534538269,
+      -0.6366941928863525,
+      -0.5615365505218506,
+      -0.7640740871429443,
+      -0.5945991277694702,
+      -0.5647491812705994,
+      -0.5606909394264221,
+      -0.60567706823349,
+      -0.5021160244941711,
+      -0.5846680402755737,
+      -0.5185971260070801,
+      -0.5730209946632385,
+      -0.32896584272384644,
+      -0.5262376666069031,
+      -0.5439128279685974,
+      -0.4771316647529602,
+      -0.5557167530059814,
+      -0.38420867919921875,
+      -0.43295836448669434,
+      -0.38912397623062134,
+      -0.4045712351799011,
+      -0.48948174715042114,
+      -0.46194642782211304,
+      -0.3053744435310364,
+      -0.3629930019378662,
+      -0.32027655839920044,
+      -0.4272693395614624,
+      -0.41490650177001953,
+      -0.3576827049255371,
+      -0.3799964189529419,
+      -0.3033410310745239,
+      -0.29865723848342896,
+      -0.2957419753074646,
+      -0.28839749097824097,
+      -0.20266690850257874,
+      -0.33067524433135986,
+      -0.2956508994102478,
+      -0.2551272213459015,
+      -0.22240173816680908,
+      -0.2055770456790924,
+      -0.20341724157333374,
+      -0.2561242878437042,
+      -0.21663248538970947,
+      -0.2259421944618225,
+      -0.1624666452407837,
+      -0.22259438037872314,
+      -0.21354800462722778,
+      -0.16300204396247864,
+      -0.22107869386672974,
+      -0.1860055923461914,
+      -0.20955437421798706,
+      -0.22589048743247986,
+      -0.1959669291973114,
+      -0.10984751582145691,
+      -0.20102742314338684,
+      -0.19643616676330566,
+      -0.17150172591209412,
+      -0.19943103194236755,
+      -0.14573311805725098,
+      -0.22132399678230286,
+      -0.1903863251209259,
+      -0.16109827160835266,
+      -0.19741639494895935,
+      -0.1631738245487213,
+      -0.15011829137802124,
+      -0.16891643404960632,
+      -0.17245212197303772,
+      -0.16002443432807922,
+      -0.2092139720916748,
+      -0.1599329710006714,
+      -0.15464594960212708,
+      -0.15251758694648743,
+      -0.17557546496391296,
+      -0.19633162021636963,
+      -0.1685791313648224,
+      -0.16263535618782043,
+      -0.12430834770202637,
+      -0.11275553703308105,
+      -0.19035077095031738,
+      -0.13514643907546997,
+      -0.13221633434295654,
+      -0.15303298830986023,
+      -0.16763857007026672,
+      -0.153389573097229,
+      -0.17510443925857544,
+      -0.18734610080718994,
+      -0.1579020917415619,
+      -0.15737786889076233,
+      -0.15064692497253418,
+      -0.1520923376083374,
+      -0.14940080046653748,
+      -0.16139566898345947,
+      -0.12160366773605347,
+      -0.15956392884254456,
+      -0.14217829704284668,
+      -0.15582504868507385,
+      -0.14486056566238403,
+      -0.17638611793518066,
+      -0.16162094473838806,
+      -0.1543361246585846,
+      -0.16008815169334412,
+      -0.1853637397289276,
+      -0.16270965337753296,
+      -0.20722848176956177,
+      -0.12509414553642273,
+      -0.15633845329284668,
+      -0.1540389358997345,
+      -0.13750392198562622,
+      -0.14044985175132751,
+      -0.20483553409576416,
+      -0.10373833775520325,
+      -0.10184478759765625,
+      -0.172575443983078,
+      -0.14007321000099182,
+      -0.14945897459983826,
+      -0.1404052972793579,
+      -0.152730792760849,
+      -0.10212671756744385,
+      -0.1529337763786316,
+      -0.127879798412323,
+      -0.1424349546432495,
+      -0.21103012561798096,
+      -0.15508770942687988,
+      -0.15998724102973938,
+      -0.1981440782546997,
+      -0.1862243115901947,
+      -0.1281237006187439,
+      -0.14692631363868713,
+      -0.15421628952026367,
+      -0.14520686864852905,
+      -0.15066051483154297,
+      -0.15450742840766907,
+      -0.15343132615089417,
+      -0.16613245010375977,
+      -0.16409596800804138,
+      -0.11432722210884094,
+      -0.11421123147010803,
+      -0.18463900685310364,
+      -0.18734890222549438,
+      -0.16918796300888062,
+      -0.15925583243370056,
+      -0.1474195420742035,
+      -0.1491110920906067,
+      -0.18679744005203247,
+      -0.15791112184524536,
+      -0.17218095064163208,
+      -0.21555030345916748,
+      -0.16788247227668762,
+      -0.1722114384174347,
+      -0.21009600162506104,
+      -0.12838611006736755,
+      -0.13789871335029602,
+      -0.17999035120010376,
+      -0.22030878067016602,
+      -0.14931786060333252,
+      -0.2031676471233368,
+      -0.12927794456481934,
+      -0.1543930172920227,
+      -0.1843828558921814,
+      -0.17761895060539246,
+      -0.16687500476837158,
+      -0.15976238250732422,
+      -0.15819570422172546,
+      -0.21991482377052307,
+      -0.1340363323688507,
+      -0.17997759580612183,
+      -0.16866591572761536,
+      -0.1605042815208435,
+      -0.18598440289497375,
+      -0.14341828227043152,
+      -0.16240501403808594,
+      -0.15829381346702576,
+      -0.170151948928833,
+      -0.18039092421531677,
+      -0.1584518551826477,
+      -0.19656941294670105,
+      -0.1448628008365631,
+      -0.15722376108169556,
+      -0.14663025736808777,
+      -0.18290838599205017,
+      -0.18007683753967285,
+      -0.19539517164230347,
+      -0.17280912399291992,
+      -0.1770569384098053,
+      -0.15808513760566711,
+      -0.14639171957969666,
+      -0.20306676626205444,
+      -0.20717424154281616,
+      -0.20062395930290222,
+      -0.19005045294761658,
+      -0.15200906991958618,
+      -0.16988474130630493,
+      -0.16308468580245972,
+      -0.17086553573608398,
+      -0.18767157196998596,
+      -0.15205052495002747,
+      -0.16452553868293762,
+      -0.20128098130226135,
+      -0.16243231296539307,
+      -0.18381425738334656,
+      -0.13005176186561584,
+      -0.1700531542301178,
+      -0.13999903202056885,
+      -0.15237721800804138,
+      -0.20155087113380432,
+      -0.1697521209716797,
+      -0.17272937297821045,
+      -0.20261144638061523,
+      -0.17432692646980286,
+      -0.20601502060890198,
+      -0.16243943572044373,
+      -0.20787155628204346,
+      -0.19462677836418152,
+      -0.15376442670822144,
+      -0.19422279298305511,
+      -0.20074373483657837,
+      -0.14949074387550354,
+      -0.14244553446769714,
+      -0.22162041068077087,
+      -0.14706194400787354
+    ],
+    "abs_loss": [
+      2.30220627784729,
+      2.3020200729370117,
+      2.297327995300293,
+      2.2975962162017822,
+      2.2630834579467773,
+      2.1765191555023193,
+      2.123943328857422,
+      2.0272209644317627,
+      1.9570759534835815,
+      1.8993927240371704,
+      1.8774352073669434,
+      1.8407106399536133,
+      1.8518379926681519,
+      1.8436359167099,
+      1.845046877861023,
+      1.8325690031051636,
+      1.8535631895065308,
+      1.8499723672866821,
+      1.835213303565979,
+      1.8594293594360352,
+      1.8574610948562622,
+      1.8389066457748413,
+      1.8507661819458008,
+      1.8515948057174683,
+      1.8454355001449585,
+      1.8561534881591797,
+      1.845703125,
+      1.8434991836547852,
+      1.8420748710632324,
+      1.8435863256454468,
+      1.8560940027236938,
+      1.8485981225967407,
+      1.8486992120742798,
+      1.8742297887802124,
+      1.8725115060806274,
+      1.8573325872421265,
+      1.8474016189575195,
+      1.8484903573989868,
+      1.8386249542236328,
+      1.84671151638031,
+      1.84579598903656,
+      1.8375964164733887,
+      1.8452197313308716,
+      1.8531140089035034,
+      1.8446292877197266,
+      1.8264933824539185,
+      1.8640602827072144,
+      1.8490185737609863,
+      1.868327260017395,
+      1.8535445928573608,
+      1.856315016746521,
+      1.8500362634658813,
+      1.8605047464370728,
+      1.8563438653945923,
+      1.863081932067871,
+      1.8476206064224243,
+      1.8537397384643555,
+      1.8645648956298828,
+      1.8775514364242554,
+      1.872701644897461,
+      1.826413631439209,
+      1.87376070022583,
+      1.8578802347183228,
+      1.8532167673110962,
+      1.8531559705734253,
+      1.8359180688858032,
+      1.8477325439453125,
+      1.8424381017684937,
+      1.8202654123306274,
+      1.8252969980239868,
+      1.8118122816085815,
+      1.8221300840377808,
+      1.8383985757827759,
+      1.814188003540039,
+      1.8050581216812134,
+      1.8271727561950684,
+      1.8115495443344116,
+      1.8208309412002563,
+      1.8224992752075195,
+      1.7885686159133911,
+      1.8172053098678589,
+      1.8242207765579224,
+      1.781652808189392,
+      1.7958316802978516,
+      1.7497321367263794,
+      1.6994233131408691,
+      1.7205805778503418,
+      1.6366468667984009,
+      1.6002146005630493,
+      1.5582994222640991,
+      1.5827299356460571,
+      1.509191870689392,
+      1.4383163452148438,
+      1.4367347955703735,
+      1.406564712524414,
+      1.3681570291519165,
+      1.294526219367981,
+      1.298731803894043,
+      1.2309962511062622,
+      1.2631953954696655,
+      1.1647754907608032,
+      1.1826599836349487,
+      1.0763381719589233,
+      1.0743846893310547,
+      1.0138899087905884,
+      1.0309642553329468,
+      0.9761902689933777,
+      0.9385819435119629,
+      0.9262393116950989,
+      0.8900609612464905,
+      0.8606628775596619,
+      0.8834860920906067,
+      0.8591017723083496,
+      0.8360124230384827,
+      0.8466210961341858,
+      0.8035519123077393,
+      0.7717851996421814,
+      0.7324153780937195,
+      0.7430999875068665,
+      0.663157045841217,
+      0.579663097858429,
+      0.590882420539856,
+      0.4899749457836151,
+      0.42981043457984924,
+      0.3855685889720917,
+      0.3650515377521515,
+      0.37482357025146484,
+      0.3443257808685303,
+      0.30352234840393066,
+      0.28127261996269226,
+      0.2570037543773651,
+      0.3061806261539459,
+      0.2609253227710724,
+      0.2383844405412674,
+      0.23392409086227417,
+      0.21278218924999237,
+      0.192348912358284,
+      0.23301690816879272,
+      0.22875840961933136,
+      0.24034100770950317,
+      0.20219844579696655,
+      0.2178707867860794,
+      0.25147247314453125,
+      0.2048385888338089,
+      0.2242809385061264,
+      0.20598942041397095,
+      0.18407393991947174,
+      0.18724918365478516,
+      0.18019075691699982,
+      0.2085704356431961,
+      0.16213588416576385,
+      0.17696678638458252,
+      0.14809666574001312,
+      0.20376606285572052,
+      0.16643618047237396,
+      0.1456851363182068,
+      0.18702729046344757,
+      0.16643057763576508,
+      0.17797093093395233,
+      0.15991371870040894,
+      0.1893058568239212,
+      0.17376239597797394,
+      0.15261325240135193,
+      0.1522006094455719,
+      0.15499384701251984,
+      0.15997610986232758,
+      0.14652855694293976,
+      0.16963058710098267,
+      0.13568313419818878,
+      0.13720117509365082,
+      0.14347034692764282,
+      0.14355270564556122,
+      0.12020000070333481,
+      0.12870250642299652,
+      0.14079205691814423,
+      0.14627568423748016,
+      0.14523349702358246,
+      0.16484005749225616,
+      0.148054301738739,
+      0.12204431742429733,
+      0.15206502377986908,
+      0.15463918447494507,
+      0.1306764781475067,
+      0.1207970678806305,
+      0.12553805112838745,
+      0.14228498935699463,
+      0.147186279296875,
+      0.1415950208902359,
+      0.1278122514486313,
+      0.129094198346138,
+      0.15491041541099548,
+      0.12417280673980713,
+      0.1242339238524437,
+      0.13601131737232208,
+      0.15513616800308228,
+      0.14911571145057678,
+      0.14021626114845276,
+      0.17228679358959198,
+      0.13032789528369904,
+      0.11838573217391968,
+      0.137715145945549,
+      0.1384749859571457,
+      0.13859133422374725,
+      0.14485764503479004,
+      0.11818784475326538,
+      0.13123421370983124,
+      0.12958703935146332,
+      0.1267758011817932,
+      0.10699955374002457,
+      0.1424816995859146,
+      0.14492733776569366,
+      0.10705948621034622,
+      0.12976239621639252,
+      0.10851290076971054,
+      0.12955591082572937,
+      0.13322141766548157,
+      0.1473109871149063,
+      0.11619482189416885,
+      0.11685353517532349,
+      0.12639732658863068,
+      0.1312195062637329,
+      0.13004010915756226,
+      0.11790201812982559,
+      0.12968654930591583,
+      0.11069496721029282,
+      0.12438997626304626,
+      0.11898013204336166,
+      0.11301834136247635,
+      0.10852862149477005,
+      0.11908483505249023,
+      0.10862568765878677,
+      0.11315381526947021,
+      0.11340445280075073,
+      0.0969037413597107,
+      0.12072455883026123,
+      0.08743100613355637,
+      0.11405970901250839,
+      0.10019014030694962,
+      0.11408897489309311,
+      0.09264203161001205,
+      0.11636162549257278,
+      0.09496424347162247,
+      0.10413985699415207,
+      0.09757804870605469,
+      0.09539119154214859,
+      0.09494751691818237,
+      0.0990312322974205,
+      0.08711335808038712,
+      0.12296789884567261,
+      0.09684770554304123,
+      0.13839595019817352,
+      0.09724470227956772,
+      0.10947561264038086,
+      0.09647106379270554,
+      0.11810692399740219,
+      0.11985272169113159,
+      0.1006196141242981,
+      0.10797067731618881,
+      0.09806424379348755,
+      0.15405064821243286,
+      0.11215709894895554,
+      0.11806517839431763,
+      0.12899713218212128,
+      0.12938092648983002,
+      0.12541724741458893,
+      0.11123824119567871,
+      0.09091644734144211,
+      0.11897752434015274,
+      0.12166506797075272,
+      0.13964661955833435,
+      0.11252722144126892,
+      0.07851729542016983,
+      0.11848396062850952,
+      0.11142794042825699,
+      0.12281394749879837,
+      0.11238782852888107,
+      0.10896693915128708,
+      0.0830840915441513,
+      0.14158326387405396,
+      0.11600643396377563,
+      0.10813343524932861,
+      0.1154547929763794,
+      0.10305937379598618,
+      0.11556471139192581,
+      0.11196726560592651,
+      0.10859853029251099,
+      0.08421885967254639,
+      0.12609051167964935,
+      0.12067227810621262,
+      0.1107231080532074,
+      0.09186488389968872,
+      0.1024598702788353,
+      0.1175227239727974,
+      0.11402594298124313,
+      0.10557401180267334,
+      0.11264423280954361,
+      0.10926514863967896,
+      0.09633132070302963,
+      0.09248441457748413,
+      0.12294084578752518,
+      0.11626702547073364,
+      0.10222375392913818,
+      0.14843891561031342,
+      0.10241835564374924,
+      0.10109380632638931,
+      0.10363570600748062,
+      0.11592870950698853,
+      0.1175919771194458,
+      0.13479028642177582,
+      0.11085989326238632
+    ],
+    "zipf_loss": [
+      9.25629997253418,
+      9.113883972167969,
+      8.77566909790039,
+      8.180863380432129,
+      7.0806169509887695,
+      5.663597106933594,
+      4.337262153625488,
+      3.2917046546936035,
+      2.61487078666687,
+      2.1914877891540527,
+      1.9926340579986572,
+      1.8926360607147217,
+      1.8331726789474487,
+      1.794809341430664,
+      1.7701541185379028,
+      1.7515437602996826,
+      1.739073634147644,
+      1.7297742366790771,
+      1.7242025136947632,
+      1.7182966470718384,
+      1.7141635417938232,
+      1.710845947265625,
+      1.7078521251678467,
+      1.7060041427612305,
+      1.704257845878601,
+      1.7026792764663696,
+      1.701700210571289,
+      1.700212001800537,
+      1.698862075805664,
+      1.6975138187408447,
+      1.6970831155776978,
+      1.6962134838104248,
+      1.6952847242355347,
+      1.6947020292282104,
+      1.6939486265182495,
+      1.693846344947815,
+      1.69297456741333,
+      1.692527413368225,
+      1.691904067993164,
+      1.691809058189392,
+      1.6912626028060913,
+      1.690569281578064,
+      1.6904425621032715,
+      1.689784049987793,
+      1.6898051500320435,
+      1.6901007890701294,
+      1.6894444227218628,
+      1.6893965005874634,
+      1.6886576414108276,
+      1.6883928775787354,
+      1.6882672309875488,
+      1.6881897449493408,
+      1.687888264656067,
+      1.687794804573059,
+      1.6878483295440674,
+      1.6878726482391357,
+      1.6873174905776978,
+      1.6873385906219482,
+      1.6871039867401123,
+      1.6865707635879517,
+      1.6870085000991821,
+      1.6869537830352783,
+      1.6867890357971191,
+      1.6878635883331299,
+      1.6894347667694092,
+      1.6907254457473755,
+      1.690929889678955,
+      1.6905936002731323,
+      1.6924415826797485,
+      1.6928272247314453,
+      1.695284128189087,
+      1.6967144012451172,
+      1.694939374923706,
+      1.6946533918380737,
+      1.6948661804199219,
+      1.6943997144699097,
+      1.6933411359786987,
+      1.6942189931869507,
+      1.6933562755584717,
+      1.692981243133545,
+      1.691911220550537,
+      1.6917247772216797,
+      1.6934055089950562,
+      1.690163016319275,
+      1.6916500329971313,
+      1.6881015300750732,
+      1.6867563724517822,
+      1.687894582748413,
+      1.6804893016815186,
+      1.677694320678711,
+      1.6786224842071533,
+      1.66029691696167,
+      1.6583465337753296,
+      1.6354482173919678,
+      1.6063408851623535,
+      1.5849279165267944,
+      1.5198214054107666,
+      1.4869130849838257,
+      1.4787943363189697,
+      1.4267616271972656,
+      1.388491153717041,
+      1.382612705230713,
+      1.3801206350326538,
+      1.3310840129852295,
+      1.361020803451538,
+      1.3495047092437744,
+      1.2987399101257324,
+      1.2914981842041016,
+      1.2910826206207275,
+      1.2416796684265137,
+      1.1686475276947021,
+      1.1756948232650757,
+      1.145681381225586,
+      1.109622836112976,
+      1.0921859741210938,
+      1.0340677499771118,
+      0.9870457053184509,
+      0.9077117443084717,
+      0.8105127215385437,
+      0.49733835458755493,
+      0.35160329937934875,
+      0.2783675789833069,
+      0.2550288736820221,
+      0.204709991812706,
+      0.18062588572502136,
+      0.15293851494789124,
+      0.14295929670333862,
+      0.111744724214077,
+      0.09660103917121887,
+      0.10616008937358856,
+      0.07072888314723969,
+      0.08458662033081055,
+      0.06391380727291107,
+      0.05017360299825668,
+      0.04808410629630089,
+      0.04102107137441635,
+      0.04455728828907013,
+      0.03778674453496933,
+      0.035504624247550964,
+      0.03774411976337433,
+      0.028763601556420326,
+      0.031082438305020332,
+      0.030610807240009308,
+      0.026510609313845634,
+      0.02506830170750618,
+      0.02342185378074646,
+      0.023272011429071426,
+      0.022143229842185974,
+      0.02412751503288746,
+      0.01968076452612877,
+      0.019568171352148056,
+      0.018294809386134148,
+      0.01968454197049141,
+      0.016952721402049065,
+      0.01626252755522728,
+      0.018494293093681335,
+      0.015860913321375847,
+      0.015014398843050003,
+      0.017592284828424454,
+      0.016457468271255493,
+      0.013525411486625671,
+      0.01385575719177723,
+      0.01459679938852787,
+      0.01437290757894516,
+      0.015641123056411743,
+      0.013575077056884766,
+      0.013856668025255203,
+      0.011717746034264565,
+      0.010318949818611145,
+      0.009954124689102173,
+      0.013591472059488297,
+      0.009699998423457146,
+      0.01566248945891857,
+      0.009856930933892727,
+      0.010465888306498528,
+      0.014857380650937557,
+      0.011323727667331696,
+      0.011449144221842289,
+      0.009385891258716583,
+      0.011688325554132462,
+      0.010018651373684406,
+      0.008935090154409409,
+      0.009476731531322002,
+      0.012027375400066376,
+      0.012441577389836311,
+      0.01148046925663948,
+      0.010226914659142494,
+      0.010507284663617611,
+      0.009491976350545883,
+      0.00887216255068779,
+      0.013202084228396416,
+      0.010775596834719181,
+      0.008832613006234169,
+      0.01029958575963974,
+      0.010102668777108192,
+      0.009324377402663231,
+      0.0103304423391819,
+      0.00952884741127491,
+      0.010056620463728905,
+      0.008271623402833939,
+      0.008587063290178776,
+      0.012269444763660431,
+      0.008488515391945839,
+      0.00863351859152317,
+      0.010008974932134151,
+      0.009047441184520721,
+      0.00795239768922329,
+      0.008406748995184898,
+      0.008274702355265617,
+      0.0086989039555192,
+      0.00892874039709568,
+      0.01011211983859539,
+      0.00975743681192398,
+      0.00723780132830143,
+      0.008514976128935814,
+      0.007793372496962547,
+      0.006942924112081528,
+      0.006950372830033302,
+      0.006544264033436775,
+      0.007571442984044552,
+      0.007972478866577148,
+      0.007175290957093239,
+      0.007465004920959473,
+      0.006825858727097511,
+      0.006469130516052246,
+      0.00583464652299881,
+      0.006854185834527016,
+      0.00890059769153595,
+      0.00626189261674881,
+      0.008394716307520866,
+      0.0058298781514167786,
+      0.006723964586853981,
+      0.007108468562364578,
+      0.007054171059280634,
+      0.010591959580779076,
+      0.006918659433722496,
+      0.006040392443537712,
+      0.007279671728610992,
+      0.007104609161615372,
+      0.0070029450580477715,
+      0.0077950600534677505,
+      0.00747024267911911,
+      0.007410754449665546,
+      0.009484970942139626,
+      0.0052174897864460945,
+      0.005793213844299316,
+      0.006111042574048042,
+      0.007170870900154114,
+      0.010764367878437042,
+      0.006439785473048687,
+      0.00684114545583725,
+      0.007494371384382248,
+      0.006883546710014343,
+      0.0061207786202430725,
+      0.007601674646139145,
+      0.008459338918328285,
+      0.005114257335662842,
+      0.005985446274280548,
+      0.005413053557276726,
+      0.00962575152516365,
+      0.008281432092189789,
+      0.008113802410662174,
+      0.007068503648042679,
+      0.009625975042581558,
+      0.00785838533192873,
+      0.007087072357535362,
+      0.006018038839101791,
+      0.007466768845915794,
+      0.00778102595359087,
+      0.008397052995860577,
+      0.0082870963960886,
+      0.008290646597743034,
+      0.00547797791659832,
+      0.006425101310014725,
+      0.007128441706299782,
+      0.005892768502235413,
+      0.01032860204577446,
+      0.008568023331463337,
+      0.005286820232868195,
+      0.005891531705856323,
+      0.007226178422570229,
+      0.005899136886000633,
+      0.005129905417561531,
+      0.005868119187653065,
+      0.008397690951824188,
+      0.006259960122406483,
+      0.0052178893238306046,
+      0.006973759271204472,
+      0.004862101748585701,
+      0.007209576666355133,
+      0.005667243152856827,
+      0.0053315721452236176,
+      0.007873043417930603,
+      0.007487917318940163,
+      0.006665840744972229,
+      0.0067255739122629166,
+      0.006254076957702637,
+      0.006335965357720852,
+      0.00724283792078495,
+      0.013310916721820831,
+      0.005519290454685688,
+      0.005747945047914982,
+      0.00763280875980854,
+      0.0070700617507100105,
+      0.005654546432197094,
+      0.005612436681985855,
+      0.007661543786525726,
+      0.007140662521123886,
+      0.01017164345830679,
+      0.006365802139043808
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.8612614274024963,
+      0.8016671538352966,
+      0.6376562118530273,
+      0.5529548525810242,
+      0.487213134765625,
+      0.4188559353351593,
+      0.3968985676765442,
+      0.36472076177597046,
+      0.3819717466831207,
+      0.3449137508869171,
+      0.3558287024497986,
+      0.3638819456100464,
+      0.3564022481441498,
+      0.35707730054855347,
+      0.35364609956741333,
+      0.35979828238487244,
+      0.37180295586586,
+      0.34970489144325256,
+      0.36201927065849304,
+      0.35735681653022766,
+      0.3490956425666809,
+      0.3370121717453003,
+      0.3661976456642151,
+      0.35761991143226624,
+      0.35063067078590393,
+      0.37178748846054077,
+      0.3578042984008789,
+      0.36101120710372925,
+      0.35397353768348694,
+      0.355873703956604,
+      0.36221230030059814,
+      0.34353312849998474,
+      0.3402397632598877,
+      0.36903098225593567,
+      0.36701905727386475,
+      0.37839940190315247,
+      0.3693663775920868,
+      0.37097227573394775,
+      0.3770667612552643,
+      0.3849909007549286,
+      0.3801876902580261,
+      0.37918296456336975,
+      0.3857121765613556,
+      0.3716696798801422,
+      0.3718411326408386,
+      0.38376492261886597,
+      0.39209532737731934,
+      0.38883596658706665,
+      0.38426607847213745,
+      0.395708292722702,
+      0.40130382776260376,
+      0.4030826985836029,
+      0.40030035376548767,
+      0.4057621657848358,
+      0.4034087359905243,
+      0.3974243402481079,
+      0.3926997482776642,
+      0.379551500082016,
+      0.3717574179172516,
+      0.35956549644470215,
+      0.360320121049881,
+      0.35186290740966797,
+      0.33709487318992615,
+      0.3470607101917267,
+      0.3481311798095703,
+      0.3570297360420227,
+      0.36510586738586426,
+      0.3691912293434143,
+      0.3768906593322754,
+      0.3805363178253174,
+      0.38554590940475464,
+      0.3948875963687897,
+      0.39736804366111755,
+      0.40309232473373413,
+      0.40349623560905457,
+      0.40225210785865784,
+      0.40144655108451843,
+      0.4063461422920227,
+      0.4068640172481537,
+      0.40520644187927246,
+      0.404690682888031,
+      0.4063727557659149,
+      0.4051498472690582,
+      0.40500056743621826,
+      0.40960177779197693,
+      0.4152390956878662,
+      0.42116326093673706,
+      0.42314326763153076,
+      0.4235926866531372,
+      0.42154932022094727,
+      0.42305120825767517,
+      0.4258868098258972,
+      0.4293067753314972,
+      0.43354400992393494,
+      0.43590062856674194,
+      0.43892383575439453,
+      0.44140514731407166,
+      0.4361226558685303,
+      0.43728816509246826,
+      0.43766650557518005,
+      0.4426195025444031,
+      0.4439590871334076,
+      0.44415393471717834,
+      0.4427838623523712,
+      0.4461572468280792,
+      0.4409407079219818,
+      0.4392300248146057,
+      0.4358401894569397,
+      0.4353138506412506,
+      0.43288812041282654,
+      0.4336419999599457,
+      0.4324011206626892,
+      0.4309177100658417,
+      0.4308708608150482,
+      0.4289792478084564,
+      0.42820799350738525,
+      0.4287213683128357,
+      0.42741137742996216,
+      0.42075255513191223,
+      0.41462472081184387,
+      0.413504421710968,
+      0.4214182198047638,
+      0.42003247141838074,
+      0.42020121216773987,
+      0.4176969826221466,
+      0.41551828384399414,
+      0.40811267495155334,
+      0.4051038920879364,
+      0.4008391499519348,
+      0.3953404128551483,
+      0.39215317368507385,
+      0.3891485631465912,
+      0.38752493262290955,
+      0.3828921914100647,
+      0.3794485926628113,
+      0.37573060393333435,
+      0.375820130109787,
+      0.3753224015235901,
+      0.3725639581680298,
+      0.3697616755962372,
+      0.3674364686012268,
+      0.3632788062095642,
+      0.3605678081512451,
+      0.35727590322494507,
+      0.35582369565963745,
+      0.35303837060928345,
+      0.3514127731323242,
+      0.35009339451789856,
+      0.34657740592956543,
+      0.34629932045936584,
+      0.3432376980781555,
+      0.34238681197166443,
+      0.3385182023048401,
+      0.3360331356525421,
+      0.3332003355026245,
+      0.33170008659362793,
+      0.32806873321533203,
+      0.3252578675746918,
+      0.319923996925354,
+      0.3184064030647278,
+      0.3158026933670044,
+      0.313507080078125,
+      0.3140915334224701,
+      0.310560941696167,
+      0.3052983582019806,
+      0.3057887554168701,
+      0.30279409885406494,
+      0.3015226423740387,
+      0.29875221848487854,
+      0.29937052726745605,
+      0.2992631494998932,
+      0.2975480556488037,
+      0.30423617362976074,
+      0.2999492287635803,
+      0.29561981558799744,
+      0.2912825345993042,
+      0.291506290435791,
+      0.2906370759010315,
+      0.28881019353866577,
+      0.287432998418808,
+      0.28537631034851074,
+      0.2833746373653412,
+      0.28295084834098816,
+      0.2816351354122162,
+      0.27952811121940613,
+      0.27837491035461426,
+      0.27690204977989197,
+      0.2759683430194855,
+      0.2742611765861511,
+      0.27653852105140686,
+      0.27665194869041443,
+      0.27422529458999634,
+      0.27421310544013977,
+      0.27777519822120667,
+      0.2767961919307709,
+      0.27734461426734924,
+      0.2787007987499237,
+      0.27804240584373474,
+      0.27716201543807983,
+      0.2739870250225067,
+      0.27515217661857605,
+      0.2748262286186218,
+      0.27342671155929565,
+      0.2726982533931732,
+      0.27094531059265137,
+      0.2709236741065979,
+      0.27097347378730774,
+      0.2701936662197113,
+      0.2681817412376404,
+      0.2733210325241089,
+      0.273675799369812,
+      0.27300065755844116,
+      0.2739092707633972,
+      0.2736546993255615,
+      0.2735222280025482,
+      0.27190786600112915,
+      0.2713083326816559,
+      0.27074316143989563,
+      0.2763568162918091,
+      0.27521294355392456,
+      0.2762555480003357,
+      0.274553507566452,
+      0.2755734920501709,
+      0.27438685297966003,
+      0.2749340236186981,
+      0.2759014666080475,
+      0.2753879129886627,
+      0.27383914589881897,
+      0.27310097217559814,
+      0.2731490731239319,
+      0.27343255281448364,
+      0.2724662125110626,
+      0.27148714661598206,
+      0.27072814106941223,
+      0.2711607813835144,
+      0.27134573459625244,
+      0.27197426557540894,
+      0.27125847339630127,
+      0.2720467448234558,
+      0.2709692418575287,
+      0.2728133201599121,
+      0.2733900547027588,
+      0.2739672064781189,
+      0.27517881989479065,
+      0.27605023980140686,
+      0.27582859992980957,
+      0.27701157331466675,
+      0.27726224064826965,
+      0.2775222957134247,
+      0.27851271629333496,
+      0.27983158826828003,
+      0.27902233600616455,
+      0.2787399888038635,
+      0.2786222994327545,
+      0.2784559428691864,
+      0.2796696722507477,
+      0.28023970127105713,
+      0.28060993552207947,
+      0.2802201807498932,
+      0.2800673544406891,
+      0.2804716229438782,
+      0.2805470824241638,
+      0.2807961106300354,
+      0.28189617395401,
+      0.2826630175113678,
+      0.2827717959880829,
+      0.28256848454475403,
+      0.2831490933895111,
+      0.2828967571258545,
+      0.2823440134525299,
+      0.28238457441329956,
+      0.2812761068344116,
+      0.28100937604904175,
+      0.28174257278442383,
+      0.28244441747665405,
+      0.282766193151474,
+      0.2822505235671997,
+      0.28302276134490967,
+      0.2834620177745819,
+      0.2831602096557617,
+      0.2831542491912842,
+      0.28293392062187195,
+      0.2828397750854492,
+      0.2833317220211029,
+      0.28331828117370605,
+      0.2841479778289795,
+      0.2836388051509857,
+      0.28366315364837646,
+      0.28394338488578796,
+      0.284136027097702,
+      0.28456246852874756,
+      0.28469717502593994,
+      0.28498443961143494,
+      0.2851629853248596,
+      0.2855220437049866,
+      0.28587767481803894,
+      0.28608614206314087,
+      0.28616443276405334,
+      0.28624650835990906,
+      0.28685617446899414,
+      0.28658923506736755,
+      0.28667718172073364,
+      0.2866398096084595,
+      0.2867295742034912,
+      0.28634944558143616,
+      0.28670117259025574,
+      0.28687241673469543,
+      0.2867128252983093,
+      0.2871307134628296,
+      0.2871498167514801
+    ],
+    "lr": [
+      2.0940170940170946e-06,
+      4.230769230769231e-06,
+      6.367521367521369e-06,
+      8.504273504273505e-06,
+      1.0641025641025641e-05,
+      1.2777777777777779e-05,
+      1.4914529914529916e-05,
+      1.7051282051282053e-05,
+      1.918803418803419e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      1.9964978235061336e-05,
+      1.981658092599921e-05,
+      1.966818361693708e-05,
+      1.951978630787495e-05,
+      1.9371388998812825e-05,
+      1.9222991689750693e-05,
+      1.9074594380688566e-05,
+      1.8926197071626434e-05,
+      1.8777799762564307e-05,
+      1.8629402453502176e-05,
+      1.8481005144440044e-05,
+      1.8332607835377923e-05,
+      1.8184210526315792e-05,
+      1.803581321725366e-05,
+      1.7887415908191533e-05,
+      1.7739018599129406e-05,
+      1.7590621290067274e-05,
+      1.7442223981005143e-05,
+      1.729382667194302e-05,
+      1.7145429362880888e-05,
+      1.6997032053818757e-05,
+      1.6848634744756632e-05,
+      1.67002374356945e-05,
+      1.655184012663237e-05,
+      1.640344281757024e-05,
+      1.6255045508508114e-05,
+      1.6106648199445983e-05,
+      1.5958250890383856e-05,
+      1.5771270280965573e-05,
+      1.562287297190344e-05,
+      1.5474475662841314e-05,
+      1.532607835377919e-05,
+      1.5177681044717056e-05,
+      1.5029283735654927e-05,
+      1.48808864265928e-05,
+      1.4732489117530668e-05,
+      1.4584091808468539e-05,
+      1.4435694499406409e-05,
+      1.4287297190344285e-05,
+      1.4138899881282154e-05,
+      1.3990502572220024e-05,
+      1.3842105263157896e-05,
+      1.3693707954095765e-05,
+      1.3545310645033636e-05,
+      1.3396913335971506e-05,
+      1.3248516026909382e-05,
+      1.310011871784725e-05,
+      1.2951721408785121e-05,
+      1.2803324099722994e-05,
+      1.2654926790660862e-05,
+      1.2506529481598733e-05,
+      1.2358132172536607e-05,
+      1.220973486347448e-05,
+      1.2061337554412346e-05,
+      1.1912940245350217e-05,
+      1.176454293628809e-05,
+      1.1616145627225958e-05,
+      1.146774831816383e-05,
+      1.1319351009101704e-05,
+      1.113237039968342e-05,
+      1.098397309062129e-05,
+      1.0835575781559164e-05,
+      1.0687178472497033e-05,
+      1.0538781163434902e-05,
+      1.0390383854372777e-05,
+      1.0241986545310648e-05,
+      1.0093589236248515e-05,
+      9.945191927186385e-06,
+      9.79679461812426e-06,
+      9.64839730906213e-06,
+      9.499999999999999e-06,
+      9.351602690937873e-06,
+      9.203205381875743e-06,
+      9.054808072813612e-06,
+      8.906410763751483e-06,
+      8.758013454689357e-06,
+      8.609616145627227e-06,
+      8.461218836565096e-06,
+      8.31282152750297e-06,
+      8.16442421844084e-06,
+      8.01602690937871e-06,
+      7.86762960031658e-06,
+      7.719232291254454e-06,
+      7.5708349821923235e-06,
+      7.422437673130192e-06,
+      7.274040364068067e-06,
+      7.125643055005938e-06,
+      6.977245745943806e-06,
+      6.828848436881681e-06,
+      6.68045112781955e-06,
+      6.493470518401265e-06,
+      6.345073209339139e-06,
+      6.196675900277009e-06,
+      6.048278591214878e-06,
+      5.899881282152753e-06,
+      5.7514839730906225e-06,
+      5.603086664028492e-06,
+      5.454689354966362e-06,
+      5.306292045904236e-06,
+      5.157894736842106e-06,
+      5.009497427779974e-06,
+      4.86110011871785e-06,
+      4.712702809655719e-06,
+      4.5643055005935885e-06,
+      4.415908191531458e-06,
+      4.267510882469333e-06,
+      4.119113573407203e-06,
+      3.9707162643450716e-06,
+      3.822318955282946e-06,
+      3.673921646220816e-06,
+      3.5255243371586857e-06,
+      3.377127028096555e-06,
+      3.22872971903443e-06,
+      3.080332409972299e-06,
+      2.9319351009101688e-06,
+      2.783537791848043e-06,
+      2.635140482785913e-06,
+      2.4867431737237825e-06,
+      2.338345864661657e-06,
+      2.1899485555995266e-06,
+      2.041551246537396e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      1550,
+      3113,
+      4676,
+      6239,
+      7802,
+      9365,
+      10928,
+      12491,
+      14054,
+      15617
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.01,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 0.25666666666666665,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.61,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9338842975206612,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9473684210526315,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.33,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9411764705882353,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.8757396449704142,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.9354838709677419,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.7804054054054054,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.18,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9141104294478528,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.8769230769230769,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.7931034482758621,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.6551724137931034,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.8205128205128205,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.08,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9669421487603306,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.8677685950413223,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.6938775510204082,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.5483870967741935,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.57847533632287,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.16,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9423076923076923,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.8679245283018868,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.7391304347826086,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.5875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.4755700325732899,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.04,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.11,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.205,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.39,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.55,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.512,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.315,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.941834451901566,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.871875,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9107142857142857,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.7674858223062382,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.7291666666666666,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.18,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.99,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.95,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.48186528497409326,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.6355140186915887,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.15,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.53515625,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.7222222222222222,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.13,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.5522875816993464,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.6907216494845361,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6475409836065574,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.8717948717948718,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.13,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7537437603993344,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.8787878787878788,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8172043010752689,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.9379310344827586,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.6666666666666666,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.3134920634920635,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.05,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8544600938967136,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.8849557522123894,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9176470588235294,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.32044198895027626,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.3333333333333333,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9441340782122905,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9223300970873787,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.3087248322147651,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.18779342723004694,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.865,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.88,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.37,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.23666666666666666,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.04,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.85,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.41,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.2575,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.06,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8116666666666666,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9138576779026217,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.8301886792452831,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.31890660592255127,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.2682926829268293,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8166666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.91,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.27918781725888325,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.2912621359223301,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.845,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.94,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.24696356275303644,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.29411764705882354,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.9,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.2651006711409396,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.297029702970297,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.14375,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.68,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.947107438016529,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.968421052631579,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.45,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.946078431372549,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9230769230769231,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.847972972972973,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.28,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9325153374233128,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.823076923076923,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9080459770114943,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.7241379310344828,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9316239316239316,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.28,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9586776859504132,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9256198347107438,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.9387755102040817,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7580645161290323,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.7488789237668162,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.16,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9519230769230769,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.8867924528301887,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.9565217391304348,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.7,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.5928338762214984,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.1,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.45,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.3675,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.64,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.4,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.52,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9530201342281879,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.928125,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9107142857142857,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.8563327032136105,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.8541666666666666,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.31,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9833333333333333,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6528497409326425,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.6728971962616822,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.26,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.6875,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.7013888888888888,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.19,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.696078431372549,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.6752577319587629,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.19,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.726775956284153,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.717948717948718,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.26,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8386023294509152,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.8080808080808081,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.26,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8924731182795699,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.896551724137931,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.875,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.75,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.22,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.892018779342723,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.8584070796460177,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9411764705882353,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.6906077348066298,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.7685185185185185,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.17,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9608938547486033,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.8737864077669902,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9642857142857143,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5838926174496645,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.6807511737089202,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.16,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.91,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.94,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.59,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.63,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.08,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.47,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.515,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.25,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.875,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.8689138576779026,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.8679245283018868,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.7403189066059226,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.7560975609756098,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.21,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8433333333333334,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.88,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.6446700507614214,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.6796116504854369,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.21,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.93,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.91,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.6356275303643725,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.7124183006535948,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.6073825503355704,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.6534653465346535,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.25666666666666665,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.25666666666666665,
+  "sft_overall_accuracy": 0.14375
+}

add_sub_sorl_v1_abs10_K1_100K_1L2H256d/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:145861f898da6af4691710f6bb58842b889edff7df9fafa7dd22498806818c68
+size 315091124

add_sub_sorl_v1_abs10_K1_100K_1L2H256d/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 2e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 1562,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs10_K1_100K_1L2H256d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 1,
+  "n_head": 2,
+  "n_embd": 256,
+  "ops": "add_sub",
+  "abs_vocab": 10,
+  "dataset_size": 100000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 78696448,
+  "run_name": "add_sub_sorl_v1_abs10_K1_100K_1L2H256d",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T19:15:59.955928+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "51hfiren",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/51hfiren",
+  "final_accuracy": 0.25666666666666665,
+  "sft_accuracy": 0.14375,
+  "eval_method": "ArithmeticEvaluator"
+}