Upload add_sub_sorl_v1_abs5_100K

add_sub_sorl_v1_abs5_100K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151649
+}

add_sub_sorl_v1_abs5_100K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs5_100K/metrics.json

@@ -0,0 +1,3537 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1613,
+      1663,
+      1713,
+      1763,
+      1813,
+      1863,
+      1913,
+      1963,
+      2013,
+      2063,
+      2113,
+      2163,
+      2213,
+      2263,
+      2313,
+      2363,
+      2413,
+      2463,
+      2513,
+      2563,
+      2613,
+      2663,
+      2713,
+      2763,
+      2813,
+      2863,
+      2913,
+      2963,
+      3013,
+      3063,
+      3113,
+      3176,
+      3226,
+      3276,
+      3326,
+      3376,
+      3426,
+      3476,
+      3526,
+      3576,
+      3626,
+      3676,
+      3726,
+      3776,
+      3826,
+      3876,
+      3926,
+      3976,
+      4026,
+      4076,
+      4126,
+      4176,
+      4226,
+      4276,
+      4326,
+      4376,
+      4426,
+      4476,
+      4526,
+      4576,
+      4626,
+      4676,
+      4739,
+      4789,
+      4839,
+      4889,
+      4939,
+      4989,
+      5039,
+      5089,
+      5139,
+      5189,
+      5239,
+      5289,
+      5339,
+      5389,
+      5439,
+      5489,
+      5539,
+      5589,
+      5639,
+      5689,
+      5739,
+      5789,
+      5839,
+      5889,
+      5939,
+      5989,
+      6039,
+      6089,
+      6139,
+      6189,
+      6239,
+      6302,
+      6352,
+      6402,
+      6452,
+      6502,
+      6552,
+      6602,
+      6652,
+      6702,
+      6752,
+      6802,
+      6852,
+      6902,
+      6952,
+      7002,
+      7052,
+      7102,
+      7152,
+      7202,
+      7252,
+      7302,
+      7352,
+      7402,
+      7452,
+      7502,
+      7552,
+      7602,
+      7652,
+      7702,
+      7752,
+      7802,
+      7865,
+      7915,
+      7965,
+      8015,
+      8065,
+      8115,
+      8165,
+      8215,
+      8265,
+      8315,
+      8365,
+      8415,
+      8465,
+      8515,
+      8565,
+      8615,
+      8665,
+      8715,
+      8765,
+      8815,
+      8865,
+      8915,
+      8965,
+      9015,
+      9065,
+      9115,
+      9165,
+      9215,
+      9265,
+      9315,
+      9365,
+      9428,
+      9478,
+      9528,
+      9578,
+      9628,
+      9678,
+      9728,
+      9778,
+      9828,
+      9878,
+      9928,
+      9978,
+      10028,
+      10078,
+      10128,
+      10178,
+      10228,
+      10278,
+      10328,
+      10378,
+      10428,
+      10478,
+      10528,
+      10578,
+      10628,
+      10678,
+      10728,
+      10778,
+      10828,
+      10878,
+      10928,
+      10991,
+      11041,
+      11091,
+      11141,
+      11191,
+      11241,
+      11291,
+      11341,
+      11391,
+      11441,
+      11491,
+      11541,
+      11591,
+      11641,
+      11691,
+      11741,
+      11791,
+      11841,
+      11891,
+      11941,
+      11991,
+      12041,
+      12091,
+      12141,
+      12191,
+      12241,
+      12291,
+      12341,
+      12391,
+      12441,
+      12491,
+      12554,
+      12604,
+      12654,
+      12704,
+      12754,
+      12804,
+      12854,
+      12904,
+      12954,
+      13004,
+      13054,
+      13104,
+      13154,
+      13204,
+      13254,
+      13304,
+      13354,
+      13404,
+      13454,
+      13504,
+      13554,
+      13604,
+      13654,
+      13704,
+      13754,
+      13804,
+      13854,
+      13904,
+      13954,
+      14004,
+      14054,
+      14117,
+      14167,
+      14217,
+      14267,
+      14317,
+      14367,
+      14417,
+      14467,
+      14517,
+      14567,
+      14617,
+      14667,
+      14717,
+      14767,
+      14817,
+      14867,
+      14917,
+      14967,
+      15017,
+      15067,
+      15117,
+      15167,
+      15217,
+      15267,
+      15317,
+      15367,
+      15417,
+      15467,
+      15517,
+      15567,
+      15617
+    ],
+    "loss": [
+      18.44952964782715,
+      13.085131645202637,
+      9.236353874206543,
+      6.564369201660156,
+      4.726602554321289,
+      4.29766845703125,
+      3.989104747772217,
+      3.6649608612060547,
+      3.7498269081115723,
+      3.190838575363159,
+      2.7308132648468018,
+      2.375901699066162,
+      2.3600034713745117,
+      1.1223392486572266,
+      -0.17718391120433807,
+      0.21777509152889252,
+      -0.49384427070617676,
+      0.11907894164323807,
+      -0.07290219515562057,
+      0.6291598081588745,
+      0.8631189465522766,
+      0.6290956139564514,
+      0.39158040285110474,
+      0.405124306678772,
+      0.0058848317712545395,
+      -2.2432780265808105,
+      -5.301384925842285,
+      -6.254613876342773,
+      -7.836089611053467,
+      -8.577259063720703,
+      -9.0380277633667,
+      -10.52203369140625,
+      -10.115748405456543,
+      -9.83785343170166,
+      -10.031425476074219,
+      -10.07608413696289,
+      -10.924676895141602,
+      -11.304506301879883,
+      -11.676875114440918,
+      -11.347262382507324,
+      -11.275835990905762,
+      -12.261313438415527,
+      -11.929290771484375,
+      -12.343928337097168,
+      -12.007375717163086,
+      -13.055440902709961,
+      -12.452829360961914,
+      -12.293601989746094,
+      -11.662895202636719,
+      -12.369410514831543,
+      -13.215283393859863,
+      -12.675363540649414,
+      -12.844850540161133,
+      -12.796992301940918,
+      -11.95716381072998,
+      -12.096969604492188,
+      -12.005210876464844,
+      -12.04047679901123,
+      -12.584002494812012,
+      -13.298893928527832,
+      -12.562734603881836,
+      -12.922416687011719,
+      -11.108821868896484,
+      -10.939716339111328,
+      -7.941163539886475,
+      -6.545567512512207,
+      -5.882503986358643,
+      -5.302046298980713,
+      -5.577780246734619,
+      -5.092601776123047,
+      -4.7532243728637695,
+      -4.90067720413208,
+      -4.917669296264648,
+      -5.0563530921936035,
+      -5.2757697105407715,
+      -4.790847301483154,
+      -5.037464618682861,
+      -4.602815628051758,
+      -4.5183281898498535,
+      -4.514697074890137,
+      -4.717558860778809,
+      -4.100062370300293,
+      -4.470508575439453,
+      -4.3069939613342285,
+      -4.153955936431885,
+      -3.684964179992676,
+      -3.7342634201049805,
+      -3.747032642364502,
+      -3.6904678344726562,
+      -3.405682325363159,
+      -3.685788154602051,
+      -3.850557565689087,
+      -3.6356136798858643,
+      -3.1476175785064697,
+      -3.251828908920288,
+      -3.135917901992798,
+      -3.240488290786743,
+      -2.811195135116577,
+      -2.6634764671325684,
+      -2.7416841983795166,
+      -2.5149338245391846,
+      -2.283905506134033,
+      -2.4918549060821533,
+      -2.5136289596557617,
+      -2.1629631519317627,
+      -2.510160446166992,
+      -2.2044715881347656,
+      -2.2444427013397217,
+      -2.527953863143921,
+      -1.9695202112197876,
+      -1.6617175340652466,
+      -2.2789957523345947,
+      -1.8233462572097778,
+      -1.653923511505127,
+      -1.7793313264846802,
+      -1.8508384227752686,
+      -1.6064250469207764,
+      -1.7450577020645142,
+      -1.7883766889572144,
+      -1.4824211597442627,
+      -1.5947823524475098,
+      -1.6336987018585205,
+      -1.444771647453308,
+      -1.6101677417755127,
+      -1.2629313468933105,
+      -1.7626909017562866,
+      -1.6012898683547974,
+      -1.4036366939544678,
+      -1.5126478672027588,
+      -1.446610927581787,
+      -1.29166579246521,
+      -1.2759038209915161,
+      -1.3668582439422607,
+      -1.4401066303253174,
+      -1.0013864040374756,
+      -0.8619095087051392,
+      -1.4348336458206177,
+      -2.4039411544799805,
+      -1.4596158266067505,
+      -1.398054599761963,
+      -1.828598976135254,
+      -1.2163857221603394,
+      -1.5147732496261597,
+      -1.6283231973648071,
+      -1.4824330806732178,
+      -1.472625970840454,
+      -1.42937171459198,
+      -1.1744720935821533,
+      -1.6057438850402832,
+      -1.3259679079055786,
+      -1.2305545806884766,
+      -1.3312289714813232,
+      -1.4408652782440186,
+      -1.5268746614456177,
+      -1.1499279737472534,
+      -1.3859295845031738,
+      -1.785064458847046,
+      -1.305832862854004,
+      -1.252516269683838,
+      -0.9547231793403625,
+      -1.7971051931381226,
+      -0.8948331475257874,
+      -1.3490352630615234,
+      -1.5993345975875854,
+      -1.225102424621582,
+      -1.217328429222107,
+      -1.1172164678573608,
+      -0.9223637580871582,
+      -1.349583625793457,
+      -0.6935772895812988,
+      -1.0280042886734009,
+      -1.1942574977874756,
+      -1.0854111909866333,
+      -0.8622591495513916,
+      -0.8715866208076477,
+      -0.9555827975273132,
+      -1.0593785047531128,
+      -0.9565988779067993,
+      -1.0662693977355957,
+      -0.9495633840560913,
+      -0.9095310568809509,
+      -1.3188884258270264,
+      -0.8761703372001648,
+      -0.8991231918334961,
+      -0.9201684594154358,
+      -0.9895074963569641,
+      -0.9261743426322937,
+      -0.6609992384910583,
+      -1.2606618404388428,
+      -0.8851873278617859,
+      -0.9310636520385742,
+      -0.5770999789237976,
+      -0.7159540057182312,
+      -0.9257229566574097,
+      -0.7062601447105408,
+      -0.9430025815963745,
+      -1.0685992240905762,
+      -0.7815626859664917,
+      -0.43587052822113037,
+      -0.6508090496063232,
+      -0.7445836067199707,
+      -0.962096095085144,
+      -0.858736515045166,
+      -0.7710210084915161,
+      -0.4534062445163727,
+      -0.6560110449790955,
+      -0.452778697013855,
+      -0.5636731386184692,
+      -0.8553899526596069,
+      -0.7786545157432556,
+      -0.6765991449356079,
+      -0.6170997023582458,
+      -0.8050151467323303,
+      -0.5370979905128479,
+      -0.9018643498420715,
+      -0.7443892359733582,
+      -0.817240297794342,
+      -0.7205577492713928,
+      -0.3836860656738281,
+      -0.663180947303772,
+      -0.5142239332199097,
+      -0.6271234750747681,
+      -0.35162416100502014,
+      -0.6422045826911926,
+      -0.5748727321624756,
+      -0.6405174732208252,
+      -0.5978391766548157,
+      -0.7506594061851501,
+      -0.41857194900512695,
+      -0.6290716528892517,
+      -0.5650010108947754,
+      -0.4634154140949249,
+      -0.4642519950866699,
+      -0.4609730839729309,
+      -0.5808088183403015,
+      -0.6225427985191345,
+      -0.6624266505241394,
+      -0.5210221409797668,
+      -0.6969188451766968,
+      -0.5200772285461426,
+      -0.6061062216758728,
+      -0.49697935581207275,
+      -0.5649558901786804,
+      -0.3923027217388153,
+      -0.44581127166748047,
+      -0.5210878252983093,
+      -0.5564239025115967,
+      -0.46670758724212646,
+      -0.4823647439479828,
+      -0.43254268169403076,
+      -0.4324648976325989,
+      -0.4741377830505371,
+      -0.8426257371902466,
+      -0.3482810854911804,
+      -0.39642080664634705,
+      -0.6418145895004272,
+      -0.5931370854377747,
+      -0.519498348236084,
+      -0.4377537667751312,
+      -0.60709547996521,
+      -0.5115737318992615,
+      -0.44077298045158386,
+      -0.5465641617774963,
+      -0.5352060794830322,
+      -0.5848107933998108,
+      -0.489984393119812,
+      -0.6054369807243347,
+      -0.4451814889907837,
+      -0.41332635283470154,
+      -0.5274278521537781,
+      -0.3277418911457062,
+      -0.4081784784793854,
+      -0.38111817836761475,
+      -0.34374675154685974,
+      -0.5816388130187988,
+      -0.4023304879665375,
+      -0.40927591919898987,
+      -0.4844159185886383,
+      -0.4173373579978943,
+      -0.35617947578430176,
+      -0.3747662603855133,
+      -0.4198073446750641,
+      -0.4224252700805664,
+      -0.4228634238243103,
+      -0.23087544739246368,
+      -0.5353215336799622,
+      -0.38394245505332947,
+      -0.46698734164237976,
+      -0.4426981508731842,
+      -0.5118622779846191,
+      -0.3715713620185852,
+      -0.24227172136306763,
+      -0.25442948937416077,
+      -0.3570634722709656,
+      -0.3031069040298462,
+      -0.8304105401039124,
+      -0.35720691084861755,
+      -0.2766706943511963,
+      -0.21187397837638855,
+      -0.3504505157470703,
+      -0.25292885303497314,
+      -0.2106543779373169,
+      -0.3711566925048828,
+      -0.2840205430984497,
+      -0.3919123709201813,
+      -0.35385793447494507,
+      -0.33695098757743835,
+      -0.368748277425766,
+      -0.41509464383125305,
+      -0.31920623779296875
+    ],
+    "base_loss": [
+      9.478815078735352,
+      7.147007465362549,
+      6.176869869232178,
+      4.685658931732178,
+      2.787061929702759,
+      2.0358169078826904,
+      1.877733826637268,
+      1.9838597774505615,
+      1.8801013231277466,
+      1.820031762123108,
+      1.879629373550415,
+      1.808982253074646,
+      1.842345952987671,
+      1.857324481010437,
+      1.866514801979065,
+      1.847004771232605,
+      1.7901628017425537,
+      1.8024442195892334,
+      1.8005038499832153,
+      1.7149734497070312,
+      1.657784104347229,
+      1.7428425550460815,
+      1.7641716003417969,
+      1.6597461700439453,
+      1.6715914011001587,
+      1.630522608757019,
+      1.6926891803741455,
+      1.5843595266342163,
+      1.5954675674438477,
+      1.6489832401275635,
+      1.5687618255615234,
+      1.6780462265014648,
+      1.6140142679214478,
+      1.5678704977035522,
+      1.5868467092514038,
+      1.5147616863250732,
+      1.6142460107803345,
+      1.6015769243240356,
+      1.6059281826019287,
+      1.5515403747558594,
+      1.5241461992263794,
+      1.6281310319900513,
+      1.522217035293579,
+      1.5715965032577515,
+      1.521254539489746,
+      1.6077793836593628,
+      1.5328742265701294,
+      1.5772770643234253,
+      1.4971659183502197,
+      1.516211748123169,
+      1.6045831441879272,
+      1.5815789699554443,
+      1.573682427406311,
+      1.5641165971755981,
+      1.4875094890594482,
+      1.4711979627609253,
+      1.41169273853302,
+      1.4731208086013794,
+      1.496317744255066,
+      1.5736018419265747,
+      1.497803807258606,
+      1.513519525527954,
+      1.3411214351654053,
+      1.294536828994751,
+      0.9521709084510803,
+      0.8137292265892029,
+      0.7460617423057556,
+      0.6637206077575684,
+      0.6727820634841919,
+      0.6252961158752441,
+      0.6120854616165161,
+      0.6280654668807983,
+      0.608968198299408,
+      0.6165491938591003,
+      0.6302626729011536,
+      0.5648792386054993,
+      0.5815601348876953,
+      0.5425162315368652,
+      0.5602009296417236,
+      0.5320432186126709,
+      0.5548504590988159,
+      0.47913244366645813,
+      0.5280306339263916,
+      0.5023998618125916,
+      0.4917462468147278,
+      0.443521648645401,
+      0.45187312364578247,
+      0.4393064081668854,
+      0.43677452206611633,
+      0.40351608395576477,
+      0.43782588839530945,
+      0.43998581171035767,
+      0.4332793354988098,
+      0.3735405504703522,
+      0.38679054379463196,
+      0.36955150961875916,
+      0.385150283575058,
+      0.32080963253974915,
+      0.31036800146102905,
+      0.3505243957042694,
+      0.30904823541641235,
+      0.2788742184638977,
+      0.2882492244243622,
+      0.30237480998039246,
+      0.24547524750232697,
+      0.3053366243839264,
+      0.259798526763916,
+      0.2723730802536011,
+      0.3113707900047302,
+      0.22505664825439453,
+      0.2224322110414505,
+      0.2788628339767456,
+      0.22666394710540771,
+      0.19256557524204254,
+      0.22168363630771637,
+      0.23763836920261383,
+      0.18722184002399445,
+      0.21815802156925201,
+      0.20957694947719574,
+      0.16867589950561523,
+      0.19812573492527008,
+      0.1907508224248886,
+      0.17580600082874298,
+      0.18482735753059387,
+      0.1696944236755371,
+      0.21134884655475616,
+      0.18224011361598969,
+      0.16670148074626923,
+      0.1823485940694809,
+      0.1764475256204605,
+      0.18433916568756104,
+      0.15989899635314941,
+      0.1958816945552826,
+      0.17056076228618622,
+      0.16027210652828217,
+      0.14637412130832672,
+      0.16548265516757965,
+      0.27574387192726135,
+      0.16816933453083038,
+      0.17710550129413605,
+      0.21935905516147614,
+      0.1367485523223877,
+      0.17449618875980377,
+      0.1823546141386032,
+      0.1821918785572052,
+      0.16603636741638184,
+      0.16063928604125977,
+      0.13537771999835968,
+      0.18368904292583466,
+      0.14854037761688232,
+      0.1395270675420761,
+      0.15166796743869781,
+      0.17095863819122314,
+      0.17842306196689606,
+      0.131307452917099,
+      0.15927840769290924,
+      0.20380905270576477,
+      0.149281308054924,
+      0.1428130716085434,
+      0.10824687033891678,
+      0.20649409294128418,
+      0.10403601080179214,
+      0.1521756649017334,
+      0.18943654000759125,
+      0.13651980459690094,
+      0.1377752125263214,
+      0.12476317584514618,
+      0.10559020936489105,
+      0.15177801251411438,
+      0.10150890797376633,
+      0.11630219966173172,
+      0.1332741230726242,
+      0.1367233544588089,
+      0.09795082360506058,
+      0.09837458282709122,
+      0.11475709825754166,
+      0.12007645517587662,
+      0.10695642977952957,
+      0.1194952055811882,
+      0.1135689988732338,
+      0.10178102552890778,
+      0.14920473098754883,
+      0.09827220439910889,
+      0.10088849067687988,
+      0.10268665105104446,
+      0.11169253289699554,
+      0.10481532663106918,
+      0.07450170069932938,
+      0.14566169679164886,
+      0.10045651346445084,
+      0.10866272449493408,
+      0.06537692993879318,
+      0.08126810938119888,
+      0.10344010591506958,
+      0.07878991216421127,
+      0.10556399077177048,
+      0.12452854216098785,
+      0.08715303987264633,
+      0.06283383071422577,
+      0.07278473675251007,
+      0.08324276655912399,
+      0.1073467880487442,
+      0.0959773063659668,
+      0.0903191789984703,
+      0.0734538659453392,
+      0.07329253107309341,
+      0.0613003671169281,
+      0.07355638593435287,
+      0.09880624711513519,
+      0.08773256838321686,
+      0.07558536529541016,
+      0.07866930961608887,
+      0.09029243141412735,
+      0.060021985322237015,
+      0.10877584666013718,
+      0.08304883539676666,
+      0.09336527436971664,
+      0.08078878372907639,
+      0.04604396969079971,
+      0.0739465281367302,
+      0.05788648873567581,
+      0.07136060297489166,
+      0.03972526267170906,
+      0.07363789528608322,
+      0.06858444213867188,
+      0.07210712879896164,
+      0.06665065139532089,
+      0.0842098519206047,
+      0.04666807875037193,
+      0.07024309784173965,
+      0.06373830139636993,
+      0.05268338695168495,
+      0.05181283503770828,
+      0.0532914437353611,
+      0.06561119109392166,
+      0.06976865977048874,
+      0.07406623661518097,
+      0.060560788959264755,
+      0.07924336940050125,
+      0.05855170264840126,
+      0.06876732409000397,
+      0.05866331234574318,
+      0.06375259906053543,
+      0.04413650557398796,
+      0.050514571368694305,
+      0.05819368362426758,
+      0.06327065825462341,
+      0.05219113826751709,
+      0.05453881248831749,
+      0.04892338067293167,
+      0.049468424171209335,
+      0.052953965961933136,
+      0.09524880349636078,
+      0.039073824882507324,
+      0.04476191848516464,
+      0.07190631330013275,
+      0.06800847500562668,
+      0.05787040665745735,
+      0.050239454954862595,
+      0.06768538057804108,
+      0.057207752019166946,
+      0.05006319656968117,
+      0.061503369361162186,
+      0.05985274165868759,
+      0.06554742902517319,
+      0.05472767353057861,
+      0.06759121268987656,
+      0.05012698844075203,
+      0.04650840535759926,
+      0.058920033276081085,
+      0.03701305016875267,
+      0.04568685218691826,
+      0.04329124093055725,
+      0.03899969533085823,
+      0.06514454632997513,
+      0.045412953943014145,
+      0.045837707817554474,
+      0.0540190152823925,
+      0.04707704111933708,
+      0.04026341438293457,
+      0.042074140161275864,
+      0.04687413573265076,
+      0.04716089740395546,
+      0.05027259513735771,
+      0.026406314224004745,
+      0.0597325824201107,
+      0.043301086872816086,
+      0.0521482415497303,
+      0.0506330169737339,
+      0.05718604102730751,
+      0.0414709709584713,
+      0.027818426489830017,
+      0.02862350083887577,
+      0.040087614208459854,
+      0.033949438482522964,
+      0.09281567484140396,
+      0.040391433984041214,
+      0.03121653012931347,
+      0.02385253645479679,
+      0.03970903158187866,
+      0.028319206088781357,
+      0.023725956678390503,
+      0.04406922310590744,
+      0.03261449560523033,
+      0.04449618607759476,
+      0.03972644731402397,
+      0.03816765546798706,
+      0.04165048152208328,
+      0.04628567025065422,
+      0.03601330518722534
+    ],
+    "info_loss": [
+      -0.24746036529541016,
+      -0.2758293151855469,
+      -0.23881292343139648,
+      -0.20038366317749023,
+      -0.1349635124206543,
+      -0.08571147918701172,
+      -0.09480726718902588,
+      -0.13600599765777588,
+      -0.11611413955688477,
+      -0.1656198501586914,
+      -0.21736717224121094,
+      -0.24530434608459473,
+      -0.24701166152954102,
+      -0.28169214725494385,
+      -0.2314901351928711,
+      -0.1858144998550415,
+      -0.23688781261444092,
+      -0.17470812797546387,
+      -0.19258201122283936,
+      -0.11244618892669678,
+      -0.08269965648651123,
+      -0.11517906188964844,
+      -0.1401442289352417,
+      -0.1294572353363037,
+      -0.17010819911956787,
+      -0.3923828601837158,
+      -0.705969512462616,
+      -0.7913603782653809,
+      -0.9512405395507812,
+      -1.029664158821106,
+      -1.066483497619629,
+      -1.2262372970581055,
+      -1.1775835752487183,
+      -1.1446707248687744,
+      -1.1663867235183716,
+      -1.163946270942688,
+      -1.2576193809509277,
+      -1.2948627471923828,
+      -1.332379698753357,
+      -1.2938930988311768,
+      -1.2832956314086914,
+      -1.3922922611236572,
+      -1.3484569787979126,
+      -1.3945118188858032,
+      -1.3569127321243286,
+      -1.468339443206787,
+      -1.4016437530517578,
+      -1.3889812231063843,
+      -1.3179041147232056,
+      -1.3906015157699585,
+      -1.4845846891403198,
+      -1.4269402027130127,
+      -1.4428510665893555,
+      -1.4374557733535767,
+      -1.3475759029388428,
+      -1.3586021661758423,
+      -1.344419240951538,
+      -1.3533596992492676,
+      -1.41006600856781,
+      -1.4885072708129883,
+      -1.4076002836227417,
+      -1.4455060958862305,
+      -1.246247410774231,
+      -1.2240954637527466,
+      -0.8900468349456787,
+      -0.7369465827941895,
+      -0.6641952991485596,
+      -0.5974841713905334,
+      -0.627163827419281,
+      -0.576034426689148,
+      -0.5380439162254333,
+      -0.5532349348068237,
+      -0.5536358952522278,
+      -0.5676578879356384,
+      -0.5917368531227112,
+      -0.5372372269630432,
+      -0.56263267993927,
+      -0.517175018787384,
+      -0.508733868598938,
+      -0.506136417388916,
+      -0.527775228023529,
+      -0.4586986005306244,
+      -0.5005542039871216,
+      -0.48131003975868225,
+      -0.46668779850006104,
+      -0.413947731256485,
+      -0.41922691464424133,
+      -0.419247567653656,
+      -0.41337156295776367,
+      -0.38181614875793457,
+      -0.41289258003234863,
+      -0.4295651912689209,
+      -0.40810590982437134,
+      -0.3561195135116577,
+      -0.36415907740592957,
+      -0.3510327637195587,
+      -0.36301058530807495,
+      -0.31381264328956604,
+      -0.2978881001472473,
+      -0.30990442633628845,
+      -0.2828633487224579,
+      -0.25747063755989075,
+      -0.27936798334121704,
+      -0.28309839963912964,
+      -0.24245284497737885,
+      -0.28460994362831116,
+      -0.24765269458293915,
+      -0.25302544236183167,
+      -0.28492802381515503,
+      -0.22052563726902008,
+      -0.18923313915729523,
+      -0.25695767998695374,
+      -0.2097332626581192,
+      -0.18484467267990112,
+      -0.20055602490901947,
+      -0.21083897352218628,
+      -0.18026942014694214,
+      -0.19725432991981506,
+      -0.20036013424396515,
+      -0.1662653386592865,
+      -0.17948386073112488,
+      -0.18314407765865326,
+      -0.16280633211135864,
+      -0.180500328540802,
+      -0.1438557207584381,
+      -0.1987953782081604,
+      -0.17859095335006714,
+      -0.16000238060951233,
+      -0.1707027405500412,
+      -0.1631038337945938,
+      -0.14900822937488556,
+      -0.14502517879009247,
+      -0.1574653685092926,
+      -0.16321077942848206,
+      -0.11656152456998825,
+      -0.10202033817768097,
+      -0.16051462292671204,
+      -0.2681931257247925,
+      -0.16319163143634796,
+      -0.15779775381088257,
+      -0.20554491877555847,
+      -0.13561710715293884,
+      -0.1709831953048706,
+      -0.1813998967409134,
+      -0.16671568155288696,
+      -0.16449113190174103,
+      -0.15988177061080933,
+      -0.13258007168769836,
+      -0.17931175231933594,
+      -0.148224338889122,
+      -0.13804210722446442,
+      -0.1484738290309906,
+      -0.16194096207618713,
+      -0.17081204056739807,
+      -0.12893566489219666,
+      -0.1547948271036148,
+      -0.19968445599079132,
+      -0.14876215159893036,
+      -0.14158760011196136,
+      -0.10761424899101257,
+      -0.20079588890075684,
+      -0.10236789286136627,
+      -0.15164384245872498,
+      -0.17914894223213196,
+      -0.13636644184589386,
+      -0.13589240610599518,
+      -0.12452322244644165,
+      -0.10313521325588226,
+      -0.15057386457920074,
+      -0.07971213012933731,
+      -0.11526139080524445,
+      -0.1332167536020279,
+      -0.12428569793701172,
+      -0.0971180722117424,
+      -0.09776707738637924,
+      -0.10750783979892731,
+      -0.11857262998819351,
+      -0.1066167801618576,
+      -0.11932756751775742,
+      -0.10674172639846802,
+      -0.10168048739433289,
+      -0.14808465540409088,
+      -0.09817315638065338,
+      -0.10079306364059448,
+      -0.10262572020292282,
+      -0.11101125925779343,
+      -0.1037193089723587,
+      -0.07370161265134811,
+      -0.1410427838563919,
+      -0.09944266825914383,
+      -0.10417608916759491,
+      -0.06530590355396271,
+      -0.0803341269493103,
+      -0.10331270843744278,
+      -0.0786375105381012,
+      -0.1054707020521164,
+      -0.119558185338974,
+      -0.08709551393985748,
+      -0.05025278776884079,
+      -0.07257865369319916,
+      -0.08294828981161118,
+      -0.107086680829525,
+      -0.09570429474115372,
+      -0.08692388236522675,
+      -0.052821774035692215,
+      -0.07326008379459381,
+      -0.0516117662191391,
+      -0.06417225301265717,
+      -0.09715473651885986,
+      -0.08743984252214432,
+      -0.07549464702606201,
+      -0.06975524127483368,
+      -0.09015349298715591,
+      -0.05998589098453522,
+      -0.10181828588247299,
+      -0.08298134803771973,
+      -0.09185295552015305,
+      -0.08075430244207382,
+      -0.043630681931972504,
+      -0.07384850084781647,
+      -0.05750010162591934,
+      -0.07025234401226044,
+      -0.039659470319747925,
+      -0.0735572874546051,
+      -0.06451164186000824,
+      -0.07173311710357666,
+      -0.06661474704742432,
+      -0.08418307453393936,
+      -0.04665147513151169,
+      -0.07021082937717438,
+      -0.0637119933962822,
+      -0.05267366021871567,
+      -0.05179346725344658,
+      -0.05323582515120506,
+      -0.06519252806901932,
+      -0.06972390413284302,
+      -0.07403936237096786,
+      -0.058382898569107056,
+      -0.07815470546483994,
+      -0.05841157212853432,
+      -0.06865391135215759,
+      -0.055786989629268646,
+      -0.06329059600830078,
+      -0.044128019362688065,
+      -0.050502464175224304,
+      -0.05816315859556198,
+      -0.06325214356184006,
+      -0.05217985436320305,
+      -0.054528024047613144,
+      -0.048883337527513504,
+      -0.048519305884838104,
+      -0.052907995879650116,
+      -0.09503248333930969,
+      -0.03906627744436264,
+      -0.04475462809205055,
+      -0.07161007076501846,
+      -0.0666651800274849,
+      -0.05786250904202461,
+      -0.050222866237163544,
+      -0.06761092692613602,
+      -0.057174813002347946,
+      -0.04933343827724457,
+      -0.061432238668203354,
+      -0.05984765663743019,
+      -0.06550679355859756,
+      -0.054724521934986115,
+      -0.06751931458711624,
+      -0.04966246709227562,
+      -0.046499382704496384,
+      -0.05887816101312637,
+      -0.036921266466379166,
+      -0.04566673934459686,
+      -0.04327061027288437,
+      -0.03899180516600609,
+      -0.06513749063014984,
+      -0.04527737572789192,
+      -0.04579845815896988,
+      -0.05398871377110481,
+      -0.047070346772670746,
+      -0.040253836661577225,
+      -0.04206092283129692,
+      -0.04686303064227104,
+      -0.04713711515069008,
+      -0.04755039140582085,
+      -0.026394391432404518,
+      -0.059719450771808624,
+      -0.043288592249155045,
+      -0.05214283987879753,
+      -0.050238627940416336,
+      -0.057172250002622604,
+      -0.04146181046962738,
+      -0.027805551886558533,
+      -0.02861672081053257,
+      -0.04002641886472702,
+      -0.03390965238213539,
+      -0.09281256049871445,
+      -0.040354739874601364,
+      -0.031207451596856117,
+      -0.023845793679356575,
+      -0.03969898819923401,
+      -0.02830086648464203,
+      -0.023720722645521164,
+      -0.04318676516413689,
+      -0.03261061757802963,
+      -0.044458597898483276,
+      -0.03972158581018448,
+      -0.038163602352142334,
+      -0.041647665202617645,
+      -0.046281177550554276,
+      -0.03601086512207985
+    ],
+    "abs_loss": [
+      1.596919059753418,
+      1.4947857856750488,
+      1.272780418395996,
+      1.1532820463180542,
+      1.0954599380493164,
+      1.0691957473754883,
+      1.0738341808319092,
+      1.0862774848937988,
+      1.076493263244629,
+      1.0707249641418457,
+      1.0633931159973145,
+      1.0700206756591797,
+      1.04740571975708,
+      0.7284477949142456,
+      0.27670353651046753,
+      0.3479034900665283,
+      0.3737618625164032,
+      0.2770856022834778,
+      0.25724223256111145,
+      0.21221217513084412,
+      0.18176080286502838,
+      0.2101953625679016,
+      0.16143487393856049,
+      0.21087051928043365,
+      0.18534056842327118,
+      0.2236458957195282,
+      0.1967623233795166,
+      0.2382446974515915,
+      0.2674140930175781,
+      0.26551109552383423,
+      0.1889098882675171,
+      0.2353925108909607,
+      0.18361970782279968,
+      0.19251857697963715,
+      0.19663862884044647,
+      0.21570132672786713,
+      0.15264488756656647,
+      0.23524253070354462,
+      0.20087885856628418,
+      0.19272246956825256,
+      0.17692875862121582,
+      0.18186329305171967,
+      0.1845722794532776,
+      0.1634385734796524,
+      0.1223602145910263,
+      0.0807499960064888,
+      0.08542364090681076,
+      0.08870146423578262,
+      0.07707524299621582,
+      0.07842298597097397,
+      0.056058961898088455,
+      0.04188915342092514,
+      0.03148265182971954,
+      0.034343309700489044,
+      0.04402808099985123,
+      0.05053535848855972,
+      0.03719761222600937,
+      0.03085540235042572,
+      0.030315913259983063,
+      0.03826269879937172,
+      0.018028227612376213,
+      0.01966400444507599,
+      0.025648467242717743,
+      0.0174420103430748,
+      0.02302210032939911,
+      0.01107926107943058,
+      0.011899472214281559,
+      0.025297150015830994,
+      0.006678640376776457,
+      0.025592511519789696,
+      0.021416539326310158,
+      0.003772859927266836,
+      0.004001403693109751,
+      0.015317991375923157,
+      0.004922089166939259,
+      0.014212886802852154,
+      0.006655131932348013,
+      0.0011241815518587828,
+      0.005378206260502338,
+      0.01237943023443222,
+      0.022908588871359825,
+      0.007470221258699894,
+      0.005204240325838327,
+      0.002487875521183014,
+      0.010804742574691772,
+      0.012848813086748123,
+      0.0012782838894054294,
+      0.026987312361598015,
+      0.017897434532642365,
+      0.0032508480362594128,
+      0.0018659287597984076,
+      0.014394900761544704,
+      0.003501119092106819,
+      0.006911094766110182,
+      0.009087074548006058,
+      0.0016418073792010546,
+      0.004669916816055775,
+      0.019071029499173164,
+      0.005373222753405571,
+      0.0006761085242033005,
+      0.00045162433525547385,
+      0.006719629280269146,
+      0.007513158954679966,
+      0.002733794739469886,
+      0.04615173116326332,
+      0.006636207923293114,
+      0.00824967585504055,
+      0.004405905492603779,
+      0.005903213284909725,
+      0.01138284895569086,
+      0.0025351380463689566,
+      0.06793241202831268,
+      0.005575981922447681,
+      0.005631899926811457,
+      0.0036704731173813343,
+      0.010995674878358841,
+      0.006839643232524395,
+      0.0016634350176900625,
+      0.00783868320286274,
+      0.0003229841822758317,
+      0.0012815918307751417,
+      0.002405180362984538,
+      0.007452835328876972,
+      0.010544407181441784,
+      0.0028931081760674715,
+      0.004344127606600523,
+      0.002171870553866029,
+      0.00365426461212337,
+      0.011692658066749573,
+      0.0009850180940702558,
+      0.0026010049041360617,
+      0.00044118365622125566,
+      0.0003429003991186619,
+      0.0060530430637300014,
+      0.0007225353037938476,
+      0.0048428988084197044,
+      0.008732423186302185,
+      0.0014445744454860687,
+      0.018670761957764626,
+      0.00017304674838669598,
+      0.009912424720823765,
+      0.00906207226216793,
+      0.005737516097724438,
+      0.0067735351622104645,
+      0.0011667599901556969,
+      0.00191461481153965,
+      0.00020122816204093397,
+      0.001410648226737976,
+      0.006405932828783989,
+      0.0015682578086853027,
+      0.001102718524634838,
+      8.373481978196651e-05,
+      0.009828781709074974,
+      0.0020935870707035065,
+      0.0002035672659985721,
+      0.00022269962937571108,
+      0.0001766529749147594,
+      0.00014260719763115048,
+      0.012527580372989178,
+      1.0576462955214083e-05,
+      0.01100655272603035,
+      8.591853838879615e-05,
+      0.006023123860359192,
+      0.0008255345746874809,
+      2.0563868019962683e-05,
+      0.004955709911882877,
+      0.001584604731760919,
+      0.0050061955116689205,
+      0.00016951750149019063,
+      3.583452053135261e-05,
+      0.00014310883125290275,
+      0.0001688483462203294,
+      0.007586191408336163,
+      0.0011576192919164896,
+      3.7255649658618495e-05,
+      6.426421168725938e-05,
+      0.00040817269473336637,
+      0.006447537802159786,
+      0.03395574912428856,
+      0.0006136471638455987,
+      0.005575705785304308,
+      0.0032049526926130056,
+      0.010097612626850605,
+      0.006191258784383535,
+      0.005436952691525221,
+      0.01112685352563858,
+      0.0045395283959805965,
+      8.249337406596169e-05,
+      0.004922164604067802,
+      0.005476090125739574,
+      0.0017399060307070613,
+      0.007035619579255581,
+      0.0007214326178655028,
+      2.291693090228364e-05,
+      3.1971553653420415e-06,
+      0.00023343230714090168,
+      0.0002320274943485856,
+      0.0003754912759177387,
+      0.003981406334787607,
+      0.00023642885207664222,
+      3.009071406268049e-06,
+      1.161950876849005e-05,
+      8.238548616645858e-05,
+      0.0009311531321145594,
+      0.00017720420146360993,
+      0.004855811595916748,
+      1.2067213901900686e-05,
+      0.00033638064633123577,
+      1.394089667883236e-05,
+      0.0009420081041753292,
+      0.0002318583137821406,
+      0.00034358506673015654,
+      0.007310624700039625,
+      0.00010950634896289557,
+      0.004586211871355772,
+      2.008353294513654e-05,
+      0.0010729751083999872,
+      0.00040907826041802764,
+      0.00013855722500011325,
+      8.351419819518924e-05,
+      5.036394668422872e-06,
+      0.0009947380749508739,
+      6.696867785649374e-05,
+      0.0020369698759168386,
+      1.8229318811791018e-05,
+      0.0001292597153224051,
+      0.00010432380076963454,
+      6.665389719273662e-06,
+      0.0002083564322674647,
+      0.0004092064336873591,
+      7.959028152981773e-06,
+      2.3773649445502087e-05,
+      0.0038372615817934275,
+      0.0002493303909432143,
+      5.2793920986005105e-06,
+      4.785179771715775e-05,
+      0.00028111590654589236,
+      3.5398265936237294e-06,
+      0.0004384462081361562,
+      7.704014933551662e-06,
+      7.928567356429994e-06,
+      2.565624708950054e-05,
+      0.0001698183041298762,
+      9.799407052923925e-06,
+      9.231342119164765e-05,
+      5.7054494391195476e-05,
+      3.831291905953549e-06,
+      2.336510078748688e-05,
+      1.7051501345122233e-05,
+      4.266294126864523e-06,
+      5.003729529562406e-05,
+      1.928949677676428e-05,
+      0.00017112484783865511,
+      6.942585969227366e-06,
+      7.328124047489837e-05,
+      5.447128387459088e-06,
+      1.1434322004788555e-05,
+      6.850100180599838e-05,
+      0.0016534426249563694,
+      8.242504009103868e-06,
+      1.7369038687320426e-06,
+      0.0019085220992565155,
+      0.0008398008649237454,
+      0.0006672751624137163,
+      0.00375841255299747,
+      9.171140845865011e-06,
+      0.00024457444669678807,
+      0.0005654327687807381,
+      6.293559636105783e-06,
+      3.222317900508642e-06,
+      3.5538059819373302e-06,
+      1.044741384248482e-05,
+      2.5573756374797085e-06,
+      7.579227258247556e-06,
+      0.00015041269944049418,
+      8.011316094780341e-05,
+      3.87199834221974e-05,
+      5.500045972439693e-06,
+      1.005646845442243e-05,
+      0.010801905766129494,
+      2.8740305424435064e-06,
+      0.0003652515879366547,
+      8.872371108736843e-05,
+      1.3331699847185519e-05,
+      0.0003143040812574327,
+      3.1608587960363366e-06,
+      4.033338427689159e-06,
+      5.55466249352321e-06,
+      5.573667294811457e-06,
+      1.3822506844007876e-05,
+      5.380641596275382e-06,
+      0.0002552082878537476,
+      0.00014873349573463202,
+      0.00019721119315363467,
+      3.990438472101232e-06,
+      3.1747381399327423e-06,
+      1.3066473002254497e-05,
+      1.3943534213467501e-05,
+      4.306153641664423e-06,
+      0.003424054477363825,
+      2.1004128939239308e-05,
+      3.551704139681533e-05,
+      0.0007386493380181491,
+      2.6840225473279133e-06,
+      2.487375240889378e-05,
+      2.2416193132812623e-06,
+      0.0058481995947659016,
+      0.004598032217472792,
+      1.1445843028923264e-06,
+      4.5684268116019666e-05
+    ],
+    "zipf_loss": [
+      11.285626411437988,
+      8.5469388961792,
+      5.320335388183594,
+      3.767218589782715,
+      3.1796300411224365,
+      3.0120468139648438,
+      2.9520602226257324,
+      2.9325332641601562,
+      2.923217535018921,
+      2.9199328422546387,
+      2.9185163974761963,
+      2.9129607677459717,
+      2.883033514022827,
+      2.009091377258301,
+      0.24353228509426117,
+      0.1941249668598175,
+      0.04749487712979317,
+      0.03600744158029556,
+      0.026689842343330383,
+      0.01742704026401043,
+      0.014155315235257149,
+      0.017024187371134758,
+      0.012707581743597984,
+      0.01886345073580742,
+      0.01684136502444744,
+      0.027663227170705795,
+      0.045945003628730774,
+      0.05080542340874672,
+      0.05410684645175934,
+      0.04384762421250343,
+      0.03915369510650635,
+      0.03875356912612915,
+      0.02771097980439663,
+      0.021731721237301826,
+      0.025930888950824738,
+      0.027046270668506622,
+      0.022006195038557053,
+      0.01902031898498535,
+      0.020905204117298126,
+      0.020855877548456192,
+      0.015282095409929752,
+      0.01529134251177311,
+      0.014604698866605759,
+      0.013249149546027184,
+      0.028260793536901474,
+      0.012099381536245346,
+      0.022192221134901047,
+      0.010063612833619118,
+      0.01127258874475956,
+      0.012551696039736271,
+      0.02037505991756916,
+      0.00827052816748619,
+      0.006829372141510248,
+      0.0100149717181921,
+      0.0266830176115036,
+      0.012799793854355812,
+      0.023570306599140167,
+      0.016914743930101395,
+      0.01730828545987606,
+      0.00875080469995737,
+      0.013662377372384071,
+      0.01715860143303871,
+      0.009965881705284119,
+      0.004956849385052919,
+      0.004832414910197258,
+      0.009061310440301895,
+      0.012197121977806091,
+      0.006544884294271469,
+      0.020408371463418007,
+      0.039887260645627975,
+      0.012988101691007614,
+      0.0032295328564941883,
+      0.00932139903306961,
+      0.002145064063370228,
+      0.010843830183148384,
+      0.015223775058984756,
+      0.006636364385485649,
+      0.0263056643307209,
+      0.008271460421383381,
+      0.013385691680014133,
+      0.0030522430315613747,
+      0.0070436131209135056,
+      0.0064826663583517075,
+      0.0034574205055832863,
+      0.02009494975209236,
+      0.009706387296319008,
+      0.006005052477121353,
+      0.0034380692522972822,
+      0.004683607257902622,
+      0.008637895807623863,
+      0.005124982446432114,
+      0.0036689992994070053,
+      0.011816032230854034,
+      0.03934585303068161,
+      0.002062798012048006,
+      0.004693880677223206,
+      0.004000416025519371,
+      0.004214541986584663,
+      0.004499017260968685,
+      0.0067680589854717255,
+      0.004606388509273529,
+      0.011254856362938881,
+      0.012824291363358498,
+      0.014706926420331001,
+      0.011474795639514923,
+      0.029938679188489914,
+      0.011431977152824402,
+      0.0129978246986866,
+      0.009365284815430641,
+      0.009541323408484459,
+      0.007928065955638885,
+      0.004924816079437733,
+      0.04676493629813194,
+      0.0013944688253104687,
+      0.0041782185435295105,
+      0.018813494592905045,
+      0.008363252505660057,
+      0.009161192923784256,
+      0.004863737151026726,
+      0.011524049565196037,
+      0.0018024137243628502,
+      0.006750702857971191,
+      0.0067403484135866165,
+      0.008953789249062538,
+      0.005642112344503403,
+      0.013479547575116158,
+      0.0021623901557177305,
+      0.029320240020751953,
+      0.010861575603485107,
+      0.00788138434290886,
+      0.013817166909575462,
+      0.014404838904738426,
+      0.01187937706708908,
+      0.020834989845752716,
+      0.003884402569383383,
+      0.011435426771640778,
+      0.003956676926463842,
+      0.0021015757229179144,
+      0.002263993490487337,
+      0.0028001414611935616,
+      0.00649994844570756,
+      0.0021306348498910666,
+      0.019988827407360077,
+      0.00264377286657691,
+      0.0024151471443474293,
+      0.006057506427168846,
+      0.008786540478467941,
+      0.015809718519449234,
+      0.0030439659021794796,
+      0.007578206714242697,
+      0.010229166597127914,
+      0.001833135262131691,
+      0.006602782756090164,
+      0.002613284159451723,
+      0.008100843988358974,
+      0.00271791685372591,
+      0.007953394204378128,
+      0.03249310329556465,
+      0.019293881952762604,
+      0.01317145861685276,
+      0.003258969634771347,
+      0.024801120162010193,
+      0.014625225216150284,
+      0.002635540906339884,
+      0.002040124498307705,
+      0.0033249035477638245,
+      0.0030941711738705635,
+      0.0028975531458854675,
+      0.00436011515557766,
+      0.002031490206718445,
+      0.008293037302792072,
+      0.0046190437860786915,
+      0.0199637021869421,
+      0.010855032131075859,
+      0.007705803960561752,
+      0.00473202858120203,
+      0.00623058807104826,
+      0.0019678021781146526,
+      0.004115616902709007,
+      0.00422345194965601,
+      0.004935308359563351,
+      0.012432852759957314,
+      0.0062792375683784485,
+      0.007299851626157761,
+      0.002858368679881096,
+      0.007799813523888588,
+      0.005749467760324478,
+      0.0015069943619892001,
+      0.003612073138356209,
+      0.008235236629843712,
+      0.0018605850636959076,
+      0.00987850222736597,
+      0.006047021597623825,
+      0.00396173819899559,
+      0.0013247602619230747,
+      0.006117135286331177,
+      0.002430863678455353,
+      0.0022018789313733578,
+      0.003425393719226122,
+      0.002169111743569374,
+      0.0016562379896640778,
+      0.0014228317886590958,
+      0.002320891013368964,
+      0.0078055839985609055,
+      0.0013398686423897743,
+      0.0028115883469581604,
+      0.0020374455489218235,
+      0.004459431394934654,
+      0.017349809408187866,
+      0.007917150855064392,
+      0.0027387626469135284,
+      0.0017491262406110764,
+      0.005496285855770111,
+      0.002727912738919258,
+      0.007084077224135399,
+      0.0023733749985694885,
+      0.007816703990101814,
+      0.006155618466436863,
+      0.006562959402799606,
+      0.001349154394119978,
+      0.0028901174664497375,
+      0.003939938731491566,
+      0.0052385590970516205,
+      0.019526738673448563,
+      0.0016574084293097258,
+      0.004693618975579739,
+      0.0016472507268190384,
+      0.0069608259946107864,
+      0.0012539103627204895,
+      0.002752535045146942,
+      0.008379869163036346,
+      0.010635429993271828,
+      0.0014861254021525383,
+      0.01806875504553318,
+      0.005504699423909187,
+      0.004922785796225071,
+      0.0038726171478629112,
+      0.0022457530722022057,
+      0.005341008305549622,
+      0.005486038513481617,
+      0.011664791032671928,
+      0.002224674681201577,
+      0.004180466756224632,
+      0.004839997738599777,
+      0.008689593523740768,
+      0.002344392938539386,
+      0.012826589867472649,
+      0.0028975000604987144,
+      0.008374970406293869,
+      0.007366897538304329,
+      0.003254727926105261,
+      0.001986307092010975,
+      0.012433165684342384,
+      0.003307197242975235,
+      0.006356198340654373,
+      0.002379254437983036,
+      0.005505101755261421,
+      0.0012494893744587898,
+      0.014070112258195877,
+      0.0013275519013404846,
+      0.002966462168842554,
+      0.0023073574993759394,
+      0.00617084838449955,
+      0.0033510299399495125,
+      0.00433387141674757,
+      0.0025322227738797665,
+      0.0021405271254479885,
+      0.001259658019989729,
+      0.005158456042408943,
+      0.002433418994769454,
+      0.004457367584109306,
+      0.002801033202558756,
+      0.008296439424157143,
+      0.007170850411057472,
+      0.004576575011014938,
+      0.0050222985446453094,
+      0.002867069561034441,
+      0.001451676245778799,
+      0.0062880851328372955,
+      0.005015293136239052,
+      0.00376853346824646,
+      0.0019123069941997528,
+      0.0017760999035090208,
+      0.0023665805347263813,
+      0.006630713120102882,
+      0.002140127355232835,
+      0.005641967058181763,
+      0.0022922551725059748,
+      0.00905453972518444,
+      0.002672804519534111,
+      0.001575235277414322,
+      0.007939880713820457,
+      0.0030993595719337463,
+      0.0030933814123272896,
+      0.0020398027263581753,
+      0.00489910040050745,
+      0.005947738885879517,
+      0.004185874946415424,
+      0.0027309805154800415,
+      0.00648793950676918,
+      0.0017585107125341892,
+      0.002823358401656151,
+      0.01656787097454071,
+      0.009470885619521141,
+      0.008174968883395195,
+      0.0036312490701675415,
+      0.005932582542300224,
+      0.005618139170110226,
+      0.0014313401188701391,
+      0.004884552210569382
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.6644464731216431,
+      0.390718936920166,
+      0.3397873342037201,
+      0.220010444521904,
+      0.17606858909130096,
+      0.11906886100769043,
+      0.09468584507703781,
+      0.0884234756231308,
+      0.06945794820785522,
+      0.0574568510055542,
+      0.04932135343551636,
+      0.06153758242726326,
+      0.05611604079604149,
+      0.05805246904492378,
+      0.07358357310295105,
+      0.05414161831140518,
+      0.05728568509221077,
+      0.05426923185586929,
+      0.05367695167660713,
+      0.05449531227350235,
+      0.06227414682507515,
+      0.0533452033996582,
+      0.04716930538415909,
+      0.0462658666074276,
+      0.05089081451296806,
+      0.04803353548049927,
+      0.044982124119997025,
+      0.05346018075942993,
+      0.05227212980389595,
+      0.05755770206451416,
+      0.07542824745178223,
+      0.09152878820896149,
+      0.10082665830850601,
+      0.09812896698713303,
+      0.12396321445703506,
+      0.12976472079753876,
+      0.10502009838819504,
+      0.11594293266534805,
+      0.12908928096294403,
+      0.1413189023733139,
+      0.13866053521633148,
+      0.12943346798419952,
+      0.13647392392158508,
+      0.1408652812242508,
+      0.15507225692272186,
+      0.16842518746852875,
+      0.2103387862443924,
+      0.20515666902065277,
+      0.21254098415374756,
+      0.21778546273708344,
+      0.22489629685878754,
+      0.23182642459869385,
+      0.2397346943616867,
+      0.2505520284175873,
+      0.25581789016723633,
+      0.2709047496318817,
+      0.2767433822154999,
+      0.28250429034233093,
+      0.27925848960876465,
+      0.29402247071266174,
+      0.29541489481925964,
+      0.30194178223609924,
+      0.2817853093147278,
+      0.29107022285461426,
+      0.2928965985774994,
+      0.30889639258384705,
+      0.30842041969299316,
+      0.30617886781692505,
+      0.3055952489376068,
+      0.3025607466697693,
+      0.3075685501098633,
+      0.3120739758014679,
+      0.320017546415329,
+      0.323820024728775,
+      0.31776610016822815,
+      0.3086092174053192,
+      0.3121654987335205,
+      0.32862967252731323,
+      0.32827210426330566,
+      0.3089290261268616,
+      0.3160431981086731,
+      0.3320402503013611,
+      0.33413752913475037,
+      0.3475281000137329,
+      0.34444400668144226,
+      0.3437744081020355,
+      0.340019553899765,
+      0.3456742465496063,
+      0.35318875312805176,
+      0.3524748682975769,
+      0.3447343111038208,
+      0.3309643268585205,
+      0.3429817855358124,
+      0.3501330316066742,
+      0.37218067049980164,
+      0.3503168821334839,
+      0.34891870617866516,
+      0.35859641432762146,
+      0.35393843054771423,
+      0.35644397139549255,
+      0.3608522117137909,
+      0.36008793115615845,
+      0.3715854287147522,
+      0.36290615797042847,
+      0.35269445180892944,
+      0.3623688220977783,
+      0.37372899055480957,
+      0.3635806739330292,
+      0.3644300401210785,
+      0.3613739013671875,
+      0.3820236027240753,
+      0.3934689462184906,
+      0.40816766023635864,
+      0.39851799607276917,
+      0.38966116309165955,
+      0.39526575803756714,
+      0.39007678627967834,
+      0.3972146511077881,
+      0.40363237261772156,
+      0.3965681791305542,
+      0.3984060287475586,
+      0.40615272521972656,
+      0.3946124017238617,
+      0.3920044004917145,
+      0.40209197998046875,
+      0.4067413806915283,
+      0.40874406695365906,
+      0.4083837568759918,
+      0.40983501076698303,
+      0.40673351287841797,
+      0.4112052023410797,
+      0.4046363830566406,
+      0.42072486877441406,
+      0.4345296025276184,
+      0.43328163027763367,
+      0.4279066026210785,
+      0.41759130358695984,
+      0.4037562906742096,
+      0.4145204722881317,
+      0.4159364402294159,
+      0.409978449344635,
+      0.40836572647094727,
+      0.42102542519569397,
+      0.4176710247993469,
+      0.4233308732509613,
+      0.41641727089881897,
+      0.437593936920166,
+      0.43382301926612854,
+      0.4270397126674652,
+      0.43054914474487305,
+      0.427566260099411,
+      0.4299742877483368,
+      0.43167996406555176,
+      0.43732354044914246,
+      0.4419519603252411,
+      0.44340643286705017,
+      0.44566842913627625,
+      0.4446880519390106,
+      0.4471912384033203,
+      0.4491994380950928,
+      0.45555001497268677,
+      0.4542686939239502,
+      0.43496283888816833,
+      0.4431009292602539,
+      0.4456893503665924,
+      0.4535500705242157,
+      0.45205479860305786,
+      0.44086360931396484,
+      0.46105995774269104,
+      0.4644389748573303,
+      0.4546182155609131,
+      0.4628783166408539,
+      0.463123619556427,
+      0.4668252468109131,
+      0.4742497503757477,
+      0.4820367395877838,
+      0.4765673577785492,
+      0.47878319025039673,
+      0.4698505401611328,
+      0.4846014082431793,
+      0.48912617564201355,
+      0.4972328245639801,
+      0.5014645457267761,
+      0.5092229247093201,
+      0.5129060745239258,
+      0.522394597530365,
+      0.5310458540916443,
+      0.5425419807434082,
+      0.5471708178520203,
+      0.5470795631408691,
+      0.5413408279418945,
+      0.5409379005432129,
+      0.5409113764762878,
+      0.537050724029541,
+      0.5376296639442444,
+      0.5384513139724731,
+      0.5365539789199829,
+      0.5468458533287048,
+      0.5518631339073181,
+      0.5440913438796997,
+      0.5436188578605652,
+      0.5479923486709595,
+      0.5452127456665039,
+      0.5501694083213806,
+      0.5610882639884949,
+      0.556338906288147,
+      0.5536699295043945,
+      0.5646674036979675,
+      0.5674131512641907,
+      0.5633151531219482,
+      0.567596435546875,
+      0.564018726348877,
+      0.5691948533058167,
+      0.5586506724357605,
+      0.5658769607543945,
+      0.5602114796638489,
+      0.5630201697349548,
+      0.5669975876808167,
+      0.5680027008056641,
+      0.562749445438385,
+      0.5649909377098083,
+      0.5620891451835632,
+      0.5636012554168701,
+      0.5655064582824707,
+      0.5570403337478638,
+      0.5550257563591003,
+      0.5560483336448669,
+      0.5633824467658997,
+      0.5675045847892761,
+      0.5643922686576843,
+      0.5696682333946228,
+      0.5690222978591919,
+      0.5665627717971802,
+      0.5666993260383606,
+      0.5652199983596802,
+      0.5618746876716614,
+      0.5575634837150574,
+      0.5577389001846313,
+      0.5654399394989014,
+      0.5689845085144043,
+      0.5666268467903137,
+      0.5639982223510742,
+      0.5644475817680359,
+      0.5620675086975098,
+      0.564915657043457,
+      0.5630634427070618,
+      0.5648998618125916,
+      0.5590072870254517,
+      0.5604531764984131,
+      0.5593754649162292,
+      0.5623536109924316,
+      0.5646090507507324,
+      0.5602173209190369,
+      0.5559281706809998,
+      0.5578214526176453,
+      0.5604626536369324,
+      0.5620313882827759,
+      0.5616292953491211,
+      0.5635948181152344,
+      0.5586933493614197,
+      0.5560607314109802,
+      0.554778516292572,
+      0.5548190474510193,
+      0.5562477707862854,
+      0.5549036860466003,
+      0.5571350455284119,
+      0.555806577205658,
+      0.5569379925727844,
+      0.5573353171348572,
+      0.5563571453094482,
+      0.5522323846817017,
+      0.5545682311058044,
+      0.5520962476730347,
+      0.5536748766899109,
+      0.5544080138206482,
+      0.550173282623291,
+      0.551995575428009,
+      0.5538771152496338,
+      0.5502065420150757,
+      0.5552479028701782,
+      0.5531644225120544,
+      0.552257239818573,
+      0.5502626299858093,
+      0.5510493516921997,
+      0.5554757118225098,
+      0.5564853549003601,
+      0.5578190684318542,
+      0.5565758943557739,
+      0.5565757155418396,
+      0.5545722842216492,
+      0.5547272562980652,
+      0.5568007826805115,
+      0.5567353367805481,
+      0.5578407645225525,
+      0.5593713521957397,
+      0.5583187937736511,
+      0.5594400763511658,
+      0.5608003735542297,
+      0.5613595247268677,
+      0.5619935989379883,
+      0.5630896687507629,
+      0.5623959302902222,
+      0.5624557137489319,
+      0.5634995698928833,
+      0.5621915459632874,
+      0.5627752542495728,
+      0.5633371472358704,
+      0.5640240907669067,
+      0.5638148188591003,
+      0.5640226006507874
+    ],
+    "lr": [
+      8.376068376068378e-06,
+      1.6923076923076924e-05,
+      2.5470085470085475e-05,
+      3.401709401709402e-05,
+      4.2564102564102564e-05,
+      5.1111111111111115e-05,
+      5.9658119658119666e-05,
+      6.820512820512821e-05,
+      7.675213675213675e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      8e-05,
+      7.985991294024534e-05,
+      7.926632370399684e-05,
+      7.867273446774832e-05,
+      7.80791452314998e-05,
+      7.74855559952513e-05,
+      7.689196675900277e-05,
+      7.629837752275426e-05,
+      7.570478828650574e-05,
+      7.511119905025723e-05,
+      7.45176098140087e-05,
+      7.392402057776018e-05,
+      7.33304313415117e-05,
+      7.273684210526317e-05,
+      7.214325286901464e-05,
+      7.154966363276613e-05,
+      7.095607439651762e-05,
+      7.03624851602691e-05,
+      6.976889592402057e-05,
+      6.917530668777208e-05,
+      6.858171745152355e-05,
+      6.798812821527503e-05,
+      6.739453897902653e-05,
+      6.6800949742778e-05,
+      6.620736050652948e-05,
+      6.561377127028096e-05,
+      6.502018203403246e-05,
+      6.442659279778393e-05,
+      6.383300356153542e-05,
+      6.308508112386229e-05,
+      6.249149188761377e-05,
+      6.189790265136525e-05,
+      6.130431341511676e-05,
+      6.0710724178868226e-05,
+      6.011713494261971e-05,
+      5.95235457063712e-05,
+      5.892995647012267e-05,
+      5.8336367233874154e-05,
+      5.7742777997625637e-05,
+      5.714918876137714e-05,
+      5.6555599525128614e-05,
+      5.59620102888801e-05,
+      5.5368421052631586e-05,
+      5.477483181638306e-05,
+      5.418124258013454e-05,
+      5.3587653343886025e-05,
+      5.299406410763753e-05,
+      5.2400474871389e-05,
+      5.1806885635140485e-05,
+      5.1213296398891974e-05,
+      5.061970716264345e-05,
+      5.002611792639493e-05,
+      4.943252869014643e-05,
+      4.883893945389792e-05,
+      4.8245350217649385e-05,
+      4.765176098140087e-05,
+      4.705817174515236e-05,
+      4.646458250890383e-05,
+      4.587099327265532e-05,
+      4.5277404036406817e-05,
+      4.452948159873368e-05,
+      4.393589236248516e-05,
+      4.3342303126236656e-05,
+      4.274871388998813e-05,
+      4.2155124653739606e-05,
+      4.156153541749111e-05,
+      4.096794618124259e-05,
+      4.037435694499406e-05,
+      3.978076770874554e-05,
+      3.918717847249704e-05,
+      3.859358923624852e-05,
+      3.7999999999999995e-05,
+      3.740641076375149e-05,
+      3.681282152750297e-05,
+      3.621923229125445e-05,
+      3.562564305500593e-05,
+      3.5032053818757427e-05,
+      3.443846458250891e-05,
+      3.3844875346260384e-05,
+      3.325128611001188e-05,
+      3.265769687376336e-05,
+      3.206410763751484e-05,
+      3.147051840126632e-05,
+      3.0876929165017815e-05,
+      3.0283339928769294e-05,
+      2.968975069252077e-05,
+      2.909616145627227e-05,
+      2.850257222002375e-05,
+      2.7908982983775223e-05,
+      2.7315393747526725e-05,
+      2.67218045112782e-05,
+      2.597388207360506e-05,
+      2.5380292837356554e-05,
+      2.4786703601108037e-05,
+      2.4193114364859512e-05,
+      2.359952512861101e-05,
+      2.300593589236249e-05,
+      2.241234665611397e-05,
+      2.1818757419865447e-05,
+      2.1225168183616943e-05,
+      2.0631578947368425e-05,
+      2.0037989711119897e-05,
+      1.94444004748714e-05,
+      1.8850811238622875e-05,
+      1.8257222002374354e-05,
+      1.7663632766125833e-05,
+      1.7070043529877332e-05,
+      1.647645429362881e-05,
+      1.5882865057380286e-05,
+      1.5289275821131785e-05,
+      1.4695686584883264e-05,
+      1.4102097348634743e-05,
+      1.350850811238622e-05,
+      1.291491887613772e-05,
+      1.2321329639889196e-05,
+      1.1727740403640675e-05,
+      1.1134151167392172e-05,
+      1.0540561931143651e-05,
+      9.94697269489513e-06,
+      9.353383458646628e-06,
+      8.759794222398106e-06,
+      8.166204986149583e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      1550,
+      3113,
+      4676,
+      6239,
+      7802,
+      9365,
+      10928,
+      12491,
+      14054,
+      15617
+    ],
+    "eval_accuracy": [
+      0.06,
+      0.81,
+      0.95,
+      0.99,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0
+    ]
+  },
+  "final_accuracy": 0.99875,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9983471074380166,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9923076923076923,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9770114942528736,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9950738916256158,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.55,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7634408602150538,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.9865470852017937,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.48,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.74375,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.8501628664495114,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.43,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.54,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.685,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.77,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.86,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.89,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.98,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9977628635346756,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.994328922495274,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.79,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8911917098445595,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.9626168224299065,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.72,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8984375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.9305555555555556,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.7,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9215686274509803,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.8762886597938144,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.7,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9316939890710383,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9572649572649573,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966722129783694,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9911504424778761,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.36,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9902912621359223,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5637583892617449,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.29,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.7066666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.32,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.5275,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.995,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9977220956719818,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.81,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9035532994923858,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.68,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8785425101214575,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.9281045751633987,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.57,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8590604026845637,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.9257425742574258,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.7304166666666667,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 4,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.97,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.99875,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.99875,
+  "sft_overall_accuracy": 0.7304166666666667
+}

add_sub_sorl_v1_abs5_100K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78256689c3ca0e47e7b7c5bdcfa771d65b002e2b5b45898c75644917cfd92c91
+size 650283250

add_sub_sorl_v1_abs5_100K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 8e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 1562,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs5_K4_100K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 5,
+  "dataset_size": 100000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162494162,
+  "run_name": "add_sub_sorl_v1_abs5_100K",
+  "git_commit": "3ae8ca0d6b88706715f25991b1e1acd0e3a6e0a6",
+  "timestamp": "2026-04-12T20:08:41.300584+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "s8jw6eh2",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/s8jw6eh2",
+  "final_accuracy": 0.99875,
+  "sft_accuracy": 0.7304166666666667,
+  "eval_method": "ArithmeticEvaluator"
+}