Upload add_sub_sorl_v1_abs20_100K

add_sub_sorl_v1_abs20_100K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151664
+}

add_sub_sorl_v1_abs20_100K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs20_100K/metrics.json

@@ -0,0 +1,3537 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1613,
+      1663,
+      1713,
+      1763,
+      1813,
+      1863,
+      1913,
+      1963,
+      2013,
+      2063,
+      2113,
+      2163,
+      2213,
+      2263,
+      2313,
+      2363,
+      2413,
+      2463,
+      2513,
+      2563,
+      2613,
+      2663,
+      2713,
+      2763,
+      2813,
+      2863,
+      2913,
+      2963,
+      3013,
+      3063,
+      3113,
+      3176,
+      3226,
+      3276,
+      3326,
+      3376,
+      3426,
+      3476,
+      3526,
+      3576,
+      3626,
+      3676,
+      3726,
+      3776,
+      3826,
+      3876,
+      3926,
+      3976,
+      4026,
+      4076,
+      4126,
+      4176,
+      4226,
+      4276,
+      4326,
+      4376,
+      4426,
+      4476,
+      4526,
+      4576,
+      4626,
+      4676,
+      4739,
+      4789,
+      4839,
+      4889,
+      4939,
+      4989,
+      5039,
+      5089,
+      5139,
+      5189,
+      5239,
+      5289,
+      5339,
+      5389,
+      5439,
+      5489,
+      5539,
+      5589,
+      5639,
+      5689,
+      5739,
+      5789,
+      5839,
+      5889,
+      5939,
+      5989,
+      6039,
+      6089,
+      6139,
+      6189,
+      6239,
+      6302,
+      6352,
+      6402,
+      6452,
+      6502,
+      6552,
+      6602,
+      6652,
+      6702,
+      6752,
+      6802,
+      6852,
+      6902,
+      6952,
+      7002,
+      7052,
+      7102,
+      7152,
+      7202,
+      7252,
+      7302,
+      7352,
+      7402,
+      7452,
+      7502,
+      7552,
+      7602,
+      7652,
+      7702,
+      7752,
+      7802,
+      7865,
+      7915,
+      7965,
+      8015,
+      8065,
+      8115,
+      8165,
+      8215,
+      8265,
+      8315,
+      8365,
+      8415,
+      8465,
+      8515,
+      8565,
+      8615,
+      8665,
+      8715,
+      8765,
+      8815,
+      8865,
+      8915,
+      8965,
+      9015,
+      9065,
+      9115,
+      9165,
+      9215,
+      9265,
+      9315,
+      9365,
+      9428,
+      9478,
+      9528,
+      9578,
+      9628,
+      9678,
+      9728,
+      9778,
+      9828,
+      9878,
+      9928,
+      9978,
+      10028,
+      10078,
+      10128,
+      10178,
+      10228,
+      10278,
+      10328,
+      10378,
+      10428,
+      10478,
+      10528,
+      10578,
+      10628,
+      10678,
+      10728,
+      10778,
+      10828,
+      10878,
+      10928,
+      10991,
+      11041,
+      11091,
+      11141,
+      11191,
+      11241,
+      11291,
+      11341,
+      11391,
+      11441,
+      11491,
+      11541,
+      11591,
+      11641,
+      11691,
+      11741,
+      11791,
+      11841,
+      11891,
+      11941,
+      11991,
+      12041,
+      12091,
+      12141,
+      12191,
+      12241,
+      12291,
+      12341,
+      12391,
+      12441,
+      12491,
+      12554,
+      12604,
+      12654,
+      12704,
+      12754,
+      12804,
+      12854,
+      12904,
+      12954,
+      13004,
+      13054,
+      13104,
+      13154,
+      13204,
+      13254,
+      13304,
+      13354,
+      13404,
+      13454,
+      13504,
+      13554,
+      13604,
+      13654,
+      13704,
+      13754,
+      13804,
+      13854,
+      13904,
+      13954,
+      14004,
+      14054,
+      14117,
+      14167,
+      14217,
+      14267,
+      14317,
+      14367,
+      14417,
+      14467,
+      14517,
+      14567,
+      14617,
+      14667,
+      14717,
+      14767,
+      14817,
+      14867,
+      14917,
+      14967,
+      15017,
+      15067,
+      15117,
+      15167,
+      15217,
+      15267,
+      15317,
+      15367,
+      15417,
+      15467,
+      15517,
+      15567,
+      15617
+    ],
+    "loss": [
+      16.79209327697754,
+      11.760915756225586,
+      8.9793062210083,
+      7.396918296813965,
+      5.559861183166504,
+      3.38045597076416,
+      3.2124176025390625,
+      2.7785732746124268,
+      2.1971092224121094,
+      2.2191262245178223,
+      2.200740337371826,
+      2.009458541870117,
+      2.3492300510406494,
+      1.3246259689331055,
+      1.1564584970474243,
+      1.4707976579666138,
+      1.1582921743392944,
+      1.1815707683563232,
+      1.0310959815979004,
+      0.8539774417877197,
+      0.6265650987625122,
+      0.34309226274490356,
+      -1.3883473873138428,
+      -4.232861518859863,
+      -7.910555362701416,
+      -9.498554229736328,
+      -10.19212818145752,
+      -9.983817100524902,
+      -10.945733070373535,
+      -11.837502479553223,
+      -12.18378734588623,
+      -14.094343185424805,
+      -13.795543670654297,
+      -13.354608535766602,
+      -14.1011962890625,
+      -13.137774467468262,
+      -13.852112770080566,
+      -14.57231616973877,
+      -14.43851375579834,
+      -13.860637664794922,
+      -13.924348831176758,
+      -14.35501480102539,
+      -14.115457534790039,
+      -13.824909210205078,
+      -13.434500694274902,
+      -14.72733211517334,
+      -14.331260681152344,
+      -14.202347755432129,
+      -13.94506549835205,
+      -14.042664527893066,
+      -14.731111526489258,
+      -14.021833419799805,
+      -14.107925415039062,
+      -14.056807518005371,
+      -14.242992401123047,
+      -13.928154945373535,
+      -13.331207275390625,
+      -13.890643119812012,
+      -14.211116790771484,
+      -14.563620567321777,
+      -14.260629653930664,
+      -14.320273399353027,
+      -13.844810485839844,
+      -14.332328796386719,
+      -14.851007461547852,
+      -14.306427001953125,
+      -14.03696346282959,
+      -14.060243606567383,
+      -14.329286575317383,
+      -13.870020866394043,
+      -13.720074653625488,
+      -13.800219535827637,
+      -14.182469367980957,
+      -13.952445983886719,
+      -14.123272895812988,
+      -14.184364318847656,
+      -14.153197288513184,
+      -13.918373107910156,
+      -14.013357162475586,
+      -14.240181922912598,
+      -14.764949798583984,
+      -13.324590682983398,
+      -13.444450378417969,
+      -13.710700988769531,
+      -13.82218074798584,
+      -13.838254928588867,
+      -13.597502708435059,
+      -13.516195297241211,
+      -12.035502433776855,
+      -10.981066703796387,
+      -11.022160530090332,
+      -8.873615264892578,
+      -8.038102149963379,
+      -6.47631311416626,
+      -5.598836898803711,
+      -6.26401948928833,
+      -5.333451271057129,
+      -4.918493270874023,
+      -4.710961818695068,
+      -4.534127712249756,
+      -4.869874000549316,
+      -4.14800500869751,
+      -4.459377765655518,
+      -3.720736503601074,
+      -4.470472812652588,
+      -3.9317538738250732,
+      -3.9913244247436523,
+      -3.6918787956237793,
+      -4.0778326988220215,
+      -3.3936843872070312,
+      -3.625615119934082,
+      -3.606598138809204,
+      -4.296830177307129,
+      -3.634775161743164,
+      -3.6730377674102783,
+      -3.0592892169952393,
+      -3.2307827472686768,
+      -3.299914598464966,
+      -3.612592935562134,
+      -3.362389087677002,
+      -3.043623208999634,
+      -3.2935516834259033,
+      -2.8091166019439697,
+      -3.3286819458007812,
+      -2.8308053016662598,
+      -3.2010338306427,
+      -3.375739574432373,
+      -3.0267436504364014,
+      -3.4792568683624268,
+      -2.9172184467315674,
+      -2.8592689037323,
+      -2.985954761505127,
+      -2.883500337600708,
+      -2.8080453872680664,
+      -2.6712381839752197,
+      -2.6109540462493896,
+      -3.006927251815796,
+      -3.044006586074829,
+      -2.3167779445648193,
+      -2.9145047664642334,
+      -2.413074254989624,
+      -2.2815890312194824,
+      -2.9332025051116943,
+      -2.440556287765503,
+      -2.3241117000579834,
+      -2.8669636249542236,
+      -2.4647488594055176,
+      -2.8345627784729004,
+      -2.40256404876709,
+      -2.841010570526123,
+      -2.350149154663086,
+      -2.423363208770752,
+      -2.970334529876709,
+      -2.630432605743408,
+      -2.3789308071136475,
+      -2.4599573612213135,
+      -2.0926942825317383,
+      -2.4133481979370117,
+      -2.117722988128662,
+      -2.2958366870880127,
+      -2.039107084274292,
+      -1.819115400314331,
+      -2.396491765975952,
+      -2.4824070930480957,
+      -1.8121613264083862,
+      -1.8639214038848877,
+      -2.159947633743286,
+      -1.6071596145629883,
+      -1.9000400304794312,
+      -1.7521079778671265,
+      -1.766842007637024,
+      -2.0209553241729736,
+      -1.7798255681991577,
+      -1.9496805667877197,
+      -1.8481358289718628,
+      -1.724064588546753,
+      -1.9545000791549683,
+      -2.355841875076294,
+      -1.8106406927108765,
+      -1.8370072841644287,
+      -2.237955331802368,
+      -1.6500630378723145,
+      -1.7646278142929077,
+      -1.7661163806915283,
+      -1.6755574941635132,
+      -1.4729588031768799,
+      -1.6085923910140991,
+      -1.3132957220077515,
+      -1.8922404050827026,
+      -1.5854400396347046,
+      -1.712070107460022,
+      -1.273451566696167,
+      -1.352250576019287,
+      -1.3046519756317139,
+      -1.3954737186431885,
+      -1.7703322172164917,
+      -1.4283695220947266,
+      -1.0123875141143799,
+      -1.6394275426864624,
+      -1.4428489208221436,
+      -1.4007066488265991,
+      -1.282835841178894,
+      -1.3286669254302979,
+      -1.479001522064209,
+      -1.1670639514923096,
+      -1.3329111337661743,
+      -0.9665555357933044,
+      -1.2764699459075928,
+      -0.9801934957504272,
+      -0.9535375237464905,
+      -1.1150256395339966,
+      -1.0306384563446045,
+      -1.2519457340240479,
+      -1.1996408700942993,
+      -1.1164659261703491,
+      -1.1038587093353271,
+      -1.3577220439910889,
+      -1.2127302885055542,
+      -0.8137933015823364,
+      -0.8732357025146484,
+      -0.8328045606613159,
+      -0.9246251583099365,
+      -0.6932074427604675,
+      -1.1156266927719116,
+      -0.9569106101989746,
+      -0.8504764437675476,
+      -1.128698468208313,
+      -0.9255280494689941,
+      -0.8243647813796997,
+      -0.9268491864204407,
+      -0.7702681422233582,
+      -0.7200023531913757,
+      -0.9779446125030518,
+      -0.7897813320159912,
+      -0.800619900226593,
+      -0.7645387053489685,
+      -1.1333955526351929,
+      -0.8054041266441345,
+      -0.8093994855880737,
+      -0.8673604726791382,
+      -0.9816060066223145,
+      -0.5911740064620972,
+      -0.9720580577850342,
+      -0.5948343873023987,
+      -0.8260560631752014,
+      -0.591710090637207,
+      -1.003006935119629,
+      -0.7442401647567749,
+      -0.6731335520744324,
+      -0.9030881524085999,
+      -0.9331020712852478,
+      -0.5333653688430786,
+      -0.7962002754211426,
+      -0.4459095001220703,
+      -0.5970262289047241,
+      -0.6425231695175171,
+      -0.9427562355995178,
+      -0.5024524927139282,
+      -0.4325365424156189,
+      -0.6038902997970581,
+      -0.6784700155258179,
+      -0.44050002098083496,
+      -0.5332865715026855,
+      -0.6820433139801025,
+      -0.6055371761322021,
+      -0.9522186517715454,
+      -0.6130183935165405,
+      -0.46341240406036377,
+      -0.3951411843299866,
+      -0.5626518726348877,
+      -0.5533226132392883,
+      -0.5690665245056152,
+      -0.4580327272415161,
+      -0.41125890612602234,
+      -0.6303701996803284,
+      -0.5366156697273254,
+      -0.5488389134407043,
+      -0.5357339382171631,
+      -0.4944663643836975,
+      -0.35123932361602783,
+      -0.652963399887085,
+      -0.5450815558433533,
+      -0.45467543601989746,
+      -0.7127432823181152,
+      -0.2613491415977478,
+      -0.5923411846160889,
+      -0.5586081147193909,
+      -0.4944308400154114,
+      -0.4661834239959717,
+      -0.6500449180603027,
+      -0.4541547894477844,
+      -0.35607436299324036,
+      -0.33617547154426575,
+      -0.4879729151725769,
+      -0.5594838857650757,
+      -0.5726749897003174,
+      -0.40385618805885315,
+      -0.475053995847702,
+      -0.2922070026397705,
+      -0.38074222207069397,
+      -0.37540552020072937,
+      -0.3134785592556,
+      -0.36059361696243286,
+      -0.2543281316757202,
+      -0.4130215346813202,
+      -0.40772899985313416,
+      -0.3520417809486389,
+      -0.3371984660625458,
+      -0.39990484714508057,
+      -0.309539794921875
+    ],
+    "base_loss": [
+      10.586007118225098,
+      8.107672691345215,
+      7.379012107849121,
+      6.3848748207092285,
+      5.396872043609619,
+      3.847005844116211,
+      2.3134782314300537,
+      2.067183017730713,
+      1.9377295970916748,
+      1.8367758989334106,
+      1.861541509628296,
+      1.8085911273956299,
+      1.8419133424758911,
+      1.8773196935653687,
+      1.8867391347885132,
+      1.859654426574707,
+      1.8019094467163086,
+      1.8483047485351562,
+      1.8342684507369995,
+      1.8257976770401,
+      1.8153953552246094,
+      1.8469390869140625,
+      1.9652775526046753,
+      1.8965301513671875,
+      1.9381773471832275,
+      1.8259631395339966,
+      1.8266514539718628,
+      1.7139222621917725,
+      1.7167195081710815,
+      1.7782014608383179,
+      1.6776527166366577,
+      1.8053874969482422,
+      1.7560675144195557,
+      1.6971855163574219,
+      1.7088196277618408,
+      1.6051843166351318,
+      1.6757994890213013,
+      1.7263540029525757,
+      1.7008579969406128,
+      1.6428000926971436,
+      1.6273185014724731,
+      1.6862186193466187,
+      1.6371279954910278,
+      1.5970579385757446,
+      1.57394540309906,
+      1.6870296001434326,
+      1.632698655128479,
+      1.6635228395462036,
+      1.6213722229003906,
+      1.629797101020813,
+      1.6824833154678345,
+      1.6239286661148071,
+      1.6119505167007446,
+      1.5938571691513062,
+      1.6186141967773438,
+      1.5952460765838623,
+      1.5063362121582031,
+      1.586992859840393,
+      1.615220308303833,
+      1.657053828239441,
+      1.6130435466766357,
+      1.6204346418380737,
+      1.5717500448226929,
+      1.642485499382019,
+      1.6788948774337769,
+      1.6228059530258179,
+      1.5818979740142822,
+      1.5864309072494507,
+      1.622089147567749,
+      1.5611313581466675,
+      1.5610262155532837,
+      1.5713090896606445,
+      1.6062123775482178,
+      1.573341727256775,
+      1.5900599956512451,
+      1.5899255275726318,
+      1.5931357145309448,
+      1.5644103288650513,
+      1.5822985172271729,
+      1.596530556678772,
+      1.6685844659805298,
+      1.5111242532730103,
+      1.5387582778930664,
+      1.5349165201187134,
+      1.551059365272522,
+      1.5500211715698242,
+      1.543041467666626,
+      1.5183570384979248,
+      1.3714021444320679,
+      1.2309716939926147,
+      1.2376619577407837,
+      1.0026013851165771,
+      0.9183998107910156,
+      0.7370001673698425,
+      0.6448150277137756,
+      0.7149214148521423,
+      0.6043591499328613,
+      0.5601844191551208,
+      0.5388034582138062,
+      0.5181083083152771,
+      0.5605003833770752,
+      0.49011102318763733,
+      0.5158273577690125,
+      0.4581403136253357,
+      0.5093480348587036,
+      0.45746392011642456,
+      0.45303425192832947,
+      0.425349622964859,
+      0.4693649113178253,
+      0.3835274875164032,
+      0.4172529876232147,
+      0.41681861877441406,
+      0.4847046732902527,
+      0.4129360616207123,
+      0.42312076687812805,
+      0.35800203680992126,
+      0.3648161292076111,
+      0.3770079016685486,
+      0.40686485171318054,
+      0.38254112005233765,
+      0.37254366278648376,
+      0.37317603826522827,
+      0.3200540542602539,
+      0.37506431341171265,
+      0.3308878540992737,
+      0.3618965744972229,
+      0.3816489279270172,
+      0.3562702536582947,
+      0.3953951895236969,
+      0.32903555035591125,
+      0.32740649580955505,
+      0.33960676193237305,
+      0.32806164026260376,
+      0.32601025700569153,
+      0.3013496398925781,
+      0.30211371183395386,
+      0.3403961956501007,
+      0.3543865382671356,
+      0.28074967861175537,
+      0.3316064178943634,
+      0.3044281005859375,
+      0.27025267481803894,
+      0.3354354500770569,
+      0.27548614144325256,
+      0.2650213837623596,
+      0.3234979212284088,
+      0.27871283888816833,
+      0.31963029503822327,
+      0.2700897455215454,
+      0.31936904788017273,
+      0.2772665321826935,
+      0.27345913648605347,
+      0.3624987006187439,
+      0.29634755849838257,
+      0.2699447274208069,
+      0.28019237518310547,
+      0.237746924161911,
+      0.27120015025138855,
+      0.23963557183742523,
+      0.2580537796020508,
+      0.23053193092346191,
+      0.21056635677814484,
+      0.2714405357837677,
+      0.2802487909793854,
+      0.20902177691459656,
+      0.21195857226848602,
+      0.2431097775697708,
+      0.20361550152301788,
+      0.21543630957603455,
+      0.2017604559659958,
+      0.20477354526519775,
+      0.23138846457004547,
+      0.2045169174671173,
+      0.22018848359584808,
+      0.2100868970155716,
+      0.19771385192871094,
+      0.22226355969905853,
+      0.264514297246933,
+      0.2051856517791748,
+      0.2082277536392212,
+      0.2532007396221161,
+      0.18809524178504944,
+      0.19889818131923676,
+      0.1996307224035263,
+      0.18925443291664124,
+      0.16877637803554535,
+      0.18221449851989746,
+      0.15362408757209778,
+      0.2141164392232895,
+      0.19857670366764069,
+      0.1922067403793335,
+      0.14474739134311676,
+      0.15556621551513672,
+      0.15097951889038086,
+      0.15775156021118164,
+      0.20066706836223602,
+      0.16310788691043854,
+      0.11550626903772354,
+      0.18654581904411316,
+      0.16781851649284363,
+      0.16049368679523468,
+      0.1473221480846405,
+      0.1495732069015503,
+      0.17065215110778809,
+      0.13307374715805054,
+      0.152320995926857,
+      0.12603043019771576,
+      0.14552898705005646,
+      0.11096616834402084,
+      0.12645430862903595,
+      0.12620307505130768,
+      0.11763323843479156,
+      0.1407073736190796,
+      0.13632047176361084,
+      0.12582434713840485,
+      0.1250101625919342,
+      0.1541365534067154,
+      0.1393197476863861,
+      0.09302148967981339,
+      0.10151750594377518,
+      0.0948488712310791,
+      0.10472721606492996,
+      0.07927661389112473,
+      0.12834584712982178,
+      0.10900652408599854,
+      0.09752453863620758,
+      0.12911449372768402,
+      0.10582436621189117,
+      0.09543760865926743,
+      0.10628031194210052,
+      0.08750653266906738,
+      0.08285124599933624,
+      0.11356587707996368,
+      0.09127108007669449,
+      0.09202581644058228,
+      0.08853127062320709,
+      0.1286425143480301,
+      0.09253621101379395,
+      0.09237678349018097,
+      0.09881012886762619,
+      0.11309915781021118,
+      0.06982738524675369,
+      0.11200392246246338,
+      0.07380207628011703,
+      0.09455603361129761,
+      0.08327490836381912,
+      0.11884456127882004,
+      0.08556420356035233,
+      0.07721924036741257,
+      0.10405800491571426,
+      0.10802986472845078,
+      0.0619015209376812,
+      0.09264067560434341,
+      0.052628617733716965,
+      0.06948359310626984,
+      0.07375140488147736,
+      0.10751468688249588,
+      0.059202563017606735,
+      0.05111440271139145,
+      0.07033004611730576,
+      0.07828278094530106,
+      0.05393128842115402,
+      0.06128722056746483,
+      0.07815670967102051,
+      0.06989593803882599,
+      0.11021001636981964,
+      0.07014123350381851,
+      0.05431556701660156,
+      0.04903032258152962,
+      0.06756965816020966,
+      0.06432704627513885,
+      0.065272256731987,
+      0.055248379707336426,
+      0.04900113865733147,
+      0.07290402799844742,
+      0.06393275409936905,
+      0.06356173008680344,
+      0.061554115265607834,
+      0.05799867957830429,
+      0.04246802255511284,
+      0.07636731117963791,
+      0.06435016542673111,
+      0.05432090908288956,
+      0.0806860476732254,
+      0.032122932374477386,
+      0.06914357095956802,
+      0.06491504609584808,
+      0.05827094987034798,
+      0.054308123886585236,
+      0.07501593977212906,
+      0.0526263602077961,
+      0.04181347414851189,
+      0.039931390434503555,
+      0.05918297544121742,
+      0.06463120877742767,
+      0.06513258814811707,
+      0.04770168289542198,
+      0.05509531870484352,
+      0.03441275283694267,
+      0.04434604197740555,
+      0.04428449645638466,
+      0.03709632158279419,
+      0.04441501945257187,
+      0.031395334750413895,
+      0.04886626824736595,
+      0.048341091722249985,
+      0.04275854304432869,
+      0.03998516872525215,
+      0.04799428954720497,
+      0.036228783428668976
+    ],
+    "info_loss": [
+      -0.15445613861083984,
+      -0.30521535873413086,
+      -0.33514976501464844,
+      -0.2423076629638672,
+      -0.22414731979370117,
+      -0.21929526329040527,
+      -0.05562019348144531,
+      -0.06447577476501465,
+      -0.10581016540527344,
+      -0.09356296062469482,
+      -0.09634482860565186,
+      -0.11017370223999023,
+      -0.0791999101638794,
+      -0.1850351095199585,
+      -0.20321202278137207,
+      -0.16833889484405518,
+      -0.193231463432312,
+      -0.19600355625152588,
+      -0.20949387550354004,
+      -0.22645235061645508,
+      -0.24515342712402344,
+      -0.2723342180252075,
+      -0.4342762231826782,
+      -0.6718428134918213,
+      -1.0231105089187622,
+      -1.161816120147705,
+      -1.2269487380981445,
+      -1.1914074420928955,
+      -1.2881202697753906,
+      -1.3813605308532715,
+      -1.4050863981246948,
+      -1.6068695783615112,
+      -1.5710461139678955,
+      -1.5195889472961426,
+      -1.5950535535812378,
+      -1.4887698888778687,
+      -1.5665760040283203,
+      -1.6432299613952637,
+      -1.6273547410964966,
+      -1.5630104541778564,
+      -1.5676462650299072,
+      -1.6183618307113647,
+      -1.5868898630142212,
+      -1.5539655685424805,
+      -1.5116292238235474,
+      -1.651062250137329,
+      -1.6061369180679321,
+      -1.596465826034546,
+      -1.5648189783096313,
+      -1.575344443321228,
+      -1.6500952243804932,
+      -1.5725255012512207,
+      -1.5798263549804688,
+      -1.5728739500045776,
+      -1.5940080881118774,
+      -1.56052565574646,
+      -1.492093801498413,
+      -1.5564193725585938,
+      -1.5899673700332642,
+      -1.62934410572052,
+      -1.5959336757659912,
+      -1.6007269620895386,
+      -1.548478364944458,
+      -1.6044995784759521,
+      -1.6587224006652832,
+      -1.5995573997497559,
+      -1.5684024095535278,
+      -1.5705070495605469,
+      -1.6026567220687866,
+      -1.5486727952957153,
+      -1.535772681236267,
+      -1.5439412593841553,
+      -1.584057331085205,
+      -1.558900237083435,
+      -1.576781153678894,
+      -1.5832421779632568,
+      -1.5791561603546143,
+      -1.552700161933899,
+      -1.564715027809143,
+      -1.5883455276489258,
+      -1.649574637413025,
+      -1.4878178834915161,
+      -1.5020684003829956,
+      -1.5291961431503296,
+      -1.5412322282791138,
+      -1.5425528287887573,
+      -1.5176763534545898,
+      -1.5075466632843018,
+      -1.3453041315078735,
+      -1.225195050239563,
+      -1.230076551437378,
+      -0.9917240142822266,
+      -0.9002496004104614,
+      -0.7257813215255737,
+      -0.627955973148346,
+      -0.702584445476532,
+      -0.5982261896133423,
+      -0.5509673357009888,
+      -0.52879798412323,
+      -0.5094592571258545,
+      -0.5463420152664185,
+      -0.4676706790924072,
+      -0.5010921359062195,
+      -0.42104434967041016,
+      -0.5016980767250061,
+      -0.44333311915397644,
+      -0.4491637349128723,
+      -0.41619834303855896,
+      -0.45768746733665466,
+      -0.38130971789360046,
+      -0.4073494076728821,
+      -0.4051756262779236,
+      -0.48252367973327637,
+      -0.4091421365737915,
+      -0.41321682929992676,
+      -0.3475387990474701,
+      -0.36227160692214966,
+      -0.37115997076034546,
+      -0.40533366799354553,
+      -0.3791196346282959,
+      -0.3453383147716522,
+      -0.3705844283103943,
+      -0.3160562217235565,
+      -0.37396329641342163,
+      -0.3193526864051819,
+      -0.3590923249721527,
+      -0.3784433603286743,
+      -0.34086278080940247,
+      -0.39094415307044983,
+      -0.32851776480674744,
+      -0.3225290775299072,
+      -0.3373115360736847,
+      -0.32603776454925537,
+      -0.31833845376968384,
+      -0.30103644728660583,
+      -0.29578495025634766,
+      -0.3381631672382355,
+      -0.34345051646232605,
+      -0.26141223311424255,
+      -0.3276233971118927,
+      -0.27610301971435547,
+      -0.25706273317337036,
+      -0.33176058530807495,
+      -0.2745024263858795,
+      -0.26327911019325256,
+      -0.32142817974090576,
+      -0.27793800830841064,
+      -0.31895729899406433,
+      -0.269852876663208,
+      -0.3181893527507782,
+      -0.26604339480400085,
+      -0.27203503251075745,
+      -0.33593982458114624,
+      -0.29593273997306824,
+      -0.2677503228187561,
+      -0.27708810567855835,
+      -0.2359430193901062,
+      -0.27079394459724426,
+      -0.23935742676258087,
+      -0.2577801048755646,
+      -0.23017160594463348,
+      -0.20575203001499176,
+      -0.27097365260124207,
+      -0.27916964888572693,
+      -0.20397672057151794,
+      -0.2101304978132248,
+      -0.24271747469902039,
+      -0.1844528317451477,
+      -0.2136078029870987,
+      -0.19852235913276672,
+      -0.19974885880947113,
+      -0.22834879159927368,
+      -0.20318017899990082,
+      -0.21981891989707947,
+      -0.2098880559206009,
+      -0.19651731848716736,
+      -0.22166648507118225,
+      -0.26429232954978943,
+      -0.20456388592720032,
+      -0.20760402083396912,
+      -0.2527919411659241,
+      -0.18789386749267578,
+      -0.19866003096103668,
+      -0.19909249246120453,
+      -0.18893584609031677,
+      -0.1676982194185257,
+      -0.1821356564760208,
+      -0.1496611088514328,
+      -0.21364998817443848,
+      -0.1831970065832138,
+      -0.1920301467180252,
+      -0.1444585919380188,
+      -0.15324868261814117,
+      -0.14899079501628876,
+      -0.15737591683864594,
+      -0.20000284910202026,
+      -0.16270118951797485,
+      -0.11530550569295883,
+      -0.18561191856861115,
+      -0.16256596148014069,
+      -0.1590348333120346,
+      -0.1453976184129715,
+      -0.14942869544029236,
+      -0.16788814961910248,
+      -0.1324063390493393,
+      -0.15227428078651428,
+      -0.11355708539485931,
+      -0.14508508145809174,
+      -0.11089273542165756,
+      -0.11030630022287369,
+      -0.1258462518453598,
+      -0.11682376265525818,
+      -0.1406470388174057,
+      -0.13544175028800964,
+      -0.1257992535829544,
+      -0.1247929185628891,
+      -0.1540917307138443,
+      -0.13887283205986023,
+      -0.09298080950975418,
+      -0.10008486360311508,
+      -0.09452833235263824,
+      -0.1043253168463707,
+      -0.07922310382127762,
+      -0.12828916311264038,
+      -0.10898083448410034,
+      -0.09729553759098053,
+      -0.12906037271022797,
+      -0.10519947111606598,
+      -0.09539952129125595,
+      -0.10615312308073044,
+      -0.08748174458742142,
+      -0.08279362320899963,
+      -0.11350344866514206,
+      -0.09122669696807861,
+      -0.09114734828472137,
+      -0.08846383541822433,
+      -0.12862056493759155,
+      -0.09247587621212006,
+      -0.09224685281515121,
+      -0.09877295047044754,
+      -0.11255116760730743,
+      -0.06978656351566315,
+      -0.11130283772945404,
+      -0.07027405500411987,
+      -0.09452458471059799,
+      -0.06903683394193649,
+      -0.11835796386003494,
+      -0.08533279597759247,
+      -0.0771995559334755,
+      -0.1040348932147026,
+      -0.10743240267038345,
+      -0.06186848506331444,
+      -0.09235931187868118,
+      -0.052592772990465164,
+      -0.06904541701078415,
+      -0.07371632754802704,
+      -0.10741108655929565,
+      -0.05915733799338341,
+      -0.051066379994153976,
+      -0.06972707062959671,
+      -0.07821008563041687,
+      -0.05367730185389519,
+      -0.06125984713435173,
+      -0.07813957333564758,
+      -0.06986301392316818,
+      -0.11017470806837082,
+      -0.06996914744377136,
+      -0.05427330732345581,
+      -0.049014732241630554,
+      -0.06484035402536392,
+      -0.06430783122777939,
+      -0.0652451142668724,
+      -0.05466066673398018,
+      -0.048970337957143784,
+      -0.07287704199552536,
+      -0.06392456591129303,
+      -0.0635431781411171,
+      -0.06152886897325516,
+      -0.057876963168382645,
+      -0.042413853108882904,
+      -0.07633647322654724,
+      -0.0643203929066658,
+      -0.05429358035326004,
+      -0.08066803961992264,
+      -0.03210672363638878,
+      -0.06904635578393936,
+      -0.06490110605955124,
+      -0.05825822427868843,
+      -0.05428257957100868,
+      -0.07499688118696213,
+      -0.05260239541530609,
+      -0.04178810492157936,
+      -0.0398712232708931,
+      -0.05697103217244148,
+      -0.06460782140493393,
+      -0.06512656062841415,
+      -0.0476703867316246,
+      -0.05507079139351845,
+      -0.034406691789627075,
+      -0.044332895427942276,
+      -0.0440828800201416,
+      -0.03708498552441597,
+      -0.0444052554666996,
+      -0.0313878059387207,
+      -0.04885256662964821,
+      -0.04832703247666359,
+      -0.04275417700409889,
+      -0.039977606385946274,
+      -0.04798528552055359,
+      -0.036218736320734024
+    ],
+    "abs_loss": [
+      2.9939699172973633,
+      2.9671630859375,
+      2.884432315826416,
+      2.8786685466766357,
+      2.776923179626465,
+      2.5604119300842285,
+      2.4716804027557373,
+      2.3948922157287598,
+      2.30157470703125,
+      2.370891571044922,
+      2.3342669010162354,
+      2.3448739051818848,
+      2.310504198074341,
+      2.3500261306762695,
+      2.317657232284546,
+      2.2523250579833984,
+      2.266146183013916,
+      2.295795202255249,
+      2.3130407333374023,
+      2.391406536102295,
+      2.162468433380127,
+      2.2781295776367188,
+      1.9174010753631592,
+      1.5865317583084106,
+      1.27726149559021,
+      1.1252155303955078,
+      1.095202922821045,
+      1.0401458740234375,
+      0.9953299760818481,
+      0.9434686899185181,
+      0.9169909954071045,
+      0.926274299621582,
+      0.8359701037406921,
+      0.7653440833091736,
+      0.8254138231277466,
+      0.8021069765090942,
+      0.7468331456184387,
+      0.7645702362060547,
+      0.8108381628990173,
+      0.6716663241386414,
+      0.746320366859436,
+      0.7823787927627563,
+      0.6703470945358276,
+      0.6590404510498047,
+      0.5716877579689026,
+      0.5281146764755249,
+      0.5692504644393921,
+      0.5552536249160767,
+      0.49144041538238525,
+      0.49382561445236206,
+      0.5187468528747559,
+      0.4593263268470764,
+      0.43579062819480896,
+      0.3469533622264862,
+      0.3825676143169403,
+      0.42909789085388184,
+      0.453450471162796,
+      0.42889830470085144,
+      0.4658021926879883,
+      0.3615960478782654,
+      0.3724570870399475,
+      0.3933645486831665,
+      0.3598719835281372,
+      0.3870697319507599,
+      0.32773709297180176,
+      0.3438999652862549,
+      0.3137407600879669,
+      0.34169697761535645,
+      0.3478160500526428,
+      0.3136068284511566,
+      0.2495996057987213,
+      0.3967747986316681,
+      0.3151877522468567,
+      0.2217535525560379,
+      0.2176458090543747,
+      0.19214972853660583,
+      0.27432841062545776,
+      0.1991918683052063,
+      0.27142807841300964,
+      0.15668797492980957,
+      0.18449118733406067,
+      0.19276940822601318,
+      0.245790496468544,
+      0.25198858976364136,
+      0.18245406448841095,
+      0.1920507699251175,
+      0.23409833014011383,
+      0.15946322679519653,
+      0.28308606147766113,
+      0.229856938123703,
+      0.14451345801353455,
+      0.22815349698066711,
+      0.28472089767456055,
+      0.13943305611610413,
+      0.1525324136018753,
+      0.1800794005393982,
+      0.1788015067577362,
+      0.14676126837730408,
+      0.1801275610923767,
+      0.1274595558643341,
+      0.20488670468330383,
+      0.18262898921966553,
+      0.13823282718658447,
+      0.14368613064289093,
+      0.16121840476989746,
+      0.14759963750839233,
+      0.16873055696487427,
+      0.12227734923362732,
+      0.20998935401439667,
+      0.1513805091381073,
+      0.14927320182323456,
+      0.16616597771644592,
+      0.11357106268405914,
+      0.1548023819923401,
+      0.09905439615249634,
+      0.12770454585552216,
+      0.1254686415195465,
+      0.16130158305168152,
+      0.14453987777233124,
+      0.09974582493305206,
+      0.1397075355052948,
+      0.0725528672337532,
+      0.1555742621421814,
+      0.15362101793289185,
+      0.13945335149765015,
+      0.0919971913099289,
+      0.0827254205942154,
+      0.0957292690873146,
+      0.10747053474187851,
+      0.06503817439079285,
+      0.15155860781669617,
+      0.04873042181134224,
+      0.08255958557128906,
+      0.08030789345502853,
+      0.1114988625049591,
+      0.06541724503040314,
+      0.08628928661346436,
+      0.10699835419654846,
+      0.08059462904930115,
+      0.061577748507261276,
+      0.11694025248289108,
+      0.06307436525821686,
+      0.09791208803653717,
+      0.08975191414356232,
+      0.06224038451910019,
+      0.09754665940999985,
+      0.08535797894001007,
+      0.04807925969362259,
+      0.09415542334318161,
+      0.07146130502223969,
+      0.059969738125801086,
+      0.04297126829624176,
+      0.0620364174246788,
+      0.1250634789466858,
+      0.0725262314081192,
+      0.03369183838367462,
+      0.06068193167448044,
+      0.08214086294174194,
+      0.03255259618163109,
+      0.058612044900655746,
+      0.0757773369550705,
+      0.08017509430646896,
+      0.06201693415641785,
+      0.10843746364116669,
+      0.07148608565330505,
+      0.06171827018260956,
+      0.06407183408737183,
+      0.06668765842914581,
+      0.0908898115158081,
+      0.05957251414656639,
+      0.05260217934846878,
+      0.051984988152980804,
+      0.0426282063126564,
+      0.06798090785741806,
+      0.05817725509405136,
+      0.04371039941906929,
+      0.04418366029858589,
+      0.0658913254737854,
+      0.07025420665740967,
+      0.06737496703863144,
+      0.0334327295422554,
+      0.05061866343021393,
+      0.03713146224617958,
+      0.06372493505477905,
+      0.05535414442420006,
+      0.06243645027279854,
+      0.05203639715909958,
+      0.034955430775880814,
+      0.03983978554606438,
+      0.04079217463731766,
+      0.07475005090236664,
+      0.04801257699728012,
+      0.04809274151921272,
+      0.05396736413240433,
+      0.0625157505273819,
+      0.04173329472541809,
+      0.04280892014503479,
+      0.052977219223976135,
+      0.05912243202328682,
+      0.03089144080877304,
+      0.04570259898900986,
+      0.07775736600160599,
+      0.040590133517980576,
+      0.034856513142585754,
+      0.029137877747416496,
+      0.049085311591625214,
+      0.03855056315660477,
+      0.038246601819992065,
+      0.04657942056655884,
+      0.05626664310693741,
+      0.04340174049139023,
+      0.044082626700401306,
+      0.045361317694187164,
+      0.04138324409723282,
+      0.04377198964357376,
+      0.035163700580596924,
+      0.05173082649707794,
+      0.024585846811532974,
+      0.06613926589488983,
+      0.05189907178282738,
+      0.05301932990550995,
+      0.06153346598148346,
+      0.05094657838344574,
+      0.03458717092871666,
+      0.07255750894546509,
+      0.05777464807033539,
+      0.060308024287223816,
+      0.05367795377969742,
+      0.040390580892562866,
+      0.050002336502075195,
+      0.045242227613925934,
+      0.067792147397995,
+      0.03644957393407822,
+      0.036423034965991974,
+      0.039786197245121,
+      0.05897361785173416,
+      0.043324150145053864,
+      0.04928205907344818,
+      0.07481065392494202,
+      0.0536021813750267,
+      0.031223205849528313,
+      0.04402320832014084,
+      0.03952324390411377,
+      0.028211724013090134,
+      0.052927181124687195,
+      0.04173257201910019,
+      0.045009590685367584,
+      0.04482002928853035,
+      0.03669920563697815,
+      0.03860503435134888,
+      0.03810741752386093,
+      0.039218682795763016,
+      0.039497148245573044,
+      0.041818778961896896,
+      0.11109116673469543,
+      0.061651840806007385,
+      0.03293173015117645,
+      0.07021323591470718,
+      0.06279326975345612,
+      0.06056281179189682,
+      0.045141540467739105,
+      0.0443272665143013,
+      0.035209350287914276,
+      0.06140761077404022,
+      0.0379883348941803,
+      0.03811391070485115,
+      0.028341155499219894,
+      0.071586973965168,
+      0.037841156125068665,
+      0.032516077160835266,
+      0.06419142335653305,
+      0.04541822522878647,
+      0.04437222331762314,
+      0.04506922513246536,
+      0.06423269212245941,
+      0.06794172525405884,
+      0.04708072543144226,
+      0.0491417795419693,
+      0.04793268069624901,
+      0.056407395750284195,
+      0.04668997973203659,
+      0.03361371532082558,
+      0.04420556128025055,
+      0.04584401845932007,
+      0.058124154806137085,
+      0.05075351148843765,
+      0.04762536659836769,
+      0.028166018426418304,
+      0.039246730506420135,
+      0.024372506886720657,
+      0.02869992144405842,
+      0.04107384383678436,
+      0.037586912512779236,
+      0.02720024436712265,
+      0.05294787138700485,
+      0.04108988493680954,
+      0.06844684481620789,
+      0.04962025582790375,
+      0.04507195204496384,
+      0.0377793088555336,
+      0.03338238596916199,
+      0.029339665547013283,
+      0.041972436010837555,
+      0.0436406284570694,
+      0.04341753572225571,
+      0.038430675864219666,
+      0.0455116331577301,
+      0.03734186291694641,
+      0.039723582565784454,
+      0.04171336814761162
+    ],
+    "zipf_loss": [
+      7.451250076293945,
+      6.408680438995361,
+      4.66334867477417,
+      3.1472527980804443,
+      2.12677001953125,
+      1.4703617095947266,
+      1.2079733610153198,
+      1.1166588068008423,
+      1.0873239040374756,
+      1.0808907747268677,
+      1.0692203044891357,
+      1.0681169033050537,
+      1.068265438079834,
+      1.062654733657837,
+      1.0700738430023193,
+      1.0692996978759766,
+      1.0620827674865723,
+      1.063722014427185,
+      1.060462236404419,
+      1.0535626411437988,
+      1.0464571714401245,
+      0.9916824102401733,
+      0.7973968982696533,
+      0.4303833246231079,
+      0.25464576482772827,
+      0.18112242221832275,
+      0.14118769764900208,
+      0.11231985688209534,
+      0.11921684443950653,
+      0.10355470329523087,
+      0.09772571921348572,
+      0.07633695006370544,
+      0.07525299489498138,
+      0.06756086647510529,
+      0.05797893926501274,
+      0.0645289495587349,
+      0.06316487491130829,
+      0.05717369168996811,
+      0.05309152975678444,
+      0.059500209987163544,
+      0.05016370862722397,
+      0.06414783000946045,
+      0.049277856945991516,
+      0.05178428068757057,
+      0.050677619874477386,
+      0.04344857484102249,
+      0.04048430174589157,
+      0.043262459337711334,
+      0.03260765224695206,
+      0.0316011905670166,
+      0.035481661558151245,
+      0.03355970233678818,
+      0.03480807691812515,
+      0.04337979480624199,
+      0.04021717607975006,
+      0.03894573450088501,
+      0.03804963082075119,
+      0.0436682403087616,
+      0.026756424456834793,
+      0.03660731017589569,
+      0.0484175942838192,
+      0.027224961668252945,
+      0.03223637491464615,
+      0.031474702060222626,
+      0.0245464276522398,
+      0.03195098042488098,
+      0.0337882936000824,
+      0.024226512759923935,
+      0.04041041061282158,
+      0.0242144837975502,
+      0.05166645348072052,
+      0.028206858783960342,
+      0.02037244290113449,
+      0.04103923961520195,
+      0.03271396830677986,
+      0.0389174148440361,
+      0.01779547706246376,
+      0.024298420175909996,
+      0.02435200661420822,
+      0.0310731902718544,
+      0.043762996792793274,
+      0.023186862468719482,
+      0.012896795757114887,
+      0.021145079284906387,
+      0.020835932344198227,
+      0.018047045916318893,
+      0.012810098007321358,
+      0.024966878816485405,
+      0.017827903851866722,
+      0.016926929354667664,
+      0.026491889730095863,
+      0.018207557499408722,
+      0.017522133886814117,
+      0.030557341873645782,
+      0.020654890686273575,
+      0.028896065428853035,
+      0.026571379974484444,
+      0.016319354996085167,
+      0.020202092826366425,
+      0.029610808938741684,
+      0.012557116337120533,
+      0.02032800018787384,
+      0.021892979741096497,
+      0.01719818264245987,
+      0.02103794738650322,
+      0.02935318648815155,
+      0.030405551195144653,
+      0.03252723067998886,
+      0.00867812056094408,
+      0.02074713632464409,
+      0.015698475763201714,
+      0.011722991243004799,
+      0.03234492614865303,
+      0.028229666873812675,
+      0.02610444463789463,
+      0.04532650113105774,
+      0.014570198953151703,
+      0.01854710280895233,
+      0.019424980506300926,
+      0.03629154711961746,
+      0.023245664313435555,
+      0.03186134248971939,
+      0.01583409681916237,
+      0.020524701103568077,
+      0.01788824424147606,
+      0.018793335184454918,
+      0.018772277981042862,
+      0.016040731221437454,
+      0.024042334407567978,
+      0.032419975847005844,
+      0.023459697142243385,
+      0.04268075153231621,
+      0.040559835731983185,
+      0.041297949850559235,
+      0.026626912876963615,
+      0.038240011781454086,
+      0.025679288432002068,
+      0.025412321090698242,
+      0.008535143919289112,
+      0.023964766412973404,
+      0.03183385357260704,
+      0.012478039599955082,
+      0.039176903665065765,
+      0.02000630460679531,
+      0.037433817982673645,
+      0.014065584167838097,
+      0.02738261967897415,
+      0.03057205118238926,
+      0.016459379345178604,
+      0.014367847703397274,
+      0.0270213820040226,
+      0.019230935722589493,
+      0.02036147192120552,
+      0.020041000097990036,
+      0.021374955773353577,
+      0.02736235409975052,
+      0.0229208841919899,
+      0.015176897868514061,
+      0.03296041488647461,
+      0.018049122765660286,
+      0.024499420076608658,
+      0.0198210421949625,
+      0.03560253605246544,
+      0.018196703866124153,
+      0.011435543186962605,
+      0.01925329864025116,
+      0.01771029829978943,
+      0.027084290981292725,
+      0.011512783356010914,
+      0.025397945195436478,
+      0.020612847059965134,
+      0.02594551257789135,
+      0.04319647699594498,
+      0.021521996706724167,
+      0.03484001383185387,
+      0.03902372717857361,
+      0.0354827418923378,
+      0.01597800850868225,
+      0.022786952555179596,
+      0.024067774415016174,
+      0.033419813960790634,
+      0.03571859002113342,
+      0.019361130893230438,
+      0.018805410712957382,
+      0.01901128701865673,
+      0.029003391042351723,
+      0.025346096605062485,
+      0.026195811107754707,
+      0.026159092783927917,
+      0.04387412592768669,
+      0.008549627847969532,
+      0.02158566564321518,
+      0.01986076310276985,
+      0.028879765421152115,
+      0.014282393269240856,
+      0.02485613524913788,
+      0.03125353530049324,
+      0.01986345276236534,
+      0.024233564734458923,
+      0.011902973987162113,
+      0.024577759206295013,
+      0.016042353585362434,
+      0.011987978592514992,
+      0.025742169469594955,
+      0.021011892706155777,
+      0.032602109014987946,
+      0.03912978619337082,
+      0.025027155876159668,
+      0.013109790161252022,
+      0.017444519326090813,
+      0.012893719598650932,
+      0.015557706356048584,
+      0.009281215257942677,
+      0.014317827299237251,
+      0.011324967257678509,
+      0.015544028021395206,
+      0.023885592818260193,
+      0.034219738095998764,
+      0.016379375010728836,
+      0.02090553380548954,
+      0.012327917851507664,
+      0.007747444324195385,
+      0.014652317389845848,
+      0.03546038642525673,
+      0.016635488718748093,
+      0.019176919013261795,
+      0.026759976521134377,
+      0.015274486504495144,
+      0.0301537923514843,
+      0.023401426151394844,
+      0.012518497183918953,
+      0.01830344647169113,
+      0.0398789681494236,
+      0.027572210878133774,
+      0.01484921295195818,
+      0.02567104622721672,
+      0.019835088402032852,
+      0.021890200674533844,
+      0.013211222365498543,
+      0.016198618337512016,
+      0.027684208005666733,
+      0.03246186301112175,
+      0.025014076381921768,
+      0.031282939016819,
+      0.019341079518198967,
+      0.01121007651090622,
+      0.05722726881504059,
+      0.01904161088168621,
+      0.017972854897379875,
+      0.029342208057641983,
+      0.02938135340809822,
+      0.01949606090784073,
+      0.030802447348833084,
+      0.02320772409439087,
+      0.012835274450480938,
+      0.014723509550094604,
+      0.020546797662973404,
+      0.022897016257047653,
+      0.0207335464656353,
+      0.016994059085845947,
+      0.020833881571888924,
+      0.03790897876024246,
+      0.014503776095807552,
+      0.015054943971335888,
+      0.019398193806409836,
+      0.03550693765282631,
+      0.013697671703994274,
+      0.017846396192908287,
+      0.04219169169664383,
+      0.014930365607142448,
+      0.019009530544281006,
+      0.013570532202720642,
+      0.028888355940580368,
+      0.024936437606811523,
+      0.019072866067290306,
+      0.03190300613641739,
+      0.01832304522395134,
+      0.013086436316370964,
+      0.021511321887373924,
+      0.02479047141969204,
+      0.02936503291130066,
+      0.030410798266530037,
+      0.029518894851207733,
+      0.0086666289716959,
+      0.02178276889026165,
+      0.023903440684080124,
+      0.02072538062930107,
+      0.027063846588134766,
+      0.018409613519906998,
+      0.022470705211162567,
+      0.016372838988900185,
+      0.015885839238762856,
+      0.018846673890948296,
+      0.019834408536553383,
+      0.016668323427438736,
+      0.009349048137664795,
+      0.018301332369446754,
+      0.015596598386764526,
+      0.012940001673996449,
+      0.014462772756814957,
+      0.017800532281398773,
+      0.017340993508696556,
+      0.03484665974974632,
+      0.023790506646037102,
+      0.022296089679002762,
+      0.023357149213552475,
+      0.02819024957716465,
+      0.018858222290873528,
+      0.027981378138065338,
+      0.012247433885931969
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.8152768015861511,
+      0.6726512312889099,
+      0.4814867675304413,
+      0.33432450890541077,
+      0.23814408481121063,
+      0.19757285714149475,
+      0.15554194152355194,
+      0.11246888339519501,
+      0.08865009248256683,
+      0.06748250871896744,
+      0.053425274789333344,
+      0.046865254640579224,
+      0.045605823397636414,
+      0.05257435888051987,
+      0.05854204297065735,
+      0.05769485980272293,
+      0.05553864315152168,
+      0.05860234424471855,
+      0.05860647186636925,
+      0.061698440462350845,
+      0.05941865220665932,
+      0.0612298808991909,
+      0.06598775833845139,
+      0.07615455240011215,
+      0.08855517953634262,
+      0.09619942307472229,
+      0.10664940625429153,
+      0.11464599519968033,
+      0.12268520891666412,
+      0.12900641560554504,
+      0.13469013571739197,
+      0.14923489093780518,
+      0.1556326299905777,
+      0.1598109006881714,
+      0.16741205751895905,
+      0.17854365706443787,
+      0.18381978571414948,
+      0.18534529209136963,
+      0.19115521013736725,
+      0.2001439779996872,
+      0.19999121129512787,
+      0.1972033977508545,
+      0.2024083286523819,
+      0.21319273114204407,
+      0.2191774547100067,
+      0.22613489627838135,
+      0.23462429642677307,
+      0.236800879240036,
+      0.23339171707630157,
+      0.23247872292995453,
+      0.22817456722259521,
+      0.23328275978565216,
+      0.22897255420684814,
+      0.2329391986131668,
+      0.2300928682088852,
+      0.23787039518356323,
+      0.23789697885513306,
+      0.2353580743074417,
+      0.2348659485578537,
+      0.2367279827594757,
+      0.23499399423599243,
+      0.22776924073696136,
+      0.23030073940753937,
+      0.2326144278049469,
+      0.2361035943031311,
+      0.2445068508386612,
+      0.24152973294258118,
+      0.2450699806213379,
+      0.24448467791080475,
+      0.250771701335907,
+      0.25215527415275574,
+      0.25353771448135376,
+      0.25054094195365906,
+      0.2454148232936859,
+      0.24659790098667145,
+      0.2457381933927536,
+      0.2508152425289154,
+      0.24626988172531128,
+      0.24414300918579102,
+      0.24193887412548065,
+      0.24609608948230743,
+      0.2430245280265808,
+      0.23835469782352448,
+      0.2364993542432785,
+      0.23474615812301636,
+      0.24188847839832306,
+      0.24022173881530762,
+      0.23649728298187256,
+      0.24024173617362976,
+      0.2449282854795456,
+      0.24722281098365784,
+      0.25538912415504456,
+      0.25413841009140015,
+      0.2641955316066742,
+      0.26872268319129944,
+      0.27901124954223633,
+      0.2832791209220886,
+      0.28473448753356934,
+      0.2893539071083069,
+      0.2969771921634674,
+      0.29966458678245544,
+      0.30459243059158325,
+      0.3059445023536682,
+      0.3044772744178772,
+      0.30405980348587036,
+      0.300637423992157,
+      0.3052726089954376,
+      0.3074321150779724,
+      0.3125714659690857,
+      0.3208770453929901,
+      0.3181338310241699,
+      0.31461143493652344,
+      0.3183312714099884,
+      0.32154226303100586,
+      0.3266172707080841,
+      0.3271406590938568,
+      0.3389417231082916,
+      0.3395279049873352,
+      0.3419077694416046,
+      0.3445950448513031,
+      0.35119402408599854,
+      0.3521731495857239,
+      0.35495105385780334,
+      0.36074918508529663,
+      0.36458420753479004,
+      0.36634451150894165,
+      0.371910035610199,
+      0.37581852078437805,
+      0.37285804748535156,
+      0.3766673505306244,
+      0.3756151497364044,
+      0.3773656189441681,
+      0.38035324215888977,
+      0.3852686882019043,
+      0.38913479447364807,
+      0.3900114595890045,
+      0.3948867619037628,
+      0.39293619990348816,
+      0.39555761218070984,
+      0.39592522382736206,
+      0.4046226143836975,
+      0.4075220823287964,
+      0.41104447841644287,
+      0.4172528088092804,
+      0.42129790782928467,
+      0.4249556362628937,
+      0.4267517328262329,
+      0.4287163317203522,
+      0.42599666118621826,
+      0.4283166229724884,
+      0.4292852580547333,
+      0.42729437351226807,
+      0.4295491576194763,
+      0.43094563484191895,
+      0.43256476521492004,
+      0.42930275201797485,
+      0.4368947148323059,
+      0.4383367896080017,
+      0.438157320022583,
+      0.4425742030143738,
+      0.4478302299976349,
+      0.45160001516342163,
+      0.4514777958393097,
+      0.4540557265281677,
+      0.45384475588798523,
+      0.45008739829063416,
+      0.4541735351085663,
+      0.45228561758995056,
+      0.4530966579914093,
+      0.453407883644104,
+      0.45556169748306274,
+      0.453635573387146,
+      0.449773371219635,
+      0.45001477003097534,
+      0.4513933062553406,
+      0.44678571820259094,
+      0.4457469880580902,
+      0.44657009840011597,
+      0.4523846507072449,
+      0.45246222615242004,
+      0.45434319972991943,
+      0.4541180431842804,
+      0.4578692615032196,
+      0.45711302757263184,
+      0.46016860008239746,
+      0.4629407227039337,
+      0.4631102979183197,
+      0.4657675623893738,
+      0.4634703993797302,
+      0.46504759788513184,
+      0.46310746669769287,
+      0.46426522731781006,
+      0.4652872085571289,
+      0.46551936864852905,
+      0.4635309875011444,
+      0.4650011360645294,
+      0.46778714656829834,
+      0.47348666191101074,
+      0.4748757779598236,
+      0.47525838017463684,
+      0.4781689941883087,
+      0.48228567838668823,
+      0.4811077117919922,
+      0.4817521274089813,
+      0.48237502574920654,
+      0.4820065498352051,
+      0.4831085503101349,
+      0.48647210001945496,
+      0.4832170307636261,
+      0.48424914479255676,
+      0.48679521679878235,
+      0.4905107319355011,
+      0.49074456095695496,
+      0.4933336079120636,
+      0.49535050988197327,
+      0.4951172173023224,
+      0.49591314792633057,
+      0.4948977530002594,
+      0.49568623304367065,
+      0.5002222657203674,
+      0.5023362040519714,
+      0.5024778842926025,
+      0.5055652856826782,
+      0.5075784921646118,
+      0.5095406770706177,
+      0.5118539929389954,
+      0.511347770690918,
+      0.5117764472961426,
+      0.5124334096908569,
+      0.5130148530006409,
+      0.5133326053619385,
+      0.515716016292572,
+      0.5170972347259521,
+      0.5180520415306091,
+      0.5196098685264587,
+      0.5221031308174133,
+      0.522267758846283,
+      0.5233892798423767,
+      0.5227095484733582,
+      0.5232641100883484,
+      0.5240395069122314,
+      0.5247151851654053,
+      0.5254992246627808,
+      0.5263919234275818,
+      0.5275459289550781,
+      0.5269024968147278,
+      0.5272529125213623,
+      0.5291876792907715,
+      0.5285334587097168,
+      0.5293962359428406,
+      0.5301678776741028,
+      0.5318194031715393,
+      0.5315480828285217,
+      0.5318052768707275,
+      0.5330978035926819,
+      0.5335772037506104,
+      0.5345197319984436,
+      0.5347660779953003,
+      0.5355949401855469,
+      0.5359938144683838,
+      0.5357021689414978,
+      0.5360286235809326,
+      0.5362545251846313,
+      0.5360153913497925,
+      0.5356314778327942,
+      0.5356934666633606,
+      0.5366938710212708,
+      0.5371251702308655,
+      0.5377495884895325,
+      0.5385035872459412,
+      0.5396519899368286,
+      0.5399116277694702,
+      0.5400606393814087,
+      0.5396886467933655,
+      0.5392147302627563,
+      0.5406867265701294,
+      0.541016161441803,
+      0.5407205820083618,
+      0.5415191650390625,
+      0.5420059561729431,
+      0.5414324402809143,
+      0.5414032340049744,
+      0.541130006313324,
+      0.541287362575531,
+      0.5413640141487122,
+      0.5415932536125183,
+      0.5416753888130188,
+      0.5421273112297058,
+      0.5426938533782959,
+      0.5429869890213013,
+      0.5438989400863647,
+      0.54445481300354,
+      0.544770359992981,
+      0.5452145338058472,
+      0.5455626249313354,
+      0.5453451871871948,
+      0.5460649728775024,
+      0.5464237928390503,
+      0.5463855266571045,
+      0.5465420484542847,
+      0.5464277267456055,
+      0.5472511053085327,
+      0.5475924015045166,
+      0.5475457906723022,
+      0.5479155778884888,
+      0.5478512048721313,
+      0.5478373765945435,
+      0.547956645488739,
+      0.5484586954116821,
+      0.5486555695533752
+    ],
+    "lr": [
+      4.188034188034189e-06,
+      8.461538461538462e-06,
+      1.2735042735042738e-05,
+      1.700854700854701e-05,
+      2.1282051282051282e-05,
+      2.5555555555555557e-05,
+      2.9829059829059833e-05,
+      3.4102564102564105e-05,
+      3.837606837606838e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.992995647012267e-05,
+      3.963316185199842e-05,
+      3.933636723387416e-05,
+      3.90395726157499e-05,
+      3.874277799762565e-05,
+      3.844598337950139e-05,
+      3.814918876137713e-05,
+      3.785239414325287e-05,
+      3.7555599525128613e-05,
+      3.725880490700435e-05,
+      3.696201028888009e-05,
+      3.666521567075585e-05,
+      3.6368421052631585e-05,
+      3.607162643450732e-05,
+      3.577483181638307e-05,
+      3.547803719825881e-05,
+      3.518124258013455e-05,
+      3.4884447962010287e-05,
+      3.458765334388604e-05,
+      3.4290858725761776e-05,
+      3.399406410763751e-05,
+      3.3697269489513265e-05,
+      3.3400474871389e-05,
+      3.310368025326474e-05,
+      3.280688563514048e-05,
+      3.251009101701623e-05,
+      3.2213296398891967e-05,
+      3.191650178076771e-05,
+      3.1542540561931145e-05,
+      3.124574594380688e-05,
+      3.094895132568263e-05,
+      3.065215670755838e-05,
+      3.0355362089434113e-05,
+      3.0058567471309854e-05,
+      2.97617728531856e-05,
+      2.9464978235061336e-05,
+      2.9168183616937077e-05,
+      2.8871388998812818e-05,
+      2.857459438068857e-05,
+      2.8277799762564307e-05,
+      2.798100514444005e-05,
+      2.7684210526315793e-05,
+      2.738741590819153e-05,
+      2.709062129006727e-05,
+      2.6793826671943013e-05,
+      2.6497032053818764e-05,
+      2.62002374356945e-05,
+      2.5903442817570243e-05,
+      2.5606648199445987e-05,
+      2.5309853581321725e-05,
+      2.5013058963197466e-05,
+      2.4716264345073214e-05,
+      2.441946972694896e-05,
+      2.4122675108824693e-05,
+      2.3825880490700434e-05,
+      2.352908587257618e-05,
+      2.3232291254451916e-05,
+      2.293549663632766e-05,
+      2.2638702018203408e-05,
+      2.226474079936684e-05,
+      2.196794618124258e-05,
+      2.1671151563118328e-05,
+      2.1374356944994066e-05,
+      2.1077562326869803e-05,
+      2.0780767708745554e-05,
+      2.0483973090621296e-05,
+      2.018717847249703e-05,
+      1.989038385437277e-05,
+      1.959358923624852e-05,
+      1.929679461812426e-05,
+      1.8999999999999998e-05,
+      1.8703205381875746e-05,
+      1.8406410763751487e-05,
+      1.8109616145627224e-05,
+      1.7812821527502965e-05,
+      1.7516026909378713e-05,
+      1.7219232291254454e-05,
+      1.6922437673130192e-05,
+      1.662564305500594e-05,
+      1.632884843688168e-05,
+      1.603205381875742e-05,
+      1.573525920063316e-05,
+      1.5438464582508908e-05,
+      1.5141669964384647e-05,
+      1.4844875346260385e-05,
+      1.4548080728136134e-05,
+      1.4251286110011875e-05,
+      1.3954491491887611e-05,
+      1.3657696873763363e-05,
+      1.33609022556391e-05,
+      1.298694103680253e-05,
+      1.2690146418678277e-05,
+      1.2393351800554018e-05,
+      1.2096557182429756e-05,
+      1.1799762564305506e-05,
+      1.1502967946181245e-05,
+      1.1206173328056984e-05,
+      1.0909378709932724e-05,
+      1.0612584091808472e-05,
+      1.0315789473684213e-05,
+      1.0018994855559949e-05,
+      9.7222002374357e-06,
+      9.425405619311438e-06,
+      9.128611001187177e-06,
+      8.831816383062916e-06,
+      8.535021764938666e-06,
+      8.238227146814405e-06,
+      7.941432528690143e-06,
+      7.644637910565893e-06,
+      7.347843292441632e-06,
+      7.0510486743173715e-06,
+      6.75425405619311e-06,
+      6.45745943806886e-06,
+      6.160664819944598e-06,
+      5.8638702018203375e-06,
+      5.567075583696086e-06,
+      5.270280965571826e-06,
+      4.973486347447565e-06,
+      4.676691729323314e-06,
+      4.379897111199053e-06,
+      4.083102493074792e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      1550,
+      3113,
+      4676,
+      6239,
+      7802,
+      9365,
+      10928,
+      12491,
+      14054,
+      15617
+    ],
+    "eval_accuracy": [
+      0.46,
+      0.94,
+      0.9,
+      0.99,
+      0.98,
+      1.0,
+      1.0,
+      1.0,
+      1.0,
+      1.0
+    ]
+  },
+  "final_accuracy": 0.9958333333333333,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9950980392156863,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9940828402366864,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9932432432432432,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9923076923076923,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9885057471264368,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.71,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.8494623655913979,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.9910313901345291,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.52,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.7375,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.8892508143322475,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.33,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.44,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.7025,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.63,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.66,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.784,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.985,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.994328922495274,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.88,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9378238341968912,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.78,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.921875,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.9236111111111112,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.74,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9183006535947712,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.9329896907216495,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.76,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9371584699453552,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9743589743589743,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966722129783694,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.9797979797979798,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.989247311827957,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.993103448275862,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9953051643192489,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.994475138121547,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.78,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9902912621359223,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.8590604026845637,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.4,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.62,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.8566666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.32,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.69,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.7375,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.945,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9866666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9908883826879271,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.86,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9933333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.949238578680203,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.9611650485436893,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.82,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.99,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9433198380566802,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.9607843137254902,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.87,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9630872483221476,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.9900990099009901,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.7966666666666666,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 4,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.9958333333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.9958333333333333,
+  "sft_overall_accuracy": 0.7966666666666666
+}

add_sub_sorl_v1_abs20_100K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb9ee27e51d6420f0e02315ecdd519a27da3ee37260946d31d57b82f22e4a5db
+size 650344480

add_sub_sorl_v1_abs20_100K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 1562,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs20_K4_100K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 20,
+  "dataset_size": 100000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162509462,
+  "run_name": "add_sub_sorl_v1_abs20_100K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T13:58:49.377924+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "xn0zx2ga",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/xn0zx2ga",
+  "final_accuracy": 0.9958333333333333,
+  "sft_accuracy": 0.7966666666666666,
+  "eval_method": "ArithmeticEvaluator"
+}