Upload add_sub_sorl_v1_abs16_K1_100K

add_sub_sorl_v1_abs16_K1_100K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151660
+}

add_sub_sorl_v1_abs16_K1_100K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs16_K1_100K/metrics.json

@@ -0,0 +1,3537 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1613,
+      1663,
+      1713,
+      1763,
+      1813,
+      1863,
+      1913,
+      1963,
+      2013,
+      2063,
+      2113,
+      2163,
+      2213,
+      2263,
+      2313,
+      2363,
+      2413,
+      2463,
+      2513,
+      2563,
+      2613,
+      2663,
+      2713,
+      2763,
+      2813,
+      2863,
+      2913,
+      2963,
+      3013,
+      3063,
+      3113,
+      3176,
+      3226,
+      3276,
+      3326,
+      3376,
+      3426,
+      3476,
+      3526,
+      3576,
+      3626,
+      3676,
+      3726,
+      3776,
+      3826,
+      3876,
+      3926,
+      3976,
+      4026,
+      4076,
+      4126,
+      4176,
+      4226,
+      4276,
+      4326,
+      4376,
+      4426,
+      4476,
+      4526,
+      4576,
+      4626,
+      4676,
+      4739,
+      4789,
+      4839,
+      4889,
+      4939,
+      4989,
+      5039,
+      5089,
+      5139,
+      5189,
+      5239,
+      5289,
+      5339,
+      5389,
+      5439,
+      5489,
+      5539,
+      5589,
+      5639,
+      5689,
+      5739,
+      5789,
+      5839,
+      5889,
+      5939,
+      5989,
+      6039,
+      6089,
+      6139,
+      6189,
+      6239,
+      6302,
+      6352,
+      6402,
+      6452,
+      6502,
+      6552,
+      6602,
+      6652,
+      6702,
+      6752,
+      6802,
+      6852,
+      6902,
+      6952,
+      7002,
+      7052,
+      7102,
+      7152,
+      7202,
+      7252,
+      7302,
+      7352,
+      7402,
+      7452,
+      7502,
+      7552,
+      7602,
+      7652,
+      7702,
+      7752,
+      7802,
+      7865,
+      7915,
+      7965,
+      8015,
+      8065,
+      8115,
+      8165,
+      8215,
+      8265,
+      8315,
+      8365,
+      8415,
+      8465,
+      8515,
+      8565,
+      8615,
+      8665,
+      8715,
+      8765,
+      8815,
+      8865,
+      8915,
+      8965,
+      9015,
+      9065,
+      9115,
+      9165,
+      9215,
+      9265,
+      9315,
+      9365,
+      9428,
+      9478,
+      9528,
+      9578,
+      9628,
+      9678,
+      9728,
+      9778,
+      9828,
+      9878,
+      9928,
+      9978,
+      10028,
+      10078,
+      10128,
+      10178,
+      10228,
+      10278,
+      10328,
+      10378,
+      10428,
+      10478,
+      10528,
+      10578,
+      10628,
+      10678,
+      10728,
+      10778,
+      10828,
+      10878,
+      10928,
+      10991,
+      11041,
+      11091,
+      11141,
+      11191,
+      11241,
+      11291,
+      11341,
+      11391,
+      11441,
+      11491,
+      11541,
+      11591,
+      11641,
+      11691,
+      11741,
+      11791,
+      11841,
+      11891,
+      11941,
+      11991,
+      12041,
+      12091,
+      12141,
+      12191,
+      12241,
+      12291,
+      12341,
+      12391,
+      12441,
+      12491,
+      12554,
+      12604,
+      12654,
+      12704,
+      12754,
+      12804,
+      12854,
+      12904,
+      12954,
+      13004,
+      13054,
+      13104,
+      13154,
+      13204,
+      13254,
+      13304,
+      13354,
+      13404,
+      13454,
+      13504,
+      13554,
+      13604,
+      13654,
+      13704,
+      13754,
+      13804,
+      13854,
+      13904,
+      13954,
+      14004,
+      14054,
+      14117,
+      14167,
+      14217,
+      14267,
+      14317,
+      14367,
+      14417,
+      14467,
+      14517,
+      14567,
+      14617,
+      14667,
+      14717,
+      14767,
+      14817,
+      14867,
+      14917,
+      14967,
+      15017,
+      15067,
+      15117,
+      15167,
+      15217,
+      15267,
+      15317,
+      15367,
+      15417,
+      15467,
+      15517,
+      15567,
+      15617
+    ],
+    "loss": [
+      0.5577878952026367,
+      -5.576192378997803,
+      2.92632794380188,
+      6.3677544593811035,
+      4.931706428527832,
+      3.444701671600342,
+      2.8243117332458496,
+      2.680835723876953,
+      2.8331117630004883,
+      2.1978397369384766,
+      1.785182237625122,
+      -1.0539252758026123,
+      -4.413449764251709,
+      -4.683836936950684,
+      -1.8393806219100952,
+      -0.9088308811187744,
+      -0.6606338024139404,
+      -1.037458896636963,
+      -0.8005858659744263,
+      -0.8346942663192749,
+      -1.2183023691177368,
+      -1.3451085090637207,
+      -0.7315034866333008,
+      -0.8666692972183228,
+      -0.9414889216423035,
+      -0.8544734716415405,
+      -0.5415238738059998,
+      -0.672846794128418,
+      -0.4627988338470459,
+      -0.5561841726303101,
+      -0.1339910626411438,
+      -0.2756516933441162,
+      -0.5867788791656494,
+      -0.23007673025131226,
+      -0.3725597858428955,
+      -0.013699114322662354,
+      -0.35232609510421753,
+      -0.31320029497146606,
+      -0.2628806233406067,
+      -0.434626042842865,
+      -0.5744713544845581,
+      -0.6800232529640198,
+      -0.3654564619064331,
+      -0.0633208155632019,
+      -0.2754508852958679,
+      -0.2020871639251709,
+      -0.4733316898345947,
+      -0.8693190813064575,
+      -0.46003156900405884,
+      -0.8825334310531616,
+      -0.8942903280258179,
+      -1.2861601114273071,
+      -1.5128978490829468,
+      -0.6883290410041809,
+      -1.2115806341171265,
+      -0.935728907585144,
+      -0.6558809280395508,
+      -1.1138315200805664,
+      -0.9283034205436707,
+      -1.3189666271209717,
+      -1.0786223411560059,
+      -1.179178237915039,
+      -1.2783474922180176,
+      -0.6550447344779968,
+      -0.973206102848053,
+      -0.771237850189209,
+      -0.9966790080070496,
+      -0.6994829773902893,
+      -0.7976064085960388,
+      -1.0985994338989258,
+      -0.5450573563575745,
+      -1.087911605834961,
+      -0.4879663586616516,
+      -0.7040844559669495,
+      -0.8600607514381409,
+      -0.7831844091415405,
+      -0.8913910984992981,
+      -0.8135383725166321,
+      -0.40366506576538086,
+      -0.5484676957130432,
+      -0.8509911298751831,
+      -0.5910856127738953,
+      -0.8854434490203857,
+      -1.198923945426941,
+      -0.8080556988716125,
+      -0.8257648944854736,
+      -0.6811696887016296,
+      -1.00200617313385,
+      -0.5987846851348877,
+      -0.85723477602005,
+      -1.3445595502853394,
+      -0.7796697616577148,
+      -0.9819375872612,
+      -0.7470568418502808,
+      -0.9209647178649902,
+      -0.623537540435791,
+      -0.8396660089492798,
+      -0.6690676212310791,
+      -0.5330244898796082,
+      -0.8458381295204163,
+      -0.9740660786628723,
+      -0.678127646446228,
+      -0.8426990509033203,
+      -0.8173584342002869,
+      -0.7635793089866638,
+      -0.37154561281204224,
+      -0.9728248119354248,
+      -0.9165170192718506,
+      -0.7311003804206848,
+      -0.5706267356872559,
+      -0.6835071444511414,
+      -0.3656216263771057,
+      -0.7594492435455322,
+      -0.6943159699440002,
+      -0.6790836453437805,
+      -0.7314068078994751,
+      -0.41682055592536926,
+      -0.4713362157344818,
+      -0.6050920486450195,
+      -0.841689944267273,
+      -0.7866406440734863,
+      -0.6980839967727661,
+      -0.47133585810661316,
+      -0.5978288054466248,
+      -0.6560570001602173,
+      -0.939233124256134,
+      -0.5798382759094238,
+      -0.48743996024131775,
+      -0.47128695249557495,
+      -0.8070036172866821,
+      -0.5778757333755493,
+      -0.22780683636665344,
+      -0.667661190032959,
+      -0.5630581378936768,
+      -0.645453155040741,
+      -0.619208574295044,
+      -0.5602623820304871,
+      -0.5149906873703003,
+      -0.7394415140151978,
+      -0.41501176357269287,
+      -0.579079270362854,
+      -0.42966264486312866,
+      -0.5831737518310547,
+      -0.36663153767585754,
+      -0.4568398594856262,
+      -0.5816318392753601,
+      -0.5446640253067017,
+      -0.47866857051849365,
+      -0.7518670558929443,
+      -0.4315052330493927,
+      -0.3823014199733734,
+      -0.40160974860191345,
+      -0.48745039105415344,
+      -0.7215253114700317,
+      -0.8161004185676575,
+      -0.3937722444534302,
+      -0.5529089570045471,
+      -0.633216142654419,
+      -0.7760704755783081,
+      -0.45726412534713745,
+      -0.4600589871406555,
+      -0.40411585569381714,
+      -0.48922643065452576,
+      -0.8256374001502991,
+      -0.27839645743370056,
+      -0.407863974571228,
+      -0.39419299364089966,
+      -0.5296006798744202,
+      -0.3459760546684265,
+      -0.4361375570297241,
+      -0.5114823579788208,
+      -0.3106434643268585,
+      -0.2481333464384079,
+      -0.3319697082042694,
+      -0.18109464645385742,
+      -0.5356259942054749,
+      -0.4689697027206421,
+      -0.5654739141464233,
+      -0.47701480984687805,
+      -0.419188916683197,
+      -0.3699048161506653,
+      -0.6753526329994202,
+      -0.5219122171401978,
+      -0.31527984142303467,
+      -0.4246196448802948,
+      -0.31242042779922485,
+      -0.6311987638473511,
+      -0.32913297414779663,
+      -0.3731696903705597,
+      -0.19184648990631104,
+      -0.3655150532722473,
+      -0.5271936655044556,
+      -0.5268361568450928,
+      -0.24535509943962097,
+      -0.34898850321769714,
+      -0.24170918762683868,
+      -0.7559969425201416,
+      -0.37633776664733887,
+      -0.17248903214931488,
+      -0.2222592979669571,
+      -0.4376280605792999,
+      -0.6540775299072266,
+      -0.3511269688606262,
+      -0.5187884569168091,
+      -0.20915870368480682,
+      -0.4059593975543976,
+      -0.2472882866859436,
+      -0.37154513597488403,
+      -0.3132503032684326,
+      -0.3342345058917999,
+      -0.35975271463394165,
+      -0.19919808208942413,
+      -0.5088240504264832,
+      -0.2929310202598572,
+      -0.30848902463912964,
+      -0.23250284790992737,
+      -0.39344364404678345,
+      -0.1125437468290329,
+      -0.2144632339477539,
+      -0.23171943426132202,
+      -0.21659919619560242,
+      -0.270476371049881,
+      -0.09964442253112793,
+      -0.45995014905929565,
+      -0.39641156792640686,
+      -0.31215667724609375,
+      -0.3928333818912506,
+      -0.5351556539535522,
+      -0.10821276158094406,
+      -0.2813533544540405,
+      -0.18374423682689667,
+      -0.31148767471313477,
+      -0.194771870970726,
+      -0.17680102586746216,
+      -0.15086638927459717,
+      -0.2995645999908447,
+      -0.06749001145362854,
+      -0.3007539212703705,
+      -0.2840707302093506,
+      -0.20631617307662964,
+      -0.37892067432403564,
+      -0.1025107353925705,
+      -0.4783245325088501,
+      -0.09549982100725174,
+      -0.049792978912591934,
+      -0.15029171109199524,
+      -0.38813191652297974,
+      -0.3833778500556946,
+      -0.1451372355222702,
+      -0.29522666335105896,
+      -0.21452951431274414,
+      -0.17052190005779266,
+      -0.21188636124134064,
+      -0.09555578976869583,
+      -0.04320298880338669,
+      -0.2637097239494324,
+      -0.10791129618883133,
+      -0.2244073450565338,
+      -0.22246918082237244,
+      -0.0481291189789772,
+      -0.15195731818675995,
+      -0.0761508047580719,
+      -0.18744207918643951,
+      -0.06599161773920059,
+      -0.13524764776229858,
+      -0.24970178306102753,
+      -0.27204227447509766,
+      -0.06557333469390869,
+      -0.05200962722301483,
+      -0.11020062863826752,
+      -0.082196444272995,
+      -0.06147323176264763,
+      -0.1582547277212143,
+      -0.09557494521141052,
+      -0.06250099837779999,
+      -0.04475772753357887,
+      -0.23515015840530396,
+      -0.025078849866986275,
+      -0.037300847470760345,
+      -0.10369005054235458,
+      -0.0211623627692461,
+      -0.17572464048862457,
+      0.12815015017986298,
+      -0.15703743696212769,
+      -0.04601459950208664,
+      -0.07747994363307953,
+      -0.0768294706940651,
+      -0.11141927540302277,
+      -0.09336625784635544,
+      -0.08718496561050415,
+      -0.024220900610089302,
+      -0.1684415489435196,
+      -0.10317432135343552,
+      -0.07274491339921951,
+      -0.04384593665599823,
+      -0.10860706865787506,
+      -0.04664432629942894,
+      -0.060206394642591476,
+      -0.00010688882321119308,
+      -0.16355904936790466,
+      -0.001930472906678915,
+      0.0015387262683361769,
+      -0.03080066479742527,
+      -0.013441846705973148,
+      -0.18671223521232605,
+      -0.014116063714027405,
+      -0.00717591680586338,
+      -0.006398052908480167,
+      -0.013063499704003334,
+      0.0017607174813747406
+    ],
+    "base_loss": [
+      11.33397388458252,
+      9.697375297546387,
+      8.228904724121094,
+      6.828713893890381,
+      5.360694885253906,
+      3.647108793258667,
+      2.2145321369171143,
+      2.0256307125091553,
+      1.882012128829956,
+      1.801147222518921,
+      1.808428168296814,
+      1.7667168378829956,
+      1.8179380893707275,
+      1.4938572645187378,
+      1.0333566665649414,
+      0.8400124311447144,
+      0.7365804314613342,
+      0.7016572952270508,
+      0.5986067056655884,
+      0.5793089270591736,
+      0.5988584756851196,
+      0.5955408811569214,
+      0.5806025862693787,
+      0.47322675585746765,
+      0.4693293571472168,
+      0.4685460031032562,
+      0.45813649892807007,
+      0.4436396658420563,
+      0.3591185510158539,
+      0.37346377968788147,
+      0.31526660919189453,
+      0.27813777327537537,
+      0.2814022898674011,
+      0.2628602087497711,
+      0.21686474978923798,
+      0.200835719704628,
+      0.21639907360076904,
+      0.19699202477931976,
+      0.19921962916851044,
+      0.25362253189086914,
+      0.22252197563648224,
+      0.2003440111875534,
+      0.1434318721294403,
+      0.1448654979467392,
+      0.1587533950805664,
+      0.14443540573120117,
+      0.15725888311862946,
+      0.1821482926607132,
+      0.16362155973911285,
+      0.16335923969745636,
+      0.1399972140789032,
+      0.21047517657279968,
+      0.2520601451396942,
+      0.12363635003566742,
+      0.20099785923957825,
+      0.17165043950080872,
+      0.1295781433582306,
+      0.1577463150024414,
+      0.1423770636320114,
+      0.18621452152729034,
+      0.15476252138614655,
+      0.1751738339662552,
+      0.2001466006040573,
+      0.1015189066529274,
+      0.15296132862567902,
+      0.13563570380210876,
+      0.15313532948493958,
+      0.12401551753282547,
+      0.120963454246521,
+      0.1488175392150879,
+      0.1064835861325264,
+      0.1720968335866928,
+      0.11307518929243088,
+      0.11429997533559799,
+      0.11505492776632309,
+      0.12347052991390228,
+      0.11809030920267105,
+      0.10877188295125961,
+      0.12448655813932419,
+      0.09284628182649612,
+      0.12196465581655502,
+      0.0911141186952591,
+      0.1277979016304016,
+      0.1557869166135788,
+      0.10748206079006195,
+      0.1222173348069191,
+      0.10195999592542648,
+      0.12632085382938385,
+      0.09333528578281403,
+      0.1125306561589241,
+      0.20255793631076813,
+      0.10652425140142441,
+      0.12565617263317108,
+      0.08823127299547195,
+      0.112433061003685,
+      0.1077231839299202,
+      0.11770574003458023,
+      0.0980721190571785,
+      0.06952495872974396,
+      0.13095995783805847,
+      0.11997576802968979,
+      0.08410906046628952,
+      0.10747019201517105,
+      0.09736161679029465,
+      0.09167403727769852,
+      0.077176533639431,
+      0.11459974199533463,
+      0.11962968856096268,
+      0.10145970433950424,
+      0.0934385433793068,
+      0.08291459083557129,
+      0.08955997228622437,
+      0.09785062074661255,
+      0.09278756380081177,
+      0.08140374720096588,
+      0.09077442437410355,
+      0.06354185193777084,
+      0.0802765041589737,
+      0.07504796981811523,
+      0.10086948424577713,
+      0.09547056257724762,
+      0.08453293144702911,
+      0.07368914037942886,
+      0.07332005351781845,
+      0.07625468820333481,
+      0.11547034233808517,
+      0.09177985042333603,
+      0.09278776496648788,
+      0.06013501435518265,
+      0.09825111925601959,
+      0.10280166566371918,
+      0.048720959573984146,
+      0.08903590589761734,
+      0.07288584858179092,
+      0.0784422978758812,
+      0.07730279117822647,
+      0.07461636513471603,
+      0.06492268294095993,
+      0.10379330068826675,
+      0.05712369456887245,
+      0.0705806091427803,
+      0.05283525958657265,
+      0.11746769398450851,
+      0.04911088943481445,
+      0.06225346773862839,
+      0.07111300528049469,
+      0.06430380791425705,
+      0.056710679084062576,
+      0.09580206871032715,
+      0.05351073667407036,
+      0.05775975435972214,
+      0.04881032556295395,
+      0.10242227464914322,
+      0.08975668996572495,
+      0.1047421470284462,
+      0.05592641234397888,
+      0.08422555029392242,
+      0.09594937413930893,
+      0.09964465349912643,
+      0.05885830894112587,
+      0.0627526119351387,
+      0.05889914557337761,
+      0.05978791043162346,
+      0.11384613811969757,
+      0.04738619178533554,
+      0.051085881888866425,
+      0.04796541482210159,
+      0.0731235221028328,
+      0.045254748314619064,
+      0.050247080624103546,
+      0.060893941670656204,
+      0.04258085414767265,
+      0.03052467480301857,
+      0.04046378657221794,
+      0.03291342407464981,
+      0.0679231509566307,
+      0.054861005395650864,
+      0.0646156519651413,
+      0.05885514244437218,
+      0.05755404010415077,
+      0.06085504963994026,
+      0.09060827642679214,
+      0.07897806912660599,
+      0.038967352360486984,
+      0.04996843263506889,
+      0.04916781932115555,
+      0.07155297696590424,
+      0.04094274714589119,
+      0.0858810767531395,
+      0.039560504257678986,
+      0.05347173660993576,
+      0.06020087003707886,
+      0.06770100444555283,
+      0.03783196955919266,
+      0.040525078773498535,
+      0.03269848972558975,
+      0.09151533991098404,
+      0.04719338193535805,
+      0.033506568521261215,
+      0.03560910373926163,
+      0.05119560286402702,
+      0.08401036262512207,
+      0.043813493102788925,
+      0.058830466121435165,
+      0.034354109317064285,
+      0.050553202629089355,
+      0.028983136638998985,
+      0.054248273372650146,
+      0.06970725208520889,
+      0.04324648901820183,
+      0.04174510017037392,
+      0.036401160061359406,
+      0.05823252350091934,
+      0.03477806597948074,
+      0.03613588958978653,
+      0.03327057138085365,
+      0.051082976162433624,
+      0.025655852630734444,
+      0.03574183210730553,
+      0.029712524265050888,
+      0.025071779265999794,
+      0.03150422126054764,
+      0.011905641295015812,
+      0.05418887734413147,
+      0.04765428975224495,
+      0.03708108887076378,
+      0.04620677977800369,
+      0.06262367218732834,
+      0.013509207405149937,
+      0.03320235386490822,
+      0.021982263773679733,
+      0.03584248572587967,
+      0.022798042744398117,
+      0.0208269115537405,
+      0.018050367012619972,
+      0.034046467393636703,
+      0.010257250629365444,
+      0.03578177094459534,
+      0.03314688429236412,
+      0.027281632646918297,
+      0.04790187254548073,
+      0.013754768297076225,
+      0.053806185722351074,
+      0.011845839209854603,
+      0.014848830178380013,
+      0.0174210574477911,
+      0.044416483491659164,
+      0.043724652379751205,
+      0.017058920115232468,
+      0.03628867119550705,
+      0.02449323795735836,
+      0.019823655486106873,
+      0.02813252992928028,
+      0.011511008255183697,
+      0.0067415363155305386,
+      0.029995249584317207,
+      0.014533408917486668,
+      0.025971829891204834,
+      0.025442421436309814,
+      0.0151898842304945,
+      0.02171529270708561,
+      0.010393083095550537,
+      0.021415045484900475,
+      0.007882730104029179,
+      0.015499244444072247,
+      0.029081273823976517,
+      0.031534165143966675,
+      0.008034524507820606,
+      0.006916637998074293,
+      0.014973251149058342,
+      0.0100376782938838,
+      0.007728648837655783,
+      0.01831640489399433,
+      0.011213655583560467,
+      0.007832633331418037,
+      0.005861278157681227,
+      0.02696451172232628,
+      0.0031450821552425623,
+      0.004722945392131805,
+      0.012314162217080593,
+      0.0028751466888934374,
+      0.019880356267094612,
+      0.024764133617281914,
+      0.01799626462161541,
+      0.006194784305989742,
+      0.009251789189875126,
+      0.009436463937163353,
+      0.012919851578772068,
+      0.010917981155216694,
+      0.011143622919917107,
+      0.0032752251718193293,
+      0.02041933499276638,
+      0.013105714693665504,
+      0.00845302827656269,
+      0.005595696158707142,
+      0.013482270762324333,
+      0.006255302578210831,
+      0.007237650919705629,
+      0.0008033419726416469,
+      0.0185103602707386,
+      0.0007832656847313046,
+      0.000437746086390689,
+      0.0038669146597385406,
+      0.0022584940306842327,
+      0.021514346823096275,
+      0.002147300634533167,
+      0.0020445866975933313,
+      0.0016196088399738073,
+      0.0020003675017505884,
+      0.00033240352058783174
+    ],
+    "info_loss": [
+      -1.8722295761108398,
+      -2.2108302116394043,
+      -1.0570645332336426,
+      -0.41002845764160156,
+      -0.2738471031188965,
+      -0.19143915176391602,
+      -0.09005904197692871,
+      -0.08029019832611084,
+      -0.049259066581726074,
+      -0.10430407524108887,
+      -0.1458674669265747,
+      -0.42581772804260254,
+      -0.7660454511642456,
+      -0.7605015635490417,
+      -0.42735755443573,
+      -0.3107941746711731,
+      -0.2712852656841278,
+      -0.3018323481082916,
+      -0.2649194598197937,
+      -0.2597028911113739,
+      -0.29537349939346313,
+      -0.3107622563838959,
+      -0.24403095245361328,
+      -0.24395069479942322,
+      -0.24592307209968567,
+      -0.23565277457237244,
+      -0.20078688859939575,
+      -0.2010294795036316,
+      -0.16992546617984772,
+      -0.18197181820869446,
+      -0.12281505763530731,
+      -0.12788164615631104,
+      -0.15659582614898682,
+      -0.11880415678024292,
+      -0.1262507140636444,
+      -0.08458173274993896,
+      -0.11665385216474533,
+      -0.11172604560852051,
+      -0.10635123401880264,
+      -0.1264892965555191,
+      -0.1336090862751007,
+      -0.13750332593917847,
+      -0.10439354181289673,
+      -0.06636995077133179,
+      -0.08734632283449173,
+      -0.07475010305643082,
+      -0.09220811724662781,
+      -0.137499138712883,
+      -0.09056003391742706,
+      -0.11914384365081787,
+      -0.11546589434146881,
+      -0.1596410572528839,
+      -0.1853722631931305,
+      -0.08901497721672058,
+      -0.14770330488681793,
+      -0.11823290586471558,
+      -0.08599165827035904,
+      -0.13189618289470673,
+      -0.11319698393344879,
+      -0.15432821214199066,
+      -0.12706328928470612,
+      -0.13852959871292114,
+      -0.1510118544101715,
+      -0.07841464132070541,
+      -0.11543506383895874,
+      -0.09320459514856339,
+      -0.11888302117586136,
+      -0.084896519780159,
+      -0.09416110813617706,
+      -0.12740209698677063,
+      -0.06747395545244217,
+      -0.12878276407718658,
+      -0.06240946054458618,
+      -0.08402750641107559,
+      -0.099144347012043,
+      -0.09311512112617493,
+      -0.10234177857637405,
+      -0.09399470686912537,
+      -0.054853230714797974,
+      -0.06678619980812073,
+      -0.09873295575380325,
+      -0.06962869316339493,
+      -0.10274700820446014,
+      -0.1372026950120926,
+      -0.09274651855230331,
+      -0.09610535204410553,
+      -0.07988561689853668,
+      -0.11397822946310043,
+      -0.07051478326320648,
+      -0.09783930331468582,
+      -0.1560787409543991,
+      -0.0905139222741127,
+      -0.11244180053472519,
+      -0.08502411842346191,
+      -0.10473114252090454,
+      -0.07410141825675964,
+      -0.09664714336395264,
+      -0.07763437181711197,
+      -0.06174994260072708,
+      -0.09884490072727203,
+      -0.1106635257601738,
+      -0.07729820907115936,
+      -0.09621607512235641,
+      -0.09244702011346817,
+      -0.08659424632787704,
+      -0.04580191895365715,
+      -0.10990729928016663,
+      -0.10430049151182175,
+      -0.08413095772266388,
+      -0.06764105707406998,
+      -0.07739929109811783,
+      -0.04647885635495186,
+      -0.08686721324920654,
+      -0.0795186385512352,
+      -0.07688325643539429,
+      -0.08349419385194778,
+      -0.04904802143573761,
+      -0.05629279464483261,
+      -0.06858016550540924,
+      -0.09524913877248764,
+      -0.08893893659114838,
+      -0.07907679677009583,
+      -0.055403802543878555,
+      -0.06796155124902725,
+      -0.07397491484880447,
+      -0.10638416558504105,
+      -0.06868835538625717,
+      -0.05880892649292946,
+      -0.05406094342470169,
+      -0.09153366833925247,
+      -0.06931941211223602,
+      -0.02864447981119156,
+      -0.0771365761756897,
+      -0.06434978544712067,
+      -0.0733770877122879,
+      -0.07072990387678146,
+      -0.064170703291893,
+      -0.05894353985786438,
+      -0.08493777364492416,
+      -0.04775642231106758,
+      -0.06566251069307327,
+      -0.048874225467443466,
+      -0.07127480208873749,
+      -0.042416784912347794,
+      -0.05257882922887802,
+      -0.06578890979290009,
+      -0.06188692897558212,
+      -0.054064612835645676,
+      -0.08549480140209198,
+      -0.049428876489400864,
+      -0.04480578005313873,
+      -0.045844681560993195,
+      -0.05952483043074608,
+      -0.08181554824113846,
+      -0.09271662682294846,
+      -0.04931681230664253,
+      -0.0682697519659996,
+      -0.07777585089206696,
+      -0.09169674664735794,
+      -0.05585508048534393,
+      -0.056745074689388275,
+      -0.0498771034181118,
+      -0.056404877454042435,
+      -0.09470423310995102,
+      -0.03340289741754532,
+      -0.04667876660823822,
+      -0.04499981179833412,
+      -0.06113607436418533,
+      -0.04001234471797943,
+      -0.04945958033204079,
+      -0.05818415805697441,
+      -0.03627463057637215,
+      -0.028624065220355988,
+      -0.03772645443677902,
+      -0.02219102531671524,
+      -0.06100322678685188,
+      -0.05316488817334175,
+      -0.06359907239675522,
+      -0.05425305664539337,
+      -0.04845690354704857,
+      -0.04391637444496155,
+      -0.07734991610050201,
+      -0.06084549427032471,
+      -0.03645513951778412,
+      -0.048169709742069244,
+      -0.036919742822647095,
+      -0.07092147320508957,
+      -0.03750623017549515,
+      -0.04641468822956085,
+      -0.02410457655787468,
+      -0.04249722138047218,
+      -0.05939014256000519,
+      -0.06024573743343353,
+      -0.0289095938205719,
+      -0.039603594690561295,
+      -0.02823018841445446,
+      -0.0855451449751854,
+      -0.04393450543284416,
+      -0.021329475566744804,
+      -0.02656519040465355,
+      -0.049449898302555084,
+      -0.07447291165590286,
+      -0.04019889980554581,
+      -0.05821258947253227,
+      -0.024868115782737732,
+      -0.04655325040221214,
+      -0.02830430492758751,
+      -0.043234482407569885,
+      -0.03885981813073158,
+      -0.038678355515003204,
+      -0.041147783398628235,
+      -0.024361640214920044,
+      -0.05759088695049286,
+      -0.03391062095761299,
+      -0.03501441702246666,
+      -0.02738049626350403,
+      -0.045669812709093094,
+      -0.014750971458852291,
+      -0.026065301150083542,
+      -0.027000879868865013,
+      -0.02488401159644127,
+      -0.031408023089170456,
+      -0.01161701139062643,
+      -0.05200820788741112,
+      -0.046356432139873505,
+      -0.03545088320970535,
+      -0.0444461815059185,
+      -0.06045794486999512,
+      -0.013400518335402012,
+      -0.0324067659676075,
+      -0.021174244582653046,
+      -0.035626500844955444,
+      -0.022704709321260452,
+      -0.020717144012451172,
+      -0.01795428805053234,
+      -0.0339086651802063,
+      -0.00871788989752531,
+      -0.03418224677443504,
+      -0.03249115124344826,
+      -0.023991618305444717,
+      -0.043025609105825424,
+      -0.012155727483332157,
+      -0.05362624302506447,
+      -0.011768711730837822,
+      -0.00748388422653079,
+      -0.017160627990961075,
+      -0.044271551072597504,
+      -0.04344240203499794,
+      -0.016890574246644974,
+      -0.03395989164710045,
+      -0.02439269982278347,
+      -0.019768577069044113,
+      -0.024975083768367767,
+      -0.011333119124174118,
+      -0.005900434218347073,
+      -0.02994467318058014,
+      -0.013382021337747574,
+      -0.025662653148174286,
+      -0.025401990860700607,
+      -0.006823656149208546,
+      -0.018227649852633476,
+      -0.009130764752626419,
+      -0.02137848362326622,
+      -0.007732507772743702,
+      -0.015459390357136726,
+      -0.028814582154154778,
+      -0.031098704785108566,
+      -0.007990011014044285,
+      -0.006862882524728775,
+      -0.013380587100982666,
+      -0.009976633824408054,
+      -0.007156243547797203,
+      -0.018262408673763275,
+      -0.011157645843923092,
+      -0.007779450621455908,
+      -0.005826402921229601,
+      -0.026892339810729027,
+      -0.0030956417322158813,
+      -0.0046788109466433525,
+      -0.012022721581161022,
+      -0.002801282564178109,
+      -0.01982787251472473,
+      0.009974746033549309,
+      -0.017948439344763756,
+      -0.0061605749651789665,
+      -0.009149443358182907,
+      -0.00940472912043333,
+      -0.012833412736654282,
+      -0.010864147916436195,
+      -0.010289489291608334,
+      -0.00324008846655488,
+      -0.019436735659837723,
+      -0.012104923836886883,
+      -0.008401187136769295,
+      -0.005556115880608559,
+      -0.013448841869831085,
+      -0.006224552169442177,
+      -0.007203326560556889,
+      -0.000765089935157448,
+      -0.018467390909790993,
+      -0.0007461291970685124,
+      -0.00041246030014008284,
+      -0.0038027395494282246,
+      -0.0021796596702188253,
+      -0.021332552656531334,
+      -0.002122311620041728,
+      -0.002004181733354926,
+      -0.0015925208572298288,
+      -0.001970382872968912,
+      -0.0003019475843757391
+    ],
+    "abs_loss": [
+      2.7695558071136475,
+      2.757258176803589,
+      2.714398145675659,
+      2.612370729446411,
+      2.4649007320404053,
+      2.323071002960205,
+      2.273024797439575,
+      2.2605059146881104,
+      2.2603180408477783,
+      2.263150453567505,
+      2.2445919513702393,
+      2.252319097518921,
+      2.178196907043457,
+      2.1821935176849365,
+      1.9552990198135376,
+      1.826480746269226,
+      1.7979575395584106,
+      1.5754321813583374,
+      1.51059091091156,
+      1.3168487548828125,
+      1.2136815786361694,
+      1.1017318964004517,
+      1.0320855379104614,
+      0.9946089386940002,
+      0.9091302752494812,
+      0.9446201920509338,
+      0.826180636882782,
+      0.8360268473625183,
+      0.7732840180397034,
+      0.6978147625923157,
+      0.7105684280395508,
+      0.6442034244537354,
+      0.6352532505989075,
+      0.5988442301750183,
+      0.5720152854919434,
+      0.6041375994682312,
+      0.5219680666923523,
+      0.505420982837677,
+      0.4944639205932617,
+      0.5343429446220398,
+      0.4464101791381836,
+      0.4692588746547699,
+      0.41510966420173645,
+      0.3662830889225006,
+      0.37847068905830383,
+      0.3181345760822296,
+      0.3445536196231842,
+      0.35482969880104065,
+      0.355588436126709,
+      0.3673471510410309,
+      0.3439052104949951,
+      0.2908039391040802,
+      0.32127058506011963,
+      0.2832740247249603,
+      0.27703914046287537,
+      0.23231802880764008,
+      0.24343381822109222,
+      0.24540118873119354,
+      0.19612455368041992,
+      0.17661559581756592,
+      0.22358156740665436,
+      0.16401363909244537,
+      0.18624664843082428,
+      0.15092390775680542,
+      0.15065866708755493,
+      0.13049976527690887,
+      0.11838517338037491,
+      0.11917757987976074,
+      0.13449154794216156,
+      0.11397969722747803,
+      0.13762305676937103,
+      0.13600696623325348,
+      0.11815961450338364,
+      0.11022745817899704,
+      0.07655175775289536,
+      0.09835848957300186,
+      0.09699732065200806,
+      0.08222716301679611,
+      0.06970739364624023,
+      0.07540616393089294,
+      0.07638673484325409,
+      0.07543123513460159,
+      0.06136744096875191,
+      0.07297835499048233,
+      0.06406604498624802,
+      0.06371459364891052,
+      0.05812959372997284,
+      0.06946182996034622,
+      0.04413483664393425,
+      0.05052522197365761,
+      0.060342367738485336,
+      0.07218147814273834,
+      0.05116567015647888,
+      0.05598502233624458,
+      0.04631705582141876,
+      0.03574751690030098,
+      0.055087048560380936,
+      0.04298175871372223,
+      0.05066549405455589,
+      0.04147268459200859,
+      0.0408233143389225,
+      0.045503806322813034,
+      0.03884589299559593,
+      0.04656253755092621,
+      0.03495315834879875,
+      0.03418383374810219,
+      0.02620612643659115,
+      0.03293178603053093,
+      0.030975719913840294,
+      0.044921740889549255,
+      0.04072905704379082,
+      0.0342169813811779,
+      0.02957664616405964,
+      0.03389245644211769,
+      0.023984842002391815,
+      0.01984020136296749,
+      0.03859636187553406,
+      0.04447132349014282,
+      0.017740679904818535,
+      0.03720969334244728,
+      0.019728712737560272,
+      0.01769821159541607,
+      0.021987849846482277,
+      0.0190268624573946,
+      0.016939802095294,
+      0.026036595925688744,
+      0.03319808840751648,
+      0.029679646715521812,
+      0.015514916740357876,
+      0.025532612577080727,
+      0.024698758497834206,
+      0.028365038335323334,
+      0.03000704012811184,
+      0.015057776123285294,
+      0.0270793866366148,
+      0.01922013610601425,
+      0.016823092475533485,
+      0.02009473741054535,
+      0.02750399149954319,
+      0.026774728670716286,
+      0.02902131713926792,
+      0.026612574234604836,
+      0.019634006544947624,
+      0.021729981526732445,
+      0.025850391015410423,
+      0.023152805864810944,
+      0.017385335639119148,
+      0.01789049804210663,
+      0.02443535067141056,
+      0.017224645242094994,
+      0.01826273463666439,
+      0.02485678531229496,
+      0.01320267841219902,
+      0.010790318250656128,
+      0.01471333485096693,
+      0.024667000398039818,
+      0.020168118178844452,
+      0.01574580743908882,
+      0.011433619074523449,
+      0.014809876680374146,
+      0.023700252175331116,
+      0.010335405357182026,
+      0.017168136313557625,
+      0.0254232045263052,
+      0.023459458723664284,
+      0.014551951549947262,
+      0.025152841582894325,
+      0.018016614019870758,
+      0.017615744844079018,
+      0.008140993304550648,
+      0.010460898280143738,
+      0.01902330107986927,
+      0.014583222568035126,
+      0.018572388216853142,
+      0.02234470844268799,
+      0.011642231605947018,
+      0.016363034024834633,
+      0.015224513597786427,
+      0.018041061237454414,
+      0.016325196251273155,
+      0.022254878655076027,
+      0.012667966075241566,
+      0.011075119487941265,
+      0.01190368551760912,
+      0.018234197050333023,
+      0.01418225932866335,
+      0.012023826129734516,
+      0.010663360357284546,
+      0.01986275613307953,
+      0.008102544583380222,
+      0.01361160445958376,
+      0.006008542608469725,
+      0.01946125365793705,
+      0.016349898651242256,
+      0.0203538965433836,
+      0.008983378298580647,
+      0.0072066509164869785,
+      0.009305398911237717,
+      0.013738664798438549,
+      0.008193968795239925,
+      0.006817294750362635,
+      0.009356106631457806,
+      0.009478329680860043,
+      0.011687666177749634,
+      0.01932637393474579,
+      0.022637570276856422,
+      0.01438119262456894,
+      0.012074408121407032,
+      0.011916994117200375,
+      0.011993207968771458,
+      0.013026487082242966,
+      0.012276235036551952,
+      0.02072794921696186,
+      0.0105925677344203,
+      0.01057432685047388,
+      0.010463658720254898,
+      0.02401764504611492,
+      0.00971772987395525,
+      0.01738486997783184,
+      0.0072584873996675014,
+      0.008841030299663544,
+      0.015144255943596363,
+      0.009510762058198452,
+      0.004616790451109409,
+      0.005089740734547377,
+      0.0033849384635686874,
+      0.009031621739268303,
+      0.0070746722631156445,
+      0.0038986431900411844,
+      0.014924501068890095,
+      0.005439704284071922,
+      0.012228280305862427,
+      0.006074471864849329,
+      0.0041548931039869785,
+      0.0036205865908414125,
+      0.003559892298653722,
+      0.0030162332113832235,
+      0.00596613110974431,
+      0.0046607633121311665,
+      0.00393908703699708,
+      0.008519414812326431,
+      0.008389890193939209,
+      0.004085652995854616,
+      0.0039666821248829365,
+      0.0025419110897928476,
+      0.004632981959730387,
+      0.003359271213412285,
+      0.006698184181004763,
+      0.0010022107744589448,
+      0.005775243043899536,
+      0.0029002123046666384,
+      0.0070551130920648575,
+      0.0052612400613725185,
+      0.005106158088892698,
+      0.007022228091955185,
+      0.004999684169888496,
+      0.022510403767228127,
+      0.007338444236665964,
+      0.003092102939262986,
+      0.003852828638628125,
+      0.007366009056568146,
+      0.002055574906989932,
+      0.002716584363952279,
+      0.004305695649236441,
+      0.0008842918905429542,
+      0.0020649160724133253,
+      0.00220546405762434,
+      0.0045172651298344135,
+      0.002134924754500389,
+      0.00198660627938807,
+      0.002547383075580001,
+      0.0018582269549369812,
+      0.004609962459653616,
+      0.0029482373502105474,
+      0.001209548208862543,
+      0.0038016096223145723,
+      0.0022803116589784622,
+      0.0009680527145974338,
+      0.0047198208048939705,
+      0.0018950551748275757,
+      0.002245184499770403,
+      0.004644941072911024,
+      0.003991647623479366,
+      0.00459820544347167,
+      0.003601460950449109,
+      0.0033293275628238916,
+      0.0008270890102721751,
+      0.0011858651414513588,
+      0.00523713231086731,
+      0.003144143847748637,
+      0.0052079614251852036,
+      0.0006806037272326648,
+      0.0058856843970716,
+      0.0013082782970741391,
+      0.0019983849488198757,
+      0.0007600346580147743,
+      0.0013144537806510925,
+      0.003953778184950352,
+      0.0017438801005482674,
+      0.0006400617421604693,
+      0.0023446576669812202,
+      0.005105599295347929,
+      0.0012087664799764752,
+      0.001198558951728046,
+      0.001190866343677044,
+      0.0010707571636885405,
+      0.004463807214051485,
+      0.0012607508106157184,
+      0.0013424685457721353,
+      0.0005274586728774011
+    ],
+    "zipf_loss": [
+      7.669154167175293,
+      6.559010028839111,
+      4.996628761291504,
+      3.3780879974365234,
+      2.062992572784424,
+      1.479677438735962,
+      1.2830677032470703,
+      1.2320564985275269,
+      1.2176584005355835,
+      1.2134181261062622,
+      1.2109695672988892,
+      1.2123031616210938,
+      1.211247444152832,
+      1.2091020345687866,
+      1.2053085565567017,
+      1.1764500141143799,
+      1.1358428001403809,
+      1.121664047241211,
+      1.0989431142807007,
+      1.0513408184051514,
+      1.015205979347229,
+      1.0567998886108398,
+      1.0249948501586914,
+      1.0001500844955444,
+      0.9574993252754211,
+      0.9390462636947632,
+      0.9255902171134949,
+      0.8102058172225952,
+      0.8000087738037109,
+      0.8202887773513794,
+      0.7078360915184021,
+      0.6606066226959229,
+      0.6342517137527466,
+      0.6352202296257019,
+      0.6158812046051025,
+      0.570868730545044,
+      0.5456165075302124,
+      0.5565260052680969,
+      0.5519656538963318,
+      0.5232100486755371,
+      0.49445641040802,
+      0.4477401375770569,
+      0.493536114692688,
+      0.41888487339019775,
+      0.40141189098358154,
+      0.36916500329971313,
+      0.2570352554321289,
+      0.28804102540016174,
+      0.24638836085796356,
+      0.10881100594997406,
+      0.0859808400273323,
+      0.07069482654333115,
+      0.056637559086084366,
+      0.04985696077346802,
+      0.03675064444541931,
+      0.05171795189380646,
+      0.05011414363980293,
+      0.02284391038119793,
+      0.041676875203847885,
+      0.020439328625798225,
+      0.014889881014823914,
+      0.01454250980168581,
+      0.012999804690480232,
+      0.012490401044487953,
+      0.013117315247654915,
+      0.0121223796159029,
+      0.02717740461230278,
+      0.01354890689253807,
+      0.009592078626155853,
+      0.015205960720777512,
+      0.009436314925551414,
+      0.014218422584235668,
+      0.011237069964408875,
+      0.010867897421121597,
+      0.008672598749399185,
+      0.014660419896245003,
+      0.004236647393554449,
+      0.009414155967533588,
+      0.0134099330753088,
+      0.019007394090294838,
+      0.006735099479556084,
+      0.006544078700244427,
+      0.008092062547802925,
+      0.010018263943493366,
+      0.005520849488675594,
+      0.006699803285300732,
+      0.009913476184010506,
+      0.004509104415774345,
+      0.008614420890808105,
+      0.0035750919487327337,
+      0.0076357703655958176,
+      0.011727109551429749,
+      0.011707671917974949,
+      0.009354616515338421,
+      0.009281913749873638,
+      0.006178762763738632,
+      0.0035909400321543217,
+      0.004905794281512499,
+      0.009883398190140724,
+      0.007503582630306482,
+      0.008511064574122429,
+      0.006195032503455877,
+      0.00810689851641655,
+      0.005093861371278763,
+      0.007193814031779766,
+      0.005878657102584839,
+      0.009027732536196709,
+      0.0035650311037898064,
+      0.00565190427005291,
+      0.00785311870276928,
+      0.0034982417710125446,
+      0.006185278296470642,
+      0.008414627984166145,
+      0.00469360314309597,
+      0.0059466976672410965,
+      0.010776687413454056,
+      0.006258154287934303,
+      0.006868111900985241,
+      0.0038875271566212177,
+      0.006210981868207455,
+      0.005305357277393341,
+      0.006381230428814888,
+      0.006814277730882168,
+      0.006563970353454351,
+      0.005743511952459812,
+      0.0065345969051122665,
+      0.011945639736950397,
+      0.004893606528639793,
+      0.007635968737304211,
+      0.00752865057438612,
+      0.010046824812889099,
+      0.007080474868416786,
+      0.011667938902974129,
+      0.00604804465547204,
+      0.007167487870901823,
+      0.008865633979439735,
+      0.005146023817360401,
+      0.007512568961828947,
+      0.0033924984745681286,
+      0.0027513052336871624,
+      0.004063080996274948,
+      0.0035831104032695293,
+      0.010143239051103592,
+      0.006252411752939224,
+      0.0041099111549556255,
+      0.002829005941748619,
+      0.008162900805473328,
+      0.0034778001718223095,
+      0.0048353467136621475,
+      0.00755035737529397,
+      0.006170344073325396,
+      0.0055410354398190975,
+      0.004055359400808811,
+      0.005794469267129898,
+      0.00485236058011651,
+      0.04100273922085762,
+      0.043546248227357864,
+      0.047018393874168396,
+      0.04010897874832153,
+      0.04094740003347397,
+      0.04226917028427124,
+      0.034722521901130676,
+      0.013317622244358063,
+      0.0050164926797151566,
+      0.005900361575186253,
+      0.006382612511515617,
+      0.005324410740286112,
+      0.006834868341684341,
+      0.007131056860089302,
+      0.0073970407247543335,
+      0.008419208228588104,
+      0.007619650103151798,
+      0.006124296225607395,
+      0.0029738247394561768,
+      0.005667706951498985,
+      0.0053189643658697605,
+      0.006181866396218538,
+      0.004378743004053831,
+      0.004856457933783531,
+      0.006193574983626604,
+      0.0061783939599990845,
+      0.006271403282880783,
+      0.006457127630710602,
+      0.009113837964832783,
+      0.005285582505166531,
+      0.006190958432853222,
+      0.005260601174086332,
+      0.003920216113328934,
+      0.003109836718067527,
+      0.0088285431265831,
+      0.004624238703399897,
+      0.005905993282794952,
+      0.00597411161288619,
+      0.0042738839983940125,
+      0.004486985504627228,
+      0.006995877251029015,
+      0.007218474522233009,
+      0.014883356168866158,
+      0.005925282835960388,
+      0.006964115425944328,
+      0.004993588663637638,
+      0.005705576855689287,
+      0.0061006853356957436,
+      0.0033382070250809193,
+      0.0032357140444219112,
+      0.006756148301064968,
+      0.005333520472049713,
+      0.005343958735466003,
+      0.004448914434760809,
+      0.00810321792960167,
+      0.008677376434206963,
+      0.006789557170122862,
+      0.006779493764042854,
+      0.010337872430682182,
+      0.004461814649403095,
+      0.006985191255807877,
+      0.009769750759005547,
+      0.008338341489434242,
+      0.008709460496902466,
+      0.00785098783671856,
+      0.006285048555582762,
+      0.01058521680533886,
+      0.0036689788103103638,
+      0.005481355357915163,
+      0.018989508971571922,
+      0.004932553041726351,
+      0.00451851449906826,
+      0.006092687137424946,
+      0.011893357150256634,
+      0.008019497618079185,
+      0.005471980199217796,
+      0.0077120475471019745,
+      0.008869717828929424,
+      0.009128015488386154,
+      0.010264063253998756,
+      0.005119603127241135,
+      0.009130019694566727,
+      0.0046901823952794075,
+      0.007227801717817783,
+      0.0059244707226753235,
+      0.0025816205888986588,
+      0.0044527817517519,
+      0.0037231813184916973,
+      0.009944789111614227,
+      0.009942839853465557,
+      0.0034302156418561935,
+      0.00983119010925293,
+      0.00665170606225729,
+      0.006609357427805662,
+      0.007506031543016434,
+      0.004614232107996941,
+      0.006634707562625408,
+      0.00920582003891468,
+      0.005753779783844948,
+      0.008357597514986992,
+      0.00524176936596632,
+      0.009124459698796272,
+      0.005513535812497139,
+      0.005799106787890196,
+      0.004532268270850182,
+      0.007867293432354927,
+      0.004558207467198372,
+      0.004656059667468071,
+      0.003020158503204584,
+      0.003758586011826992,
+      0.009156239219009876,
+      0.007190089672803879,
+      0.005840515252202749,
+      0.00948906596750021,
+      0.008433329872786999,
+      0.007277483120560646,
+      0.0021747304126620293,
+      0.005591952241957188,
+      0.004493037238717079,
+      0.007339922711253166,
+      0.007264859974384308,
+      0.006580689921975136,
+      0.0026356796734035015,
+      0.00429233442991972,
+      0.004033500328660011,
+      0.00375079782679677,
+      0.0022092361468821764,
+      0.0032393874134868383,
+      0.003990879748016596,
+      0.00903621967881918,
+      0.004429771099239588,
+      0.007698647677898407,
+      0.0038764220662415028,
+      0.0038335155695676804,
+      0.004251889884471893,
+      0.0043839626014232635,
+      0.0054384139366447926,
+      0.004180637653917074,
+      0.002683102386072278,
+      0.005919689312577248,
+      0.01232307031750679,
+      0.009214445948600769,
+      0.004193838220089674,
+      0.006566280499100685,
+      0.002540494315326214,
+      0.004513087682425976,
+      0.004715023562312126,
+      0.003238936886191368,
+      0.005976400338113308,
+      0.004979868419468403,
+      0.004852676764130592,
+      0.01037493348121643,
+      0.007781471125781536,
+      0.004505714867264032,
+      0.004395043943077326
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.9198145866394043,
+      0.6551633477210999,
+      0.5257541537284851,
+      0.42173081636428833,
+      0.3018621802330017,
+      0.195886492729187,
+      0.150644451379776,
+      0.1258474886417389,
+      0.12946873903274536,
+      0.09569515287876129,
+      0.10323487222194672,
+      0.12677723169326782,
+      0.16025961935520172,
+      0.19041621685028076,
+      0.2066192328929901,
+      0.2271208018064499,
+      0.24800638854503632,
+      0.2565635144710541,
+      0.2705942392349243,
+      0.290367990732193,
+      0.3045266270637512,
+      0.3073248267173767,
+      0.31250330805778503,
+      0.31037792563438416,
+      0.315677672624588,
+      0.3114228844642639,
+      0.3105468153953552,
+      0.3135741949081421,
+      0.30231669545173645,
+      0.29873502254486084,
+      0.29192081093788147,
+      0.2933517396450043,
+      0.2920157313346863,
+      0.2938830256462097,
+      0.2886824309825897,
+      0.28544124960899353,
+      0.28644096851348877,
+      0.2897089719772339,
+      0.29015612602233887,
+      0.291336327791214,
+      0.2915355861186981,
+      0.2854246497154236,
+      0.27548834681510925,
+      0.2664736211299896,
+      0.26724085211753845,
+      0.27037864923477173,
+      0.27064433693885803,
+      0.2666192650794983,
+      0.2614651620388031,
+      0.261039137840271,
+      0.26465243101119995,
+      0.26105886697769165,
+      0.257486492395401,
+      0.254778653383255,
+      0.25796666741371155,
+      0.25369012355804443,
+      0.24878060817718506,
+      0.25096824765205383,
+      0.25279730558395386,
+      0.24874061346054077,
+      0.24506402015686035,
+      0.24815328419208527,
+      0.24886399507522583,
+      0.24957183003425598,
+      0.2510982155799866,
+      0.24786141514778137,
+      0.24842151999473572,
+      0.24109771847724915,
+      0.24033305048942566,
+      0.24120938777923584,
+      0.24030791223049164,
+      0.23503310978412628,
+      0.2326727658510208,
+      0.2296709269285202,
+      0.23139061033725739,
+      0.2293580323457718,
+      0.2292603999376297,
+      0.22952620685100555,
+      0.23281599581241608,
+      0.23360049724578857,
+      0.23076213896274567,
+      0.229042187333107,
+      0.2326192706823349,
+      0.23553304374217987,
+      0.2308352291584015,
+      0.22562457621097565,
+      0.2211405485868454,
+      0.21634133160114288,
+      0.21848219633102417,
+      0.21689638495445251,
+      0.21321192383766174,
+      0.21015682816505432,
+      0.2143428772687912,
+      0.21006622910499573,
+      0.21031954884529114,
+      0.20823293924331665,
+      0.2088361233472824,
+      0.20616331696510315,
+      0.20600469410419464,
+      0.21204686164855957,
+      0.213396817445755,
+      0.20801101624965668,
+      0.2040943056344986,
+      0.20433804392814636,
+      0.20261657238006592,
+      0.20230965316295624,
+      0.2002040445804596,
+      0.19732622802257538,
+      0.1936998963356018,
+      0.19260728359222412,
+      0.19174639880657196,
+      0.18709009885787964,
+      0.19108328223228455,
+      0.19075725972652435,
+      0.19367077946662903,
+      0.18925613164901733,
+      0.18720345199108124,
+      0.18714788556098938,
+      0.1857219785451889,
+      0.18250976502895355,
+      0.18321950733661652,
+      0.18575584888458252,
+      0.18673887848854065,
+      0.18795277178287506,
+      0.18284600973129272,
+      0.1859154999256134,
+      0.19134750962257385,
+      0.19157521426677704,
+      0.19418273866176605,
+      0.19611477851867676,
+      0.19762173295021057,
+      0.19960592687129974,
+      0.19484658539295197,
+      0.1920938342809677,
+      0.19605068862438202,
+      0.1998741626739502,
+      0.19696621596813202,
+      0.19233518838882446,
+      0.1940525770187378,
+      0.19531334936618805,
+      0.1941678524017334,
+      0.1895352452993393,
+      0.188797265291214,
+      0.18817740678787231,
+      0.18502366542816162,
+      0.18578609824180603,
+      0.18805979192256927,
+      0.18685424327850342,
+      0.18059173226356506,
+      0.1785411387681961,
+      0.18304483592510223,
+      0.1815871000289917,
+      0.18020877242088318,
+      0.1785038709640503,
+      0.17913782596588135,
+      0.18085111677646637,
+      0.17924872040748596,
+      0.18093283474445343,
+      0.17957285046577454,
+      0.17980830371379852,
+      0.1819077879190445,
+      0.18055644631385803,
+      0.17821896076202393,
+      0.1766616553068161,
+      0.1761750727891922,
+      0.17780886590480804,
+      0.17631782591342926,
+      0.17762765288352966,
+      0.1772899031639099,
+      0.1732548475265503,
+      0.17357861995697021,
+      0.1753465235233307,
+      0.17705193161964417,
+      0.17899462580680847,
+      0.18039874732494354,
+      0.17960233986377716,
+      0.17767232656478882,
+      0.17840833961963654,
+      0.17904219031333923,
+      0.1762559711933136,
+      0.17740243673324585,
+      0.1820743978023529,
+      0.18073254823684692,
+      0.17860734462738037,
+      0.1795828491449356,
+      0.17847418785095215,
+      0.18097732961177826,
+      0.1796371191740036,
+      0.17679157853126526,
+      0.17841604351997375,
+      0.17764584720134735,
+      0.17880985140800476,
+      0.1773495376110077,
+      0.1767076849937439,
+      0.17657145857810974,
+      0.17661401629447937,
+      0.1773444563150406,
+      0.17760634422302246,
+      0.17683789134025574,
+      0.17719542980194092,
+      0.17727608978748322,
+      0.17656034231185913,
+      0.17764820158481598,
+      0.175861656665802,
+      0.1761469841003418,
+      0.17879027128219604,
+      0.17545416951179504,
+      0.17584295570850372,
+      0.17725321650505066,
+      0.17724880576133728,
+      0.1775873601436615,
+      0.1769874095916748,
+      0.1775965392589569,
+      0.1793995201587677,
+      0.17863869667053223,
+      0.17909114062786102,
+      0.1779758185148239,
+      0.17678499221801758,
+      0.17685092985630035,
+      0.17727036774158478,
+      0.17810432612895966,
+      0.1758294552564621,
+      0.1793285757303238,
+      0.1807268261909485,
+      0.1795637607574463,
+      0.17858168482780457,
+      0.17781469225883484,
+      0.17838957905769348,
+      0.17817437648773193,
+      0.17893055081367493,
+      0.17974941432476044,
+      0.18173956871032715,
+      0.17994102835655212,
+      0.1794210821390152,
+      0.18177224695682526,
+      0.17975547909736633,
+      0.17758183181285858,
+      0.17822329699993134,
+      0.17720657587051392,
+      0.1767847090959549,
+      0.18060030043125153,
+      0.1803709715604782,
+      0.1815047562122345,
+      0.18107862770557404,
+      0.18273945152759552,
+      0.18144862353801727,
+      0.1819262057542801,
+      0.18287310004234314,
+      0.18173839151859283,
+      0.18238605558872223,
+      0.1825030893087387,
+      0.1808522492647171,
+      0.18008854985237122,
+      0.17948704957962036,
+      0.17876680195331573,
+      0.17993193864822388,
+      0.17882338166236877,
+      0.178974911570549,
+      0.17935578525066376,
+      0.17965644598007202,
+      0.18048611283302307,
+      0.17919977009296417,
+      0.17858874797821045,
+      0.1775817573070526,
+      0.17882919311523438,
+      0.17989391088485718,
+      0.1786574274301529,
+      0.18099862337112427,
+      0.18243511021137238,
+      0.18324783444404602,
+      0.18357829749584198,
+      0.18426436185836792,
+      0.18502143025398254,
+      0.18604235351085663,
+      0.1857336014509201,
+      0.18722490966320038,
+      0.1879812479019165,
+      0.1885811686515808,
+      0.19002453982830048,
+      0.19144999980926514,
+      0.19173073768615723,
+      0.19194410741329193,
+      0.19225633144378662,
+      0.19304108619689941,
+      0.19343622028827667,
+      0.1938202679157257,
+      0.19398020207881927,
+      0.19369561970233917,
+      0.19482459127902985,
+      0.19427832961082458,
+      0.19423837959766388,
+      0.1951482743024826,
+      0.19510076940059662,
+      0.1958136409521103,
+      0.19633986055850983,
+      0.19718037545681,
+      0.1969831883907318,
+      0.19735197722911835,
+      0.19798672199249268,
+      0.1981828659772873,
+      0.19840724766254425,
+      0.198574498295784,
+      0.19880661368370056,
+      0.1986214518547058,
+      0.1990007758140564,
+      0.1990588903427124,
+      0.1990559697151184,
+      0.19906410574913025,
+      0.19931460916996002,
+      0.19968949258327484
+    ],
+    "lr": [
+      4.188034188034189e-06,
+      8.461538461538462e-06,
+      1.2735042735042738e-05,
+      1.700854700854701e-05,
+      2.1282051282051282e-05,
+      2.5555555555555557e-05,
+      2.9829059829059833e-05,
+      3.4102564102564105e-05,
+      3.837606837606838e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.992995647012267e-05,
+      3.963316185199842e-05,
+      3.933636723387416e-05,
+      3.90395726157499e-05,
+      3.874277799762565e-05,
+      3.844598337950139e-05,
+      3.814918876137713e-05,
+      3.785239414325287e-05,
+      3.7555599525128613e-05,
+      3.725880490700435e-05,
+      3.696201028888009e-05,
+      3.666521567075585e-05,
+      3.6368421052631585e-05,
+      3.607162643450732e-05,
+      3.577483181638307e-05,
+      3.547803719825881e-05,
+      3.518124258013455e-05,
+      3.4884447962010287e-05,
+      3.458765334388604e-05,
+      3.4290858725761776e-05,
+      3.399406410763751e-05,
+      3.3697269489513265e-05,
+      3.3400474871389e-05,
+      3.310368025326474e-05,
+      3.280688563514048e-05,
+      3.251009101701623e-05,
+      3.2213296398891967e-05,
+      3.191650178076771e-05,
+      3.1542540561931145e-05,
+      3.124574594380688e-05,
+      3.094895132568263e-05,
+      3.065215670755838e-05,
+      3.0355362089434113e-05,
+      3.0058567471309854e-05,
+      2.97617728531856e-05,
+      2.9464978235061336e-05,
+      2.9168183616937077e-05,
+      2.8871388998812818e-05,
+      2.857459438068857e-05,
+      2.8277799762564307e-05,
+      2.798100514444005e-05,
+      2.7684210526315793e-05,
+      2.738741590819153e-05,
+      2.709062129006727e-05,
+      2.6793826671943013e-05,
+      2.6497032053818764e-05,
+      2.62002374356945e-05,
+      2.5903442817570243e-05,
+      2.5606648199445987e-05,
+      2.5309853581321725e-05,
+      2.5013058963197466e-05,
+      2.4716264345073214e-05,
+      2.441946972694896e-05,
+      2.4122675108824693e-05,
+      2.3825880490700434e-05,
+      2.352908587257618e-05,
+      2.3232291254451916e-05,
+      2.293549663632766e-05,
+      2.2638702018203408e-05,
+      2.226474079936684e-05,
+      2.196794618124258e-05,
+      2.1671151563118328e-05,
+      2.1374356944994066e-05,
+      2.1077562326869803e-05,
+      2.0780767708745554e-05,
+      2.0483973090621296e-05,
+      2.018717847249703e-05,
+      1.989038385437277e-05,
+      1.959358923624852e-05,
+      1.929679461812426e-05,
+      1.8999999999999998e-05,
+      1.8703205381875746e-05,
+      1.8406410763751487e-05,
+      1.8109616145627224e-05,
+      1.7812821527502965e-05,
+      1.7516026909378713e-05,
+      1.7219232291254454e-05,
+      1.6922437673130192e-05,
+      1.662564305500594e-05,
+      1.632884843688168e-05,
+      1.603205381875742e-05,
+      1.573525920063316e-05,
+      1.5438464582508908e-05,
+      1.5141669964384647e-05,
+      1.4844875346260385e-05,
+      1.4548080728136134e-05,
+      1.4251286110011875e-05,
+      1.3954491491887611e-05,
+      1.3657696873763363e-05,
+      1.33609022556391e-05,
+      1.298694103680253e-05,
+      1.2690146418678277e-05,
+      1.2393351800554018e-05,
+      1.2096557182429756e-05,
+      1.1799762564305506e-05,
+      1.1502967946181245e-05,
+      1.1206173328056984e-05,
+      1.0909378709932724e-05,
+      1.0612584091808472e-05,
+      1.0315789473684213e-05,
+      1.0018994855559949e-05,
+      9.7222002374357e-06,
+      9.425405619311438e-06,
+      9.128611001187177e-06,
+      8.831816383062916e-06,
+      8.535021764938666e-06,
+      8.238227146814405e-06,
+      7.941432528690143e-06,
+      7.644637910565893e-06,
+      7.347843292441632e-06,
+      7.0510486743173715e-06,
+      6.75425405619311e-06,
+      6.45745943806886e-06,
+      6.160664819944598e-06,
+      5.8638702018203375e-06,
+      5.567075583696086e-06,
+      5.270280965571826e-06,
+      4.973486347447565e-06,
+      4.676691729323314e-06,
+      4.379897111199053e-06,
+      4.083102493074792e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      1550,
+      3113,
+      4676,
+      6239,
+      7802,
+      9365,
+      10928,
+      12491,
+      14054,
+      15617
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 0.9945833333333334,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.989247311827957,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.79,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.86875,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.9869706840390879,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.64,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.71,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.9025,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.88,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.94,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.96,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9948186528497409,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.94,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.98046875,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.9930555555555556,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.92,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9738562091503268,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.9948453608247423,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.996031746031746,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.57,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.58,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.35,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.61,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.86,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9949238578680203,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.94,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9757085020242915,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.98,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.9966442953020134,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.995049504950495,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.9145833333333333,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.998,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.99609375,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.93,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.9925,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 1.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 1.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.9945833333333334,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.9945833333333334,
+  "sft_overall_accuracy": 0.9145833333333333
+}

add_sub_sorl_v1_abs16_K1_100K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2653ee350cd31d8cb32a9ea8e72573d552ba4ec8838802acee1f18aaf909b8af
+size 650328152

add_sub_sorl_v1_abs16_K1_100K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 468,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 1562,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs16_K1_100K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 16,
+  "dataset_size": 100000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162505382,
+  "run_name": "add_sub_sorl_v1_abs16_K1_100K",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T17:42:20.080941+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "79zwxeez",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/79zwxeez",
+  "final_accuracy": 0.9945833333333334,
+  "sft_accuracy": 0.9145833333333333,
+  "eval_method": "ArithmeticEvaluator"
+}