Upload add_sub_sorl_v1_abs10_K1_25K_1L2H256d

Browse files

Files changed (3) hide show

add_sub_sorl_v1_abs10_K1_25K_1L2H256d/metrics.json +1554 -974
add_sub_sorl_v1_abs10_K1_25K_1L2H256d/model.safetensors +1 -1
add_sub_sorl_v1_abs10_K1_25K_1L2H256d/train_config.json +12 -8

add_sub_sorl_v1_abs10_K1_25K_1L2H256d/metrics.json CHANGED Viewed

@@ -70,444 +70,936 @@
       3719,
       3769,
       3819,
-      3869
     ],
     "loss": [
-      8.524430274963379,
-      9.103901863098145,
-      11.476999282836914,
-      11.07170295715332,
-      9.617006301879883,
-      8.757355690002441,
-      7.827155113220215,
-      6.2943830490112305,
-      5.568047523498535,
-      4.920413970947266,
-      4.179653167724609,
-      4.015377998352051,
-      3.8744664192199707,
-      3.6563000679016113,
-      3.714816093444824,
-      3.138887882232666,
-      3.476172924041748,
-      3.5434532165527344,
-      3.2397027015686035,
-      3.118135929107666,
-      3.188394546508789,
-      3.03411602973938,
-      3.202545166015625,
-      3.2864937782287598,
-      2.669011116027832,
-      3.0694193840026855,
-      2.755049705505371,
-      2.658294677734375,
-      3.083820343017578,
-      2.523517608642578,
-      2.064281463623047,
-      2.1752371788024902,
-      2.3656363487243652,
-      1.9907090663909912,
-      1.5538188219070435,
-      0.5142228603363037,
-      0.4625096321105957,
-      -0.34851527214050293,
-      -0.9209808111190796,
-      -0.848351240158081,
-      -1.2908415794372559,
-      -1.9059382677078247,
-      -2.5187909603118896,
-      -2.6178600788116455,
-      -2.9667201042175293,
-      -3.4197542667388916,
-      -3.4599928855895996,
-      -4.1904425621032715,
-      -5.044626235961914,
-      -4.426793098449707,
-      -4.773674011230469,
-      -5.006862640380859,
-      -5.084996223449707,
-      -5.224812030792236,
-      -5.75303316116333,
-      -5.572091102600098,
-      -5.412012100219727,
-      -5.735694408416748,
-      -4.803545951843262,
-      -4.7789177894592285,
-      -4.737234115600586,
-      -4.012823104858398,
-      -3.9731242656707764,
-      -4.444588661193848,
-      -4.273018836975098,
-      -4.604288101196289,
-      -4.26378059387207,
-      -4.285333156585693,
-      -3.932020902633667,
-      -4.814791202545166
     ],
     "base_loss": [
-      11.692604064941406,
-      10.586286544799805,
-      9.575356483459473,
-      8.626004219055176,
-      7.766415119171143,
-      6.867415904998779,
-      5.944822788238525,
-      4.488890647888184,
-      3.712851047515869,
-      3.1012394428253174,
-      2.6632230281829834,
-      2.547546863555908,
-      2.3669917583465576,
-      2.289210081100464,
-      2.217154026031494,
-      2.0566632747650146,
-      2.032634735107422,
-      1.9611835479736328,
-      2.062990427017212,
-      1.9896939992904663,
-      1.9653056859970093,
-      1.9700582027435303,
-      1.9406718015670776,
-      1.9608590602874756,
-      1.9188401699066162,
-      1.9645260572433472,
-      1.8941099643707275,
-      1.8756904602050781,
-      1.9264477491378784,
-      1.9439666271209717,
-      1.8485826253890991,
-      1.9204241037368774,
-      1.8739402294158936,
-      1.8911492824554443,
-      1.90359628200531,
-      1.8942089080810547,
-      1.9322471618652344,
-      1.8236247301101685,
-      1.8280603885650635,
-      1.8796042203903198,
-      1.8643499612808228,
-      1.9163367748260498,
-      1.8653056621551514,
-      1.8866708278656006,
-      1.8857192993164062,
-      1.8382318019866943,
-      1.8878859281539917,
-      1.8952440023422241,
-      1.9479080438613892,
-      1.8315229415893555,
-      1.853525161743164,
-      1.7897322177886963,
-      1.7672021389007568,
-      1.67568838596344,
-      1.732727289199829,
-      1.6849744319915771,
-      1.6561976671218872,
-      1.6894086599349976,
-      1.5698803663253784,
-      1.526742935180664,
-      1.5828590393066406,
-      1.4620859622955322,
-      1.427542805671692,
-      1.5105113983154297,
-      1.487876296043396,
-      1.4823344945907593,
-      1.458412766456604,
-      1.479615330696106,
-      1.3868151903152466,
-      1.540739893913269
     ],
     "info_loss": [
-      -1.2530698776245117,
-      -0.9802732467651367,
-      -0.4201974868774414,
-      -0.18035411834716797,
-      -0.13341093063354492,
-      -0.07659339904785156,
-      -0.04645824432373047,
-      -0.028890609741210938,
-      -0.016974687576293945,
-      -0.016387462615966797,
-      -0.04442238807678223,
-      -0.047545671463012695,
-      -0.042586565017700195,
-      -0.056116580963134766,
-      -0.04165053367614746,
-      -0.08310163021087646,
-      -0.04619729518890381,
-      -0.03194534778594971,
-      -0.07237017154693604,
-      -0.076835036277771,
-      -0.06701540946960449,
-      -0.08263790607452393,
-      -0.06275796890258789,
-      -0.05613744258880615,
-      -0.11362743377685547,
-      -0.07795798778533936,
-      -0.10230779647827148,
-      -0.11010193824768066,
-      -0.07229125499725342,
-      -0.13019275665283203,
-      -0.16647255420684814,
-      -0.16245436668395996,
-      -0.138879656791687,
-      -0.17805254459381104,
-      -0.22295153141021729,
-      -0.32607901096343994,
-      -0.3350032567977905,
-      -0.40525615215301514,
-      -0.4629077911376953,
-      -0.46074652671813965,
-      -0.5033977031707764,
-      -0.5704048871994019,
-      -0.6261879205703735,
-      -0.6387062072753906,
-      -0.6733493804931641,
-      -0.7140244245529175,
-      -0.7230769395828247,
-      -0.796980619430542,
-      -0.887682318687439,
-      -0.8138977289199829,
-      -0.8508479595184326,
-      -0.8680267930030823,
-      -0.8732814192771912,
-      -0.8783826231956482,
-      -0.9365927577018738,
-      -0.9135531187057495,
-      -0.8946322798728943,
-      -0.9301126599311829,
-      -0.8245593905448914,
-      -0.8180517554283142,
-      -0.8192586898803711,
-      -0.734249472618103,
-      -0.7269594073295593,
-      -0.7820543646812439,
-      -0.7622780799865723,
-      -0.7947943210601807,
-      -0.7579862475395203,
-      -0.762677013874054,
-      -0.7179456353187561,
-      -0.821166455745697
     ],
     "abs_loss": [
-      2.301368474960327,
-      2.272613763809204,
-      2.1877543926239014,
-      2.0855491161346436,
-      2.003074884414673,
-      1.9530082941055298,
-      1.8817201852798462,
-      1.862039566040039,
-      1.8272932767868042,
-      1.8210479021072388,
-      1.8502413034439087,
-      1.8328882455825806,
-      1.8329845666885376,
-      1.8578821420669556,
-      1.8351255655288696,
-      1.8775984048843384,
-      1.8421878814697266,
-      1.8364969491958618,
-      1.8568716049194336,
-      1.8509222269058228,
-      1.840528130531311,
-      1.8526434898376465,
-      1.8566508293151855,
-      1.8434433937072754,
-      1.8501802682876587,
-      1.8451181650161743,
-      1.8511062860488892,
-      1.8564246892929077,
-      1.8391847610473633,
-      1.8499897718429565,
-      1.8488394021987915,
-      1.8438078165054321,
-      1.8546184301376343,
-      1.857653021812439,
-      1.8427098989486694,
-      1.837051272392273,
-      1.8287359476089478,
-      1.8250846862792969,
-      1.8261111974716187,
-      1.8157788515090942,
-      1.813087821006775,
-      1.8383451700210571,
-      1.7974052429199219,
-      1.8346837759017944,
-      1.8273576498031616,
-      1.8329792022705078,
-      1.8257931470870972,
-      1.8227308988571167,
-      1.8331661224365234,
-      1.8032366037368774,
-      1.8001683950424194,
-      1.8143806457519531,
-      1.8048582077026367,
-      1.8383716344833374,
-      1.8124042749404907,
-      1.7793735265731812,
-      1.7747602462768555,
-      1.7881097793579102,
-      1.7533124685287476,
-      1.776807188987732,
-      1.735535979270935,
-      1.737404465675354,
-      1.7615875005722046,
-      1.7293590307235718,
-      1.7104082107543945,
-      1.7312254905700684,
-      1.700640082359314,
-      1.708137035369873,
-      1.7061353921890259,
-      1.6751855611801147
     ],
     "zipf_loss": [
-      9.1323881149292,
-      8.093086242675781,
-      5.884842872619629,
-      4.040685653686523,
-      2.9843926429748535,
-      2.4605727195739746,
-      2.158742666244507,
-      1.9081945419311523,
-      1.8422138690948486,
-      1.8009443283081055,
-      1.775629997253418,
-      1.75999915599823,
-      1.7500418424606323,
-      1.7424676418304443,
-      1.7306548357009888,
-      1.7254811525344849,
-      1.7212923765182495,
-      1.7180733680725098,
-      1.7147266864776611,
-      1.711700201034546,
-      1.709190011024475,
-      1.7051725387573242,
-      1.703787922859192,
-      1.702664852142334,
-      1.7014273405075073,
-      1.699961543083191,
-      1.6989071369171143,
-      1.6979811191558838,
-      1.696366548538208,
-      1.6964795589447021,
-      1.695540428161621,
-      1.6949759721755981,
-      1.6950308084487915,
-      1.6943199634552002,
-      1.6954668760299683,
-      1.697098970413208,
-      1.6974214315414429,
-      1.697913408279419,
-      1.6974254846572876,
-      1.6979317665100098,
-      1.697476863861084,
-      1.6979395151138306,
-      1.6980416774749756,
-      1.6990625858306885,
-      1.6983184814453125,
-      1.6989600658416748,
-      1.7003107070922852,
-      1.701846718788147,
-      1.7009726762771606,
-      1.700337290763855,
-      1.7012641429901123,
-      1.7022359371185303,
-      1.7001292705535889,
-      1.6994887590408325,
-      1.6989268064498901,
-      1.7005280256271362,
-      1.7006367444992065,
-      1.6972121000289917,
-      1.6968367099761963,
-      1.6971760988235474,
-      1.6989400386810303,
-      1.6938459873199463,
-      1.6927683353424072,
-      1.6925078630447388,
-      1.6908442974090576,
-      1.688198208808899,
-      1.687605381011963,
-      1.6910079717636108,
-      1.6900064945220947,
-      1.6886142492294312
     ],
     "denoise_loss": [],
     "ortho_loss": [
-      0.7410624623298645,
-      0.5486676096916199,
-      0.4325517416000366,
-      0.4293130338191986,
-      0.42301344871520996,
-      0.43677014112472534,
-      0.4322786331176758,
-      0.43111148476600647,
-      0.42917412519454956,
-      0.4202711284160614,
-      0.40449219942092896,
-      0.38009271025657654,
-      0.38311055302619934,
-      0.38355082273483276,
-      0.3772588074207306,
-      0.37517693638801575,
-      0.35672980546951294,
-      0.3655826151371002,
-      0.36484745144844055,
-      0.36458733677864075,
-      0.3716451823711395,
-      0.3796772062778473,
-      0.37268635630607605,
-      0.36658456921577454,
-      0.36402279138565063,
-      0.36522912979125977,
-      0.3712886571884155,
-      0.3701127767562866,
-      0.3669368028640747,
-      0.35840272903442383,
-      0.36034736037254333,
-      0.36091148853302,
-      0.35899659991264343,
-      0.356486439704895,
-      0.3690042495727539,
-      0.3797115981578827,
-      0.3873102366924286,
-      0.39992478489875793,
-      0.41160136461257935,
-      0.42042696475982666,
-      0.42867836356163025,
-      0.43672993779182434,
-      0.4450317621231079,
-      0.45142629742622375,
-      0.451881468296051,
-      0.45120909810066223,
-      0.4502365291118622,
-      0.44785618782043457,
-      0.4534400999546051,
-      0.45658400654792786,
-      0.45758968591690063,
-      0.45804986357688904,
-      0.4550400674343109,
-      0.451234370470047,
-      0.4503278136253357,
-      0.45379236340522766,
-      0.4566922187805176,
-      0.4583297073841095,
-      0.46125417947769165,
-      0.46176251769065857,
-      0.4625169634819031,
-      0.46337494254112244,
-      0.46481263637542725,
-      0.4634028375148773,
-      0.4632355272769928,
-      0.46351659297943115,
-      0.46328744292259216,
-      0.4644387364387512,
-      0.46469375491142273,
-      0.4648001492023468
     ],
     "lr": [
-      8.376068376068378e-06,
-      1.6923076923076924e-05,
       2e-05,
       2e-05,
       2e-05,
@@ -548,34 +1040,102 @@
       2e-05,
       2e-05,
       2e-05,
-      1.9973899288162407e-05,
-      1.9380701291853413e-05,
-      1.8787503295544426e-05,
-      1.8194305299235432e-05,
-      1.7601107302926442e-05,
-      1.7007909306617455e-05,
-      1.6414711310308464e-05,
-      1.5335090957026102e-05,
-      1.474189296071711e-05,
-      1.4148694964408121e-05,
-      1.355549696809913e-05,
-      1.2962298971790142e-05,
-      1.2369100975481152e-05,
-      1.1775902979172158e-05,
-      1.0696282625889797e-05,
-      1.0103084629580805e-05,
-      9.50988663327182e-06,
-      8.91668863696283e-06,
-      8.323490640653837e-06,
-      7.730292644344845e-06,
-      7.137094648035855e-06,
-      6.057474294753492e-06,
-      5.4642762984445065e-06,
-      4.871078302135514e-06,
-      4.277880305826523e-06,
-      3.684682309517532e-06,
-      3.091484313208541e-06,
-      2.4982863168995496e-06
     ],
     "emb_lr": [],
     "eval_step": [
@@ -588,588 +1148,608 @@
       2696,
       3087,
       3478,
-      3869
     ],
     "eval_accuracy": [
       0.01,
       0.0,
       0.0,
-      0.01,
       0.0,
       0.0,
-      0.01,
       0.0,
       0.0,
       0.0
     ]
   },
-  "final_accuracy": 0.06291666666666666,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5771428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5457627118644067,
-            "count": 295
           },
           "SS": {
-            "accuracy": 0.7454545454545455,
-            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4142857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5079365079365079,
-            "count": 126
           },
           "SC": {
-            "accuracy": 0.2911392405063291,
-            "count": 79
           },
           "SS": {
-            "accuracy": 0.6666666666666666,
-            "count": 21
           },
           "UC": {
-            "accuracy": 0.3548387096774194,
-            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5028571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6133333333333333,
-            "count": 75
           },
           "SC": {
-            "accuracy": 0.3709677419354839,
-            "count": 62
           },
           "SS": {
-            "accuracy": 0.6923076923076923,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.45045045045045046,
-            "count": 111
           },
           "US": {
-            "accuracy": 0.47619047619047616,
-            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.44571428571428573,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7166666666666667,
-            "count": 60
           },
           "SC": {
-            "accuracy": 0.3157894736842105,
-            "count": 57
           },
           "SS": {
-            "accuracy": 0.47368421052631576,
-            "count": 19
           },
           "UC": {
-            "accuracy": 0.4519230769230769,
-            "count": 104
           },
           "US": {
-            "accuracy": 0.35454545454545455,
-            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.3914285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.625,
-            "count": 48
           },
           "SC": {
-            "accuracy": 0.38461538461538464,
-            "count": 52
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 7
           },
           "UC": {
-            "accuracy": 0.3595505617977528,
-            "count": 89
           },
           "US": {
-            "accuracy": 0.3116883116883117,
-            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.3342857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.72,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.28,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.38,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.24,
-            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.18,
-        "digit_accuracy": 0.3628571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.48,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.44,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.324,
-            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.44142857142857145,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.531322505800464,
-            "count": 431
           },
           "SC": {
-            "accuracy": 0.439873417721519,
-            "count": 316
           },
           "SS": {
-            "accuracy": 0.7948717948717948,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.3482142857142857,
-            "count": 560
           },
           "US": {
-            "accuracy": 0.4444444444444444,
-            "count": 54
           }
         }
       },
       "add_C1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5514285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.66,
-            "count": 250
           },
           "SC": {
-            "accuracy": 0.24,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.32,
-            "count": 50
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.4714285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.545,
-            "count": 200
           },
           "SC": {
-            "accuracy": 0.46,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.3132530120481928,
-            "count": 83
           },
           "US": {
-            "accuracy": 0.4117647058823529,
-            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.42,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5933333333333334,
-            "count": 150
           },
           "SC": {
-            "accuracy": 0.36,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.28,
-            "count": 100
           },
           "US": {
-            "accuracy": 0.24,
-            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.46285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.71,
-            "count": 100
           },
           "SC": {
-            "accuracy": 0.36,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.32575757575757575,
-            "count": 132
           },
           "US": {
-            "accuracy": 0.4411764705882353,
-            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4142857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.72,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.38,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.3287671232876712,
-            "count": 146
           },
           "US": {
-            "accuracy": 0.40384615384615385,
-            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.40285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.4,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.3862433862433862,
-            "count": 189
           },
           "US": {
-            "accuracy": 0.43243243243243246,
-            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.45714285714285713,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.39933993399339934,
-            "count": 303
           },
           "ME": {
-            "accuracy": 0.8297872340425532,
-            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.34285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5177304964539007,
-            "count": 141
           },
           "MB": {
-            "accuracy": 0.09722222222222222,
-            "count": 72
           },
           "ME": {
-            "accuracy": 0.8333333333333334,
-            "count": 18
           },
           "UB": {
-            "accuracy": 0.21008403361344538,
-            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4228571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7053571428571429,
-            "count": 112
           },
           "MB": {
-            "accuracy": 0.11320754716981132,
-            "count": 53
           },
           "ME": {
-            "accuracy": 0.8297872340425532,
-            "count": 47
           },
           "UB": {
-            "accuracy": 0.2235294117647059,
-            "count": 85
           },
           "UD": {
-            "accuracy": 0.09433962264150944,
-            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.37142857142857144,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7216494845360825,
-            "count": 97
           },
           "MB": {
-            "accuracy": 0.058823529411764705,
-            "count": 51
           },
           "ME": {
-            "accuracy": 0.8888888888888888,
-            "count": 27
           },
           "UB": {
-            "accuracy": 0.25675675675675674,
-            "count": 74
           },
           "UD": {
-            "accuracy": 0.13861386138613863,
-            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.2857142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.57,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.1,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.6,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.05333333333333334,
-            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.2542857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.02,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.44,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.08,
-            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.32785714285714285,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5596491228070175,
-            "count": 570
           },
           "MB": {
-            "accuracy": 0.06859205776173286,
-            "count": 277
           },
           "ME": {
-            "accuracy": 0.7547169811320755,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.16348195329087048,
-            "count": 471
           },
           "UD": {
-            "accuracy": 0.13793103448275862,
-            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.28,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.48,
-            "count": 150
           },
           "MB": {
-            "accuracy": 0.04,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.18811881188118812,
-            "count": 101
           },
           "UD": {
-            "accuracy": 0.10204081632653061,
-            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.25142857142857145,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.61,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.02,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.17355371900826447,
-            "count": 121
           },
           "UD": {
-            "accuracy": 0.06329113924050633,
-            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.2742857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.04,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.21710526315789475,
-            "count": 152
           },
           "UD": {
-            "accuracy": 0.11224489795918367,
-            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.012,
-      "digit_accuracy": 0.3922857142857143,
-      "total_examples": 1500,
       "n_splits": 24
     }
   },
@@ -1179,569 +1759,569 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.62,
-        "digit_accuracy": 0.9228571428571428,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9355932203389831,
-            "count": 295
           },
           "SS": {
-            "accuracy": 0.8545454545454545,
-            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.66,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9365079365079365,
-            "count": 126
           },
           "SC": {
-            "accuracy": 0.8607594936708861,
-            "count": 79
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 21
           },
           "UC": {
-            "accuracy": 0.21774193548387097,
-            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5457142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9333333333333333,
-            "count": 75
           },
           "SC": {
-            "accuracy": 0.7580645161290323,
-            "count": 62
           },
           "SS": {
-            "accuracy": 0.8461538461538461,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.24324324324324326,
-            "count": 111
           },
           "US": {
-            "accuracy": 0.2222222222222222,
-            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5142857142857142,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9666666666666667,
-            "count": 60
           },
           "SC": {
-            "accuracy": 0.8421052631578947,
-            "count": 57
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 19
           },
           "UC": {
-            "accuracy": 0.3557692307692308,
-            "count": 104
           },
           "US": {
-            "accuracy": 0.16363636363636364,
-            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9583333333333334,
-            "count": 48
           },
           "SC": {
-            "accuracy": 0.9038461538461539,
-            "count": 52
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 7
           },
           "UC": {
-            "accuracy": 0.2808988764044944,
-            "count": 89
           },
           "US": {
-            "accuracy": 0.1038961038961039,
-            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.2914285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.92,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.66,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.1,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.09,
-            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.11142857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.4,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.1,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.056,
-            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.045,
-        "digit_accuracy": 0.6085714285714285,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9327146171693735,
-            "count": 431
           },
           "SC": {
-            "accuracy": 0.8670886075949367,
-            "count": 316
           },
           "SS": {
-            "accuracy": 0.8461538461538461,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.2357142857142857,
-            "count": 560
           },
           "US": {
-            "accuracy": 0.2037037037037037,
-            "count": 54
           }
         }
       },
       "add_C1": {
-        "full_accuracy": 0.12,
-        "digit_accuracy": 0.8314285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.956,
-            "count": 250
           },
           "SC": {
-            "accuracy": 0.88,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.16,
-            "count": 50
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.7114285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.945,
-            "count": 200
           },
           "SC": {
-            "accuracy": 0.9,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.1686746987951807,
-            "count": 83
           },
           "US": {
-            "accuracy": 0.058823529411764705,
-            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5971428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.94,
-            "count": 150
           },
           "SC": {
-            "accuracy": 0.74,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.21,
-            "count": 100
           },
           "US": {
-            "accuracy": 0.2,
-            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4742857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.95,
-            "count": 100
           },
           "SC": {
-            "accuracy": 0.8,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.20454545454545456,
-            "count": 132
           },
           "US": {
-            "accuracy": 0.058823529411764705,
-            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.96,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.8,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.2328767123287671,
-            "count": 146
           },
           "US": {
-            "accuracy": 0.17307692307692307,
-            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.28285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.68,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.31216931216931215,
-            "count": 189
           },
           "US": {
-            "accuracy": 0.05405405405405406,
-            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.6,
-        "digit_accuracy": 0.9285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9174917491749175,
-            "count": 303
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.64,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9361702127659575,
-            "count": 141
           },
           "MB": {
-            "accuracy": 0.8194444444444444,
-            "count": 72
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 18
           },
           "UB": {
-            "accuracy": 0.12605042016806722,
-            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5885714285714285,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9553571428571429,
-            "count": 112
           },
           "MB": {
-            "accuracy": 0.660377358490566,
-            "count": 53
           },
           "ME": {
-            "accuracy": 0.9787234042553191,
-            "count": 47
           },
           "UB": {
-            "accuracy": 0.16470588235294117,
-            "count": 85
           },
           "UD": {
-            "accuracy": 0.07547169811320754,
-            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.47714285714285715,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9381443298969072,
-            "count": 97
           },
           "MB": {
-            "accuracy": 0.6078431372549019,
-            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 27
           },
           "UB": {
-            "accuracy": 0.16216216216216217,
-            "count": 74
           },
           "UD": {
-            "accuracy": 0.0594059405940594,
-            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.44,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.95,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.8,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.2,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.06,
-            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.28285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.52,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.3,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.04,
-            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.04,
-        "digit_accuracy": 0.6114285714285714,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9245614035087719,
-            "count": 570
           },
           "MB": {
-            "accuracy": 0.7906137184115524,
-            "count": 277
           },
           "ME": {
-            "accuracy": 0.9433962264150944,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.12526539278131635,
-            "count": 471
           },
           "UD": {
-            "accuracy": 0.034482758620689655,
-            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5685714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9333333333333333,
-            "count": 150
           },
           "MB": {
-            "accuracy": 0.86,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.1188118811881188,
-            "count": 101
           },
           "UD": {
-            "accuracy": 0.08163265306122448,
-            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.4514285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.94,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.84,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.1652892561983471,
-            "count": 121
           },
           "UD": {
-            "accuracy": 0.02531645569620253,
-            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.32857142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.68,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.14473684210526316,
-            "count": 152
           },
           "UD": {
-            "accuracy": 0.09183673469387756,
-            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.058,
-      "digit_accuracy": 0.5440952380952381,
-      "total_examples": 1500,
       "n_splits": 24
     }
   },
-  "sorl_overall_accuracy": 0.06291666666666666,
-  "sft_overall_accuracy": 0.011666666666666667
 }

       3719,
       3769,
       3819,
+      3869,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4351,
+      4401,
+      4451,
+      4501,
+      4551,
+      4601,
+      4651,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5133,
+      5183,
+      5233,
+      5283,
+      5333,
+      5383,
+      5433,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5915,
+      5965,
+      6015,
+      6065,
+      6115,
+      6165,
+      6215,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6697,
+      6747,
+      6797,
+      6847,
+      6897,
+      6947,
+      6997,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7479,
+      7529,
+      7579,
+      7629,
+      7679,
+      7729,
+      7779
     ],
     "loss": [
+      13.336702346801758,
+      6.918850421905518,
+      8.896209716796875,
+      11.282072067260742,
+      10.738224029541016,
+      9.437276840209961,
+      8.512653350830078,
+      6.82270622253418,
+      6.035636901855469,
+      5.339687824249268,
+      4.746975421905518,
+      4.351130962371826,
+      4.000054359436035,
+      3.8665781021118164,
+      3.4704408645629883,
+      3.5424342155456543,
+      3.418386459350586,
+      3.4378161430358887,
+      3.1492156982421875,
+      3.2364323139190674,
+      3.2923150062561035,
+      3.3853559494018555,
+      3.247429370880127,
+      3.1995742321014404,
+      3.0241119861602783,
+      2.8602428436279297,
+      2.9540209770202637,
+      2.8344078063964844,
+      3.0442986488342285,
+      2.776155948638916,
+      3.210538864135742,
+      2.941742420196533,
+      2.6947340965270996,
+      2.501694917678833,
+      2.366105079650879,
+      2.6169660091400146,
+      2.829813241958618,
+      2.588465690612793,
+      2.4476141929626465,
+      2.5533101558685303,
+      2.3008124828338623,
+      2.4801294803619385,
+      2.1546616554260254,
+      2.3855626583099365,
+      2.0368154048919678,
+      2.2342143058776855,
+      1.8597867488861084,
+      1.8611640930175781,
+      1.6935548782348633,
+      0.8804562091827393,
+      0.11107325553894043,
+      -0.9678399562835693,
+      -1.721615195274353,
+      -1.8610954284667969,
+      -2.863313913345337,
+      -3.38641357421875,
+      -4.709446907043457,
+      -5.262845993041992,
+      -6.146366119384766,
+      -5.718625545501709,
+      -6.643040657043457,
+      -6.837361812591553,
+      -6.452937126159668,
+      -7.447319030761719,
+      -7.173651218414307,
+      -6.313607215881348,
+      -6.0170488357543945,
+      -6.332575798034668,
+      -6.148547172546387,
+      -5.757007598876953,
+      -5.9003777503967285,
+      -5.504100799560547,
+      -5.5883588790893555,
+      -5.009939670562744,
+      -5.31670618057251,
+      -4.6090087890625,
+      -4.64957332611084,
+      -4.408687591552734,
+      -3.7260475158691406,
+      -4.101681709289551,
+      -3.8994011878967285,
+      -5.165737152099609,
+      -3.7062177658081055,
+      -2.60445237159729,
+      -2.9606399536132812,
+      -3.406674385070801,
+      -2.8535101413726807,
+      -3.822021961212158,
+      -2.915630578994751,
+      -2.7904458045959473,
+      -1.838794469833374,
+      -2.1989693641662598,
+      -3.1347155570983887,
+      -2.6108860969543457,
+      -2.125248908996582,
+      -3.6851775646209717,
+      -3.1484384536743164,
+      -2.594548463821411,
+      -1.9205899238586426,
+      -1.8759794235229492,
+      -1.4699983596801758,
+      -2.4754478931427,
+      -1.2241714000701904,
+      -1.7784931659698486,
+      -1.4043972492218018,
+      -1.5459847450256348,
+      -1.0906867980957031,
+      -1.367735743522644,
+      -1.1391258239746094,
+      -1.3844239711761475,
+      -0.9637696743011475,
+      -1.7850940227508545,
+      -1.3724936246871948,
+      -1.0476679801940918,
+      -0.7958612442016602,
+      -1.0953247547149658,
+      -0.923236608505249,
+      -0.6573870182037354,
+      -0.963498592376709,
+      -0.4147820472717285,
+      -0.6397067308425903,
+      0.05295729637145996,
+      -0.4455200433731079,
+      -0.09821927547454834,
+      -0.48106658458709717,
+      -0.7404338121414185,
+      -0.7213606834411621,
+      -0.2695554494857788,
+      -0.538608193397522,
+      -0.25685858726501465,
+      -0.5154925584793091,
+      -0.575459361076355,
+      0.10156595706939697,
+      -0.0369570255279541,
+      -0.06800520420074463,
+      -0.22091543674468994,
+      -0.5098806619644165,
+      -0.40339577198028564,
+      -0.06847453117370605,
+      0.28865164518356323
     ],
     "base_loss": [
+      11.838828086853027,
+      11.370939254760742,
+      10.588272094726562,
+      9.626557350158691,
+      8.752111434936523,
+      7.883986949920654,
+      6.886651039123535,
+      5.24971866607666,
+      4.344367027282715,
+      3.6988837718963623,
+      3.1608898639678955,
+      2.7533035278320312,
+      2.4814517498016357,
+      2.2925686836242676,
+      2.2401986122131348,
+      2.186060667037964,
+      2.111450672149658,
+      2.060096025466919,
+      2.0241243839263916,
+      2.045396089553833,
+      2.0305867195129395,
+      1.9218031167984009,
+      2.0242366790771484,
+      1.9403225183486938,
+      1.939099669456482,
+      1.912737488746643,
+      1.9754208326339722,
+      1.9057916402816772,
+      1.9977864027023315,
+      1.9481257200241089,
+      1.875176191329956,
+      1.887980341911316,
+      1.9379119873046875,
+      1.9210156202316284,
+      1.9257110357284546,
+      1.8627643585205078,
+      1.883230209350586,
+      1.9365102052688599,
+      1.8662726879119873,
+      1.9016093015670776,
+      1.8828037977218628,
+      1.8137520551681519,
+      1.8668559789657593,
+      1.9005358219146729,
+      1.8279279470443726,
+      1.821746826171875,
+      1.8822076320648193,
+      1.8654073476791382,
+      1.914265751838684,
+      1.9144601821899414,
+      1.8745934963226318,
+      1.8415658473968506,
+      1.8748531341552734,
+      1.890276551246643,
+      1.8496427536010742,
+      1.8990840911865234,
+      1.9223690032958984,
+      1.8435914516448975,
+      1.903731346130371,
+      1.8424265384674072,
+      1.8502159118652344,
+      1.8517528772354126,
+      1.807663917541504,
+      1.8589733839035034,
+      1.8163392543792725,
+      1.7087652683258057,
+      1.6609666347503662,
+      1.6727839708328247,
+      1.6066601276397705,
+      1.5752772092819214,
+      1.5740200281143188,
+      1.5229791402816772,
+      1.5197570323944092,
+      1.4390777349472046,
+      1.522583246231079,
+      1.4195992946624756,
+      1.3857381343841553,
+      1.3357621431350708,
+      1.2583914995193481,
+      1.2635949850082397,
+      1.2785142660140991,
+      1.433748483657837,
+      1.2576029300689697,
+      1.075629472732544,
+      1.1458590030670166,
+      1.1873705387115479,
+      1.0882703065872192,
+      1.2453869581222534,
+      1.123284935951233,
+      1.067132592201233,
+      0.991763710975647,
+      1.0049587488174438,
+      1.1444469690322876,
+      1.0937929153442383,
+      0.9533916115760803,
+      1.1876996755599976,
+      1.1239768266677856,
+      1.06907320022583,
+      0.9426320791244507,
+      0.9662951231002808,
+      0.9075624346733093,
+      1.0121568441390991,
+      0.8707554936408997,
+      0.9627531170845032,
+      0.87786865234375,
+      0.9018198251724243,
+      0.8150426149368286,
+      0.8885507583618164,
+      0.8482800722122192,
+      0.8568582534790039,
+      0.8357753753662109,
+      0.9259698987007141,
+      0.8818322420120239,
+      0.8244481086730957,
+      0.7622126340866089,
+      0.8197087049484253,
+      0.8249402642250061,
+      0.7731627225875854,
+      0.8350104093551636,
+      0.7889235615730286,
+      0.76270991563797,
+      0.650455892086029,
+      0.7395504713058472,
+      0.6885175704956055,
+      0.7411212921142578,
+      0.7502254247665405,
+      0.7916255593299866,
+      0.7127252221107483,
+      0.7604287266731262,
+      0.696518063545227,
+      0.7437885999679565,
+      0.7705757021903992,
+      0.6630964875221252,
+      0.6888636946678162,
+      0.6941862106323242,
+      0.6990187764167786,
+      0.7434395551681519,
+      0.7088843584060669,
+      0.6719340682029724,
+      0.6367870569229126
     ],
     "info_loss": [
+      -0.7947063446044922,
+      -1.3534460067749023,
+      -0.974909782409668,
+      -0.4544658660888672,
+      -0.24358177185058594,
+      -0.1736612319946289,
+      -0.09863615036010742,
+      -0.06042051315307617,
+      -0.03946113586425781,
+      -0.03833341598510742,
+      -0.03979611396789551,
+      -0.03621935844421387,
+      -0.042429447174072266,
+      -0.035790443420410156,
+      -0.06885123252868652,
+      -0.05568408966064453,
+      -0.06048703193664551,
+      -0.052857398986816406,
+      -0.0776216983795166,
+      -0.07078409194946289,
+      -0.06339800357818604,
+      -0.042737722396850586,
+      -0.0667877197265625,
+      -0.06311452388763428,
+      -0.08029758930206299,
+      -0.09371721744537354,
+      -0.09059333801269531,
+      -0.09547686576843262,
+      -0.08352577686309814,
+      -0.10552775859832764,
+      -0.05450236797332764,
+      -0.08229613304138184,
+      -0.11237549781799316,
+      -0.12987017631530762,
+      -0.143774151802063,
+      -0.11218726634979248,
+      -0.0928647518157959,
+      -0.12236285209655762,
+      -0.1294715404510498,
+      -0.12258577346801758,
+      -0.1456829309463501,
+      -0.12091708183288574,
+      -0.15867328643798828,
+      -0.1391061544418335,
+      -0.16621637344360352,
+      -0.14623987674713135,
+      -0.189733624458313,
+      -0.1877741813659668,
+      -0.20934617519378662,
+      -0.29095458984375,
+      -0.3639545440673828,
+      -0.468471884727478,
+      -0.5470935106277466,
+      -0.5625057220458984,
+      -0.658776044845581,
+      -0.7161940336227417,
+      -0.8510096073150635,
+      -0.8982928395271301,
+      -0.9927720427513123,
+      -0.9438121914863586,
+      -1.0370187759399414,
+      -1.0566596984863281,
+      -1.013656497001648,
+      -1.1180551052093506,
+      -1.0859358310699463,
+      -0.9888584613800049,
+      -0.95399409532547,
+      -0.9860228896141052,
+      -0.9598300457000732,
+      -0.9168238043785095,
+      -0.9273094534873962,
+      -0.8808233737945557,
+      -0.8881385326385498,
+      -0.819733202457428,
+      -0.8567293286323547,
+      -0.7732676267623901,
+      -0.7744763493537903,
+      -0.7403684258460999,
+      -0.6630790829658508,
+      -0.70015549659729,
+      -0.6803638935089111,
+      -0.8223849534988403,
+      -0.6571109294891357,
+      -0.5277228951454163,
+      -0.5673674941062927,
+      -0.6172389388084412,
+      -0.5512561798095703,
+      -0.6607455015182495,
+      -0.5600118637084961,
+      -0.5406466126441956,
+      -0.4401552081108093,
+      -0.4710603952407837,
+      -0.578697681427002,
+      -0.5256502628326416,
+      -0.4602707326412201,
+      -0.6362345218658447,
+      -0.5754833817481995,
+      -0.5146251320838928,
+      -0.4332084655761719,
+      -0.42987990379333496,
+      -0.3861679434776306,
+      -0.49510324001312256,
+      -0.3570902347564697,
+      -0.41823434829711914,
+      -0.3782045841217041,
+      -0.3882964253425598,
+      -0.3307146728038788,
+      -0.3685101866722107,
+      -0.3422055244445801,
+      -0.3646612763404846,
+      -0.32243889570236206,
+      -0.4127432703971863,
+      -0.36529260873794556,
+      -0.32932430505752563,
+      -0.29582005739212036,
+      -0.3240930736064911,
+      -0.3139222264289856,
+      -0.2795545160770416,
+      -0.3168061375617981,
+      -0.25903958082199097,
+      -0.2791879177093506,
+      -0.19818127155303955,
+      -0.2572588324546814,
+      -0.2158147394657135,
+      -0.2577873766422272,
+      -0.28448453545570374,
+      -0.2858678102493286,
+      -0.23259294033050537,
+      -0.2640204131603241,
+      -0.2303641140460968,
+      -0.26194649934768677,
+      -0.2717556953430176,
+      -0.1934002935886383,
+      -0.20960962772369385,
+      -0.21173539757728577,
+      -0.22608724236488342,
+      -0.25635597109794617,
+      -0.24326494336128235,
+      -0.20668640732765198,
+      -0.16726526618003845
     ],
     "abs_loss": [
+      2.302412748336792,
+      2.3009450435638428,
+      2.27870512008667,
+      2.174964189529419,
+      2.084486722946167,
+      1.9960142374038696,
+      1.9213024377822876,
+      1.8566575050354004,
+      1.8366934061050415,
+      1.851310133934021,
+      1.8485580682754517,
+      1.838810920715332,
+      1.8309296369552612,
+      1.8317207098007202,
+      1.8420051336288452,
+      1.8463129997253418,
+      1.8668345212936401,
+      1.853007435798645,
+      1.8500884771347046,
+      1.8505192995071411,
+      1.8474435806274414,
+      1.8385688066482544,
+      1.861424446105957,
+      1.8699045181274414,
+      1.862662672996521,
+      1.8386759757995605,
+      1.850511908531189,
+      1.8497024774551392,
+      1.8500269651412964,
+      1.8689703941345215,
+      1.8515371084213257,
+      1.818575382232666,
+      1.8620004653930664,
+      1.8577165603637695,
+      1.8511066436767578,
+      1.838352084159851,
+      1.8330349922180176,
+      1.84242582321167,
+      1.8524103164672852,
+      1.8676964044570923,
+      1.8425636291503906,
+      1.8451820611953735,
+      1.8523412942886353,
+      1.8677786588668823,
+      1.817083716392517,
+      1.8620718717575073,
+      1.8646188974380493,
+      1.84919273853302,
+      1.845678448677063,
+      1.8551315069198608,
+      1.848828911781311,
+      1.838497281074524,
+      1.828985571861267,
+      1.8191531896591187,
+      1.8209171295166016,
+      1.8266960382461548,
+      1.8298794031143188,
+      1.8037595748901367,
+      1.8071837425231934,
+      1.7994571924209595,
+      1.795587182044983,
+      1.8066879510879517,
+      1.7850884199142456,
+      1.7945705652236938,
+      1.7674384117126465,
+      1.7592116594314575,
+      1.7298604249954224,
+      1.702737808227539,
+      1.6485201120376587,
+      1.5819498300552368,
+      1.511340618133545,
+      1.4684982299804688,
+      1.4436193704605103,
+      1.4264718294143677,
+      1.3067470788955688,
+      1.2788523435592651,
+      1.2897067070007324,
+      1.226319670677185,
+      1.1635907888412476,
+      1.1739449501037598,
+      1.0503578186035156,
+      1.10030996799469,
+      1.0934771299362183,
+      1.0553520917892456,
+      1.0155118703842163,
+      0.9916570782661438,
+      0.9301231503486633,
+      0.8723134994506836,
+      0.8847482204437256,
+      0.9377768039703369,
+      0.8362235426902771,
+      0.8876826167106628,
+      0.8097426295280457,
+      0.7707312703132629,
+      0.783437967300415,
+      0.7612269520759583,
+      0.7890119552612305,
+      0.7253015041351318,
+      0.7303829789161682,
+      0.701012909412384,
+      0.707586944103241,
+      0.7077252268791199,
+      0.6791206002235413,
+      0.6962205767631531,
+      0.649402379989624,
+      0.6630510091781616,
+      0.6399484872817993,
+      0.6480059027671814,
+      0.6101812720298767,
+      0.6286943554878235,
+      0.6452814936637878,
+      0.60471510887146,
+      0.6004258990287781,
+      0.6153071522712708,
+      0.6862974762916565,
+      0.6473205089569092,
+      0.5864364504814148,
+      0.5834178924560547,
+      0.584533154964447,
+      0.6051172614097595,
+      0.6042944192886353,
+      0.5947246551513672,
+      0.6029585003852844,
+      0.5866187214851379,
+      0.5548930168151855,
+      0.5721796751022339,
+      0.5757569670677185,
+      0.6093991994857788,
+      0.545515239238739,
+      0.5773008465766907,
+      0.5797798037528992,
+      0.5739028453826904,
+      0.5971900820732117,
+      0.5974938273429871,
+      0.5584818124771118,
+      0.5314763188362122,
+      0.5410658121109009,
+      0.581427276134491,
+      0.556355893611908,
+      0.5683047771453857
     ],
     "zipf_loss": [
+      9.214696884155273,
+      8.852276802062988,
+      7.829165458679199,
+      5.982677459716797,
+      4.213481426239014,
+      3.090301036834717,
+      2.420233964920044,
+      1.9915273189544678,
+      1.9022116661071777,
+      1.8390072584152222,
+      1.799190878868103,
+      1.7761399745941162,
+      1.7598042488098145,
+      1.748741865158081,
+      1.7345539331436157,
+      1.7285829782485962,
+      1.7251226902008057,
+      1.7209933996200562,
+      1.7162994146347046,
+      1.7138252258300781,
+      1.7109640836715698,
+      1.7070730924606323,
+      1.7049274444580078,
+      1.7034064531326294,
+      1.7017219066619873,
+      1.7008098363876343,
+      1.6994823217391968,
+      1.69841468334198,
+      1.6967672109603882,
+      1.6964107751846313,
+      1.6952325105667114,
+      1.6948657035827637,
+      1.6943769454956055,
+      1.6936094760894775,
+      1.693024754524231,
+      1.6922390460968018,
+      1.6919270753860474,
+      1.6913414001464844,
+      1.6908159255981445,
+      1.690788984298706,
+      1.6905815601348877,
+      1.6910300254821777,
+      1.6893043518066406,
+      1.6893105506896973,
+      1.689342737197876,
+      1.6886589527130127,
+      1.6884534358978271,
+      1.6885793209075928,
+      1.688183069229126,
+      1.6900287866592407,
+      1.6911423206329346,
+      1.6914634704589844,
+      1.6915680170059204,
+      1.691770076751709,
+      1.6927120685577393,
+      1.6937731504440308,
+      1.695291519165039,
+      1.6961145401000977,
+      1.696904182434082,
+      1.6971238851547241,
+      1.6973726749420166,
+      1.6968129873275757,
+      1.6974554061889648,
+      1.6948020458221436,
+      1.6926236152648926,
+      1.6902903318405151,
+      1.6889389753341675,
+      1.6845951080322266,
+      1.6782419681549072,
+      1.6777575016021729,
+      1.6475625038146973,
+      1.634303331375122,
+      1.6289076805114746,
+      1.6056675910949707,
+      1.5973291397094727,
+      1.5761830806732178,
+      1.5804808139801025,
+      1.536602258682251,
+      1.5299925804138184,
+      1.5188838243484497,
+      1.520687460899353,
+      1.5143325328826904,
+      1.4979405403137207,
+      1.4916117191314697,
+      1.4656249284744263,
+      1.479178547859192,
+      1.477769136428833,
+      1.4528149366378784,
+      1.4727284908294678,
+      1.45510995388031,
+      1.4873716831207275,
+      1.417907476425171,
+      1.4268403053283691,
+      1.4747503995895386,
+      1.445723056793213,
+      1.4133450984954834,
+      1.403517246246338,
+      1.4100992679595947,
+      1.3958241939544678,
+      1.386423110961914,
+      1.4133601188659668,
+      1.392655372619629,
+      1.4080634117126465,
+      1.3714752197265625,
+      1.4348397254943848,
+      1.3688545227050781,
+      1.3374223709106445,
+      1.364014983177185,
+      1.373631238937378,
+      1.3424608707427979,
+      1.3603157997131348,
+      1.3558974266052246,
+      1.3385576009750366,
+      1.3595960140228271,
+      1.3314967155456543,
+      1.2611653804779053,
+      1.3324017524719238,
+      1.3066534996032715,
+      1.3110988140106201,
+      1.3261786699295044,
+      1.3290331363677979,
+      1.3248416185379028,
+      1.3272221088409424,
+      1.312748670578003,
+      1.3001965284347534,
+      1.29696786403656,
+      1.288116216659546,
+      1.2827088832855225,
+      1.2866158485412598,
+      1.292534351348877,
+      1.3022056818008423,
+      1.314131498336792,
+      1.312753438949585,
+      1.3105261325836182,
+      1.2993143796920776,
+      1.2877905368804932,
+      1.256132960319519,
+      1.2622264623641968,
+      1.2708197832107544,
+      1.2676868438720703
     ],
     "denoise_loss": [],
     "ortho_loss": [
+      0.8263990879058838,
+      0.6398636102676392,
+      0.5248991847038269,
+      0.47035273909568787,
+      0.36485227942466736,
+      0.3518960177898407,
+      0.37278884649276733,
+      0.37953755259513855,
+      0.369808554649353,
+      0.35956481099128723,
+      0.34914538264274597,
+      0.34211647510528564,
+      0.35236310958862305,
+      0.34380295872688293,
+      0.3560903072357178,
+      0.36143550276756287,
+      0.35988783836364746,
+      0.3778095841407776,
+      0.36071673035621643,
+      0.3772004246711731,
+      0.3913002610206604,
+      0.4016982316970825,
+      0.3970619738101959,
+      0.38298705220222473,
+      0.38086575269699097,
+      0.3925340473651886,
+      0.39670127630233765,
+      0.39084312319755554,
+      0.405480295419693,
+      0.42086395621299744,
+      0.411644846200943,
+      0.41944432258605957,
+      0.4163327217102051,
+      0.42740166187286377,
+      0.4257947504520416,
+      0.4076772928237915,
+      0.3938109576702118,
+      0.3850422501564026,
+      0.3819939196109772,
+      0.4029465615749359,
+      0.4009494483470917,
+      0.4083222448825836,
+      0.41084668040275574,
+      0.41851985454559326,
+      0.42693084478378296,
+      0.4023897051811218,
+      0.39613503217697144,
+      0.38698458671569824,
+      0.4010132849216461,
+      0.391172856092453,
+      0.39587458968162537,
+      0.40000119805336,
+      0.4087057411670685,
+      0.41791754961013794,
+      0.42740243673324585,
+      0.43657243251800537,
+      0.44869521260261536,
+      0.4524320960044861,
+      0.4557957649230957,
+      0.4605225622653961,
+      0.46363839507102966,
+      0.46766993403434753,
+      0.47213712334632874,
+      0.47606295347213745,
+      0.4780144989490509,
+      0.4824178218841553,
+      0.48773542046546936,
+      0.490782767534256,
+      0.491031289100647,
+      0.4955742061138153,
+      0.49922680854797363,
+      0.49962109327316284,
+      0.4963490962982178,
+      0.5031077265739441,
+      0.5041939616203308,
+      0.5056381225585938,
+      0.5040112733840942,
+      0.5016523003578186,
+      0.5010254979133606,
+      0.501035749912262,
+      0.4998578429222107,
+      0.496683806180954,
+      0.4952710270881653,
+      0.49363234639167786,
+      0.4910505712032318,
+      0.48717889189720154,
+      0.4841475784778595,
+      0.4831303656101227,
+      0.4807543158531189,
+      0.47504204511642456,
+      0.4743756353855133,
+      0.4675210118293762,
+      0.4632830321788788,
+      0.459530234336853,
+      0.45602646470069885,
+      0.45379316806793213,
+      0.4504814147949219,
+      0.4467865824699402,
+      0.4409796893596649,
+      0.43924814462661743,
+      0.43782851099967957,
+      0.43564632534980774,
+      0.43388527631759644,
+      0.4302870035171509,
+      0.43054941296577454,
+      0.4253215491771698,
+      0.4239141345024109,
+      0.4234074652194977,
+      0.42124322056770325,
+      0.4201543629169464,
+      0.4194556474685669,
+      0.4171600937843323,
+      0.41740599274635315,
+      0.41401854157447815,
+      0.4137851297855377,
+      0.4160709083080292,
+      0.41565027832984924,
+      0.4148636758327484,
+      0.41444507241249084,
+      0.41314461827278137,
+      0.4130113422870636,
+      0.4124045968055725,
+      0.41070786118507385,
+      0.4096534848213196,
+      0.40888485312461853,
+      0.4067349433898926,
+      0.40550580620765686,
+      0.4057595133781433,
+      0.40574514865875244,
+      0.40414148569107056,
+      0.4031132161617279,
+      0.4023817777633667,
+      0.40334025025367737,
+      0.4025489091873169,
+      0.4025358557701111,
+      0.4025465250015259,
+      0.40197664499282837,
+      0.4014347195625305,
+      0.40142199397087097,
+      0.40162885189056396
     ],
     "lr": [
+      4.188034188034189e-06,
+      8.461538461538462e-06,
+      1.2735042735042738e-05,
+      1.700854700854701e-05,
       2e-05,
       2e-05,
       2e-05,
       2e-05,
       2e-05,
       2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      1.9967967308199317e-05,
+      1.967136831004482e-05,
+      1.9374769311890326e-05,
+      1.907817031373583e-05,
+      1.8781571315581332e-05,
+      1.848497231742684e-05,
+      1.7945162140785656e-05,
+      1.7648563142631164e-05,
+      1.735196414447667e-05,
+      1.705536514632217e-05,
+      1.675876614816768e-05,
+      1.6462167150013183e-05,
+      1.6165568151858688e-05,
+      1.5625757975217507e-05,
+      1.532915897706301e-05,
+      1.5032559978908518e-05,
+      1.473596098075402e-05,
+      1.4439361982599525e-05,
+      1.4142762984445033e-05,
+      1.3846163986290536e-05,
+      1.3306353809649356e-05,
+      1.300975481149486e-05,
+      1.2713155813340366e-05,
+      1.2416556815185872e-05,
+      1.2119957817031374e-05,
+      1.182335881887688e-05,
+      1.1526759820722382e-05,
+      1.0986949644081204e-05,
+      1.0690350645926705e-05,
+      1.0393751647772212e-05,
+      1.009715264961772e-05,
+      9.800553651463221e-06,
+      9.50395465330873e-06,
+      9.207355655154231e-06,
+      8.66754547851305e-06,
+      8.370946480358558e-06,
+      8.07434748220406e-06,
+      7.777748484049568e-06,
+      7.481149485895069e-06,
+      7.184550487740576e-06,
+      6.887951489586082e-06,
+      6.3481413129448985e-06,
+      6.051542314790406e-06,
+      5.754943316635908e-06,
+      5.4583443184814145e-06,
+      5.161745320326917e-06,
+      4.865146322172423e-06,
+      4.5685473240179305e-06,
+      4.028737147376746e-06,
+      3.732138149222253e-06,
+      3.435539151067755e-06,
+      3.1389401529132617e-06,
+      2.8423411547587686e-06,
+      2.5457421566042704e-06,
+      2.2491431584497773e-06
     ],
     "emb_lr": [],
     "eval_step": [
       2696,
       3087,
       3478,
+      3869,
+      4260,
+      4651,
+      5042,
+      5433,
+      5824,
+      6215,
+      6606,
+      6997,
+      7388,
+      7779
     ],
     "eval_accuracy": [
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.0,
       0.01,
       0.0,
       0.0,
       0.0,
       0.0,
+      0.0,
+      0.0,
       0.0,
       0.0,
       0.0
     ]
   },
+  "final_accuracy": 0.08461538461538462,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
+        "full_accuracy": 0.26,
+        "digit_accuracy": 0.8314285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.828099173553719,
+            "count": 605
           },
           "SS": {
+            "accuracy": 0.8526315789473684,
+            "count": 95
           }
         }
       },
       "add_S1": {
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.6657142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8529411764705882,
+            "count": 204
           },
           "SC": {
+            "accuracy": 0.9053254437869822,
+            "count": 169
           },
           "SS": {
+            "accuracy": 0.8064516129032258,
+            "count": 31
           },
           "UC": {
+            "accuracy": 0.38513513513513514,
+            "count": 296
           }
         }
       },
       "add_S2": {
+        "full_accuracy": 0.09,
+        "digit_accuracy": 0.7042857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9141104294478528,
+            "count": 163
           },
           "SC": {
+            "accuracy": 0.9,
+            "count": 130
           },
           "SS": {
+            "accuracy": 0.7701149425287356,
+            "count": 87
           },
           "UC": {
+            "accuracy": 0.4187192118226601,
+            "count": 203
           },
           "US": {
+            "accuracy": 0.6410256410256411,
+            "count": 117
           }
         }
       },
       "add_S3": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.5542857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8347107438016529,
+            "count": 121
           },
           "SC": {
+            "accuracy": 0.8347107438016529,
+            "count": 121
           },
           "SS": {
+            "accuracy": 0.7755102040816326,
+            "count": 49
           },
           "UC": {
+            "accuracy": 0.34408602150537637,
+            "count": 186
           },
           "US": {
+            "accuracy": 0.37668161434977576,
+            "count": 223
           }
         }
       },
       "add_S4": {
+        "full_accuracy": 0.09,
+        "digit_accuracy": 0.5085714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8653846153846154,
+            "count": 104
           },
           "SC": {
+            "accuracy": 0.8584905660377359,
+            "count": 106
           },
           "SS": {
+            "accuracy": 0.8695652173913043,
+            "count": 23
           },
           "UC": {
+            "accuracy": 0.34375,
+            "count": 160
           },
           "US": {
+            "accuracy": 0.3257328990228013,
+            "count": 307
           }
         }
       },
       "add_S5": {
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.35,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.91,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.79,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.26,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.1225,
+            "count": 400
           }
         }
       },
       "add_S6": {
+        "full_accuracy": 0.12,
+        "digit_accuracy": 0.36142857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 0.86,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.37,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.26,
+            "count": 500
           }
         }
       },
       "add_random": {
+        "full_accuracy": 0.065,
+        "digit_accuracy": 0.6621428571428571,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8478747203579419,
+            "count": 447
           },
           "SC": {
+            "accuracy": 0.884375,
+            "count": 320
           },
           "SS": {
+            "accuracy": 0.8571428571428571,
+            "count": 56
           },
           "UC": {
+            "accuracy": 0.3724007561436673,
+            "count": 529
           },
           "US": {
+            "accuracy": 0.4166666666666667,
+            "count": 48
           }
         }
       },
       "add_C1": {
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.7485714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.808,
+            "count": 500
           },
           "SC": {
+            "accuracy": 0.71,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.49,
+            "count": 100
           }
         }
       },
       "add_C2": {
+        "full_accuracy": 0.1,
+        "digit_accuracy": 0.7142857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.82,
+            "count": 400
           },
           "SC": {
+            "accuracy": 0.87,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.42948717948717946,
+            "count": 156
           },
           "US": {
+            "accuracy": 0.4090909090909091,
+            "count": 44
           }
         }
       },
       "add_C3": {
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.6485714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8233333333333334,
+            "count": 300
           },
           "SC": {
+            "accuracy": 0.96,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.32160804020100503,
+            "count": 199
           },
           "US": {
+            "accuracy": 0.46534653465346537,
+            "count": 101
           }
         }
       },
       "add_C4": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.5728571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.825,
+            "count": 200
           },
           "SC": {
+            "accuracy": 0.94,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.29924242424242425,
+            "count": 264
           },
           "US": {
+            "accuracy": 0.4632352941176471,
+            "count": 136
           }
         }
       },
       "add_C5": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.5328571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.89,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.94,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.33225806451612905,
+            "count": 310
           },
           "US": {
+            "accuracy": 0.45789473684210524,
+            "count": 190
           }
         }
       },
       "add_C6": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.49857142857142855,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 0.88,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.34864864864864864,
+            "count": 370
           },
           "US": {
+            "accuracy": 0.5739130434782609,
+            "count": 230
           }
         }
       },
       "sub_M0": {
+        "full_accuracy": 0.17,
+        "digit_accuracy": 0.7985714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.7853658536585366,
+            "count": 615
           },
           "ME": {
+            "accuracy": 0.8941176470588236,
+            "count": 85
           }
         }
       },
       "sub_M1": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.5885714285714285,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8082191780821918,
+            "count": 292
           },
           "MB": {
+            "accuracy": 0.6736111111111112,
+            "count": 144
           },
           "ME": {
+            "accuracy": 0.88,
+            "count": 25
           },
           "UB": {
+            "accuracy": 0.2384937238493724,
+            "count": 239
           }
         }
       },
       "sub_M2": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.5742857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8862559241706162,
+            "count": 211
           },
           "MB": {
+            "accuracy": 0.591304347826087,
+            "count": 115
           },
           "ME": {
+            "accuracy": 0.9529411764705882,
+            "count": 85
           },
           "UB": {
+            "accuracy": 0.30939226519337015,
+            "count": 181
           },
           "UD": {
+            "accuracy": 0.09259259259259259,
+            "count": 108
           }
         }
       },
       "sub_M3": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.5,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9329608938547486,
+            "count": 179
           },
           "MB": {
+            "accuracy": 0.6310679611650486,
+            "count": 103
           },
           "ME": {
+            "accuracy": 0.9642857142857143,
+            "count": 56
           },
           "UB": {
+            "accuracy": 0.26174496644295303,
+            "count": 149
           },
           "UD": {
+            "accuracy": 0.11737089201877934,
+            "count": 213
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.4228571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.75,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.34,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.09,
+            "count": 300
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.37,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 0.81,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.37,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.1025,
+            "count": 400
           }
         }
       },
       "sub_random": {
+        "full_accuracy": 0.015,
+        "digit_accuracy": 0.57,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8083333333333333,
+            "count": 600
           },
           "MB": {
+            "accuracy": 0.6441947565543071,
+            "count": 267
           },
           "ME": {
+            "accuracy": 0.9245283018867925,
             "count": 53
           },
           "UB": {
+            "accuracy": 0.19817767653758542,
+            "count": 439
           },
           "UD": {
+            "accuracy": 0.12195121951219512,
+            "count": 41
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.52,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.7366666666666667,
+            "count": 300
           },
           "MB": {
+            "accuracy": 0.71,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.3096446700507614,
+            "count": 197
           },
           "UD": {
+            "accuracy": 0.10679611650485436,
+            "count": 103
           }
         }
       },
       "sub_B4": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.4742857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.825,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.73,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.2591093117408907,
+            "count": 247
           },
           "UD": {
+            "accuracy": 0.19607843137254902,
+            "count": 153
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.4114285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 0.82,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.24496644295302014,
+            "count": 298
           },
           "UD": {
+            "accuracy": 0.16336633663366337,
+            "count": 202
           }
         }
       }
     },
     "summary": {
+      "overall_accuracy": 0.04923076923076923,
+      "digit_accuracy": 0.5696703296703297,
+      "total_examples": 2600,
       "n_splits": 24
     }
   },
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
+        "full_accuracy": 0.41,
+        "digit_accuracy": 0.87,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8925619834710744,
+            "count": 605
           },
           "SS": {
+            "accuracy": 0.7263157894736842,
+            "count": 95
           }
         }
       },
       "add_S1": {
+        "full_accuracy": 0.1,
+        "digit_accuracy": 0.7042857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9019607843137255,
+            "count": 204
           },
           "SC": {
+            "accuracy": 0.8461538461538461,
+            "count": 169
           },
           "SS": {
+            "accuracy": 0.8709677419354839,
+            "count": 31
           },
           "UC": {
+            "accuracy": 0.46959459459459457,
+            "count": 296
           }
         }
       },
       "add_S2": {
+        "full_accuracy": 0.12,
+        "digit_accuracy": 0.7385714285714285,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9325153374233128,
+            "count": 163
           },
           "SC": {
+            "accuracy": 0.9,
+            "count": 130
           },
           "SS": {
+            "accuracy": 0.735632183908046,
+            "count": 87
           },
           "UC": {
+            "accuracy": 0.4630541871921182,
+            "count": 203
           },
           "US": {
+            "accuracy": 0.7692307692307693,
+            "count": 117
           }
         }
       },
       "add_S3": {
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.6657142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9504132231404959,
+            "count": 121
           },
           "SC": {
+            "accuracy": 0.859504132231405,
+            "count": 121
           },
           "SS": {
+            "accuracy": 0.8979591836734694,
+            "count": 49
           },
           "UC": {
+            "accuracy": 0.44086021505376344,
+            "count": 186
           },
           "US": {
+            "accuracy": 0.5426008968609866,
+            "count": 223
           }
         }
       },
       "add_S4": {
+        "full_accuracy": 0.11,
+        "digit_accuracy": 0.6028571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9711538461538461,
+            "count": 104
           },
           "SC": {
+            "accuracy": 0.8018867924528302,
+            "count": 106
           },
           "SS": {
+            "accuracy": 0.9130434782608695,
+            "count": 23
           },
           "UC": {
+            "accuracy": 0.44375,
+            "count": 160
           },
           "US": {
+            "accuracy": 0.46905537459283386,
+            "count": 307
           }
         }
       },
       "add_S5": {
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.55,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.87,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.28,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.425,
+            "count": 400
           }
         }
       },
       "add_S6": {
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.40714285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 0.89,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.3,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.332,
+            "count": 500
           }
         }
       },
       "add_random": {
+        "full_accuracy": 0.085,
+        "digit_accuracy": 0.7257142857142858,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9284116331096197,
+            "count": 447
           },
           "SC": {
+            "accuracy": 0.896875,
+            "count": 320
           },
           "SS": {
+            "accuracy": 0.75,
+            "count": 56
           },
           "UC": {
+            "accuracy": 0.4499054820415879,
+            "count": 529
           },
           "US": {
+            "accuracy": 0.7083333333333334,
+            "count": 48
           }
         }
       },
       "add_C1": {
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.8142857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.894,
+            "count": 500
           },
           "SC": {
+            "accuracy": 0.81,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.42,
+            "count": 100
           }
         }
       },
       "add_C2": {
+        "full_accuracy": 0.16,
+        "digit_accuracy": 0.7828571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9325,
+            "count": 400
           },
           "SC": {
+            "accuracy": 0.84,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.41025641025641024,
+            "count": 156
           },
           "US": {
+            "accuracy": 0.6136363636363636,
+            "count": 44
           }
         }
       },
       "add_C3": {
+        "full_accuracy": 0.08,
+        "digit_accuracy": 0.71,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8966666666666666,
+            "count": 300
           },
           "SC": {
+            "accuracy": 0.87,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.36180904522613067,
+            "count": 199
           },
           "US": {
+            "accuracy": 0.6831683168316832,
+            "count": 101
           }
         }
       },
       "add_C4": {
+        "full_accuracy": 0.05,
+        "digit_accuracy": 0.6157142857142858,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.92,
+            "count": 200
           },
           "SC": {
+            "accuracy": 0.85,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.2916666666666667,
+            "count": 264
           },
           "US": {
+            "accuracy": 0.625,
+            "count": 136
           }
         }
       },
       "add_C5": {
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.6071428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.99,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.95,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.4161290322580645,
+            "count": 310
           },
           "US": {
+            "accuracy": 0.5368421052631579,
+            "count": 190
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.5742857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 0.94,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.45135135135135135,
+            "count": 370
           },
           "US": {
+            "accuracy": 0.6130434782608696,
+            "count": 230
           }
         }
       },
       "sub_M0": {
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.7957142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.7804878048780488,
+            "count": 615
           },
           "ME": {
+            "accuracy": 0.9058823529411765,
+            "count": 85
           }
         }
       },
       "sub_M1": {
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.6628571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.7705479452054794,
+            "count": 292
           },
           "MB": {
+            "accuracy": 0.8680555555555556,
+            "count": 144
           },
           "ME": {
+            "accuracy": 0.88,
+            "count": 25
           },
           "UB": {
+            "accuracy": 0.38493723849372385,
+            "count": 239
           }
         }
       },
       "sub_M2": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.6585714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8672985781990521,
+            "count": 211
           },
           "MB": {
+            "accuracy": 0.8608695652173913,
+            "count": 115
           },
           "ME": {
+            "accuracy": 0.9764705882352941,
+            "count": 85
           },
           "UB": {
+            "accuracy": 0.430939226519337,
+            "count": 181
           },
           "UD": {
+            "accuracy": 0.16666666666666666,
+            "count": 108
           }
         }
       },
       "sub_M3": {
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.6,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9329608938547486,
+            "count": 179
           },
           "MB": {
+            "accuracy": 0.8932038834951457,
+            "count": 103
           },
           "ME": {
             "accuracy": 1.0,
+            "count": 56
           },
           "UB": {
+            "accuracy": 0.4228187919463087,
+            "count": 149
           },
           "UD": {
+            "accuracy": 0.19718309859154928,
+            "count": 213
           }
         }
       },
       "sub_M4": {
+        "full_accuracy": 0.05,
+        "digit_accuracy": 0.5314285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.86,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.92,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.54,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.18,
+            "count": 300
           }
         }
       },
       "sub_M5": {
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.3942857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 0.97,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.51,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.07,
+            "count": 400
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.04,
+        "digit_accuracy": 0.6657142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8083333333333333,
+            "count": 600
           },
           "MB": {
+            "accuracy": 0.8539325842696629,
+            "count": 267
           },
           "ME": {
+            "accuracy": 0.9245283018867925,
             "count": 53
           },
           "UB": {
+            "accuracy": 0.3690205011389522,
+            "count": 439
           },
           "UD": {
+            "accuracy": 0.1951219512195122,
+            "count": 41
           }
         }
       },
       "sub_B3": {
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.5985714285714285,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8133333333333334,
+            "count": 300
           },
           "MB": {
+            "accuracy": 0.84,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.38578680203045684,
+            "count": 197
           },
           "UD": {
+            "accuracy": 0.14563106796116504,
+            "count": 103
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.5528571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.875,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.91,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.3360323886639676,
+            "count": 247
           },
           "UD": {
+            "accuracy": 0.24836601307189543,
+            "count": 153
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.5,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.348993288590604,
+            "count": 298
           },
           "UD": {
+            "accuracy": 0.22772277227722773,
+            "count": 202
           }
         }
       }
     },
     "summary": {
+      "overall_accuracy": 0.08461538461538462,
+      "digit_accuracy": 0.642967032967033,
+      "total_examples": 2600,
       "n_splits": 24
     }
   },
+  "sorl_overall_accuracy": 0.08461538461538462,
+  "sft_overall_accuracy": 0.04923076923076923
 }

add_sub_sorl_v1_abs10_K1_25K_1L2H256d/model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a10bb1f5b2eb181b8afa8885bc9617b44e94d4a7398d421530a00ee21fe002b3
 size 315091124

 version https://git-lfs.github.com/spec/v1
+oid sha256:47628d01fe602b58dee9cf66e44ae762c69be2e2df852104d940b6c1aca5d2b0
 size 315091124

add_sub_sorl_v1_abs10_K1_25K_1L2H256d/train_config.json CHANGED Viewed

@@ -20,7 +20,7 @@
   "lr": 2e-05,
   "emb_lr_mult": 1.0,
   "weight_decay": 0.01,
-  "warmup_steps": 117,
   "cooldown_frac": 0.4,
   "max_grad_norm": 1.0,
   "vq_abs_pretrain_steps": 0,
@@ -30,7 +30,7 @@
   "vq_abs_pretrain_target_vectors": 20000,
   "batch_size": 64,
   "gradient_accumulation_steps": 1,
-  "num_epochs": 10,
   "emb_warmup_steps": 0,
   "log_every": 50,
   "eval_every": 390,
@@ -69,16 +69,20 @@
   "no_wandb": false,
   "n_params": 78696448,
   "run_name": "add_sub_sorl_v1_abs10_K1_25K_1L2H256d",
-  "git_commit": "3ae8ca0d6b88706715f25991b1e1acd0e3a6e0a6",
-  "timestamp": "2026-04-12T20:10:10.507089+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "v1",
-  "wandb_run_id": "8olk3u7a",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/8olk3u7a",
-  "final_accuracy": 0.06291666666666666,
-  "sft_accuracy": 0.011666666666666667,
   "eval_method": "ArithmeticEvaluator"
 }

   "lr": 2e-05,
   "emb_lr_mult": 1.0,
   "weight_decay": 0.01,
+  "warmup_steps": 234,
   "cooldown_frac": 0.4,
   "max_grad_norm": 1.0,
   "vq_abs_pretrain_steps": 0,
   "vq_abs_pretrain_target_vectors": 20000,
   "batch_size": 64,
   "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
   "emb_warmup_steps": 0,
   "log_every": 50,
   "eval_every": 390,
   "no_wandb": false,
   "n_params": 78696448,
   "run_name": "add_sub_sorl_v1_abs10_K1_25K_1L2H256d",
+  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
+  "timestamp": "2026-04-15T20:34:04.553671+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_25K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "v1",
+  "wandb_run_id": "lz3nsjvy",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/lz3nsjvy",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "final_accuracy": 0.08461538461538462,
+  "sft_accuracy": 0.04923076923076923,
   "eval_method": "ArithmeticEvaluator"
 }