Upload add_sub_sorl_v1_abs10_K1_25K_2L1H128d

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 1,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 1,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151654
+}

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/metrics.json

@@ -0,0 +1,1617 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      441,
+      491,
+      541,
+      591,
+      641,
+      691,
+      741,
+      832,
+      882,
+      932,
+      982,
+      1032,
+      1082,
+      1132,
+      1223,
+      1273,
+      1323,
+      1373,
+      1423,
+      1473,
+      1523,
+      1614,
+      1664,
+      1714,
+      1764,
+      1814,
+      1864,
+      1914,
+      2005,
+      2055,
+      2105,
+      2155,
+      2205,
+      2255,
+      2305,
+      2396,
+      2446,
+      2496,
+      2546,
+      2596,
+      2646,
+      2696,
+      2787,
+      2837,
+      2887,
+      2937,
+      2987,
+      3037,
+      3087,
+      3178,
+      3228,
+      3278,
+      3328,
+      3378,
+      3428,
+      3478,
+      3569,
+      3619,
+      3669,
+      3719,
+      3769,
+      3819,
+      3869
+    ],
+    "loss": [
+      13.454885482788086,
+      11.478792190551758,
+      13.050385475158691,
+      14.577110290527344,
+      14.17066764831543,
+      13.036783218383789,
+      11.886457443237305,
+      10.250679016113281,
+      9.566696166992188,
+      8.962414741516113,
+      8.374322891235352,
+      7.972871780395508,
+      7.557723045349121,
+      7.117671012878418,
+      6.462696075439453,
+      6.050843238830566,
+      5.741342067718506,
+      5.391325950622559,
+      5.264486789703369,
+      4.956804275512695,
+      4.797525405883789,
+      4.500096321105957,
+      4.433718681335449,
+      4.317060470581055,
+      4.279264450073242,
+      4.234098434448242,
+      4.042954921722412,
+      4.035151481628418,
+      4.045926094055176,
+      3.9358861446380615,
+      3.753159761428833,
+      3.7763824462890625,
+      3.8419253826141357,
+      3.6180100440979004,
+      3.3646740913391113,
+      3.16855525970459,
+      3.360339641571045,
+      2.8317813873291016,
+      2.9290781021118164,
+      2.785353660583496,
+      2.54134464263916,
+      2.228949785232544,
+      2.0700716972351074,
+      1.8415426015853882,
+      1.5988508462905884,
+      1.569923758506775,
+      1.2837797403335571,
+      1.1172637939453125,
+      0.7710850834846497,
+      0.6829363107681274,
+      0.49595534801483154,
+      0.17024922370910645,
+      0.21459567546844482,
+      -0.10922026634216309,
+      -0.38342714309692383,
+      -0.10504281520843506,
+      -0.5501455068588257,
+      -0.7160437107086182,
+      -0.707771897315979,
+      -0.9133639335632324,
+      -1.1110361814498901,
+      -0.8491053581237793,
+      -0.9960892200469971,
+      -1.0281953811645508,
+      -1.2926632165908813,
+      -1.2633980512619019,
+      -1.2680187225341797,
+      -1.4050925970077515,
+      -1.2741034030914307,
+      -1.4242706298828125
+    ],
+    "base_loss": [
+      11.880859375,
+      11.633416175842285,
+      11.221420288085938,
+      10.539042472839355,
+      10.006148338317871,
+      9.441154479980469,
+      8.854585647583008,
+      7.859694480895996,
+      7.329966068267822,
+      6.851498603820801,
+      6.352517127990723,
+      6.018871784210205,
+      5.628158092498779,
+      5.2450385093688965,
+      4.57428503036499,
+      4.198834419250488,
+      3.868682861328125,
+      3.5653653144836426,
+      3.412782907485962,
+      3.165587902069092,
+      2.9751205444335938,
+      2.7248756885528564,
+      2.6281933784484863,
+      2.598451852798462,
+      2.5072898864746094,
+      2.5161983966827393,
+      2.393761157989502,
+      2.3599026203155518,
+      2.3197877407073975,
+      2.3430163860321045,
+      2.2583587169647217,
+      2.2844526767730713,
+      2.254804849624634,
+      2.290438413619995,
+      2.2097489833831787,
+      2.197057008743286,
+      2.2262990474700928,
+      2.1319544315338135,
+      2.107748031616211,
+      2.173170328140259,
+      2.1438701152801514,
+      2.181342363357544,
+      2.0934159755706787,
+      2.126227617263794,
+      2.09466814994812,
+      2.0434868335723877,
+      2.1309261322021484,
+      2.075498342514038,
+      2.1343188285827637,
+      2.0033481121063232,
+      2.0886590480804443,
+      2.092684507369995,
+      2.020324945449829,
+      1.984065055847168,
+      2.0311124324798584,
+      2.026329755783081,
+      1.9996259212493896,
+      2.0779316425323486,
+      1.9736812114715576,
+      2.030266284942627,
+      2.1000616550445557,
+      2.038893461227417,
+      1.9640614986419678,
+      2.036902904510498,
+      2.0785582065582275,
+      2.0229272842407227,
+      2.0268771648406982,
+      2.0155155658721924,
+      1.9266163110733032,
+      2.076498508453369
+    ],
+    "info_loss": [
+      -0.7863264083862305,
+      -0.9126081466674805,
+      -0.61785888671875,
+      -0.2595643997192383,
+      -0.1060333251953125,
+      -0.05567169189453125,
+      -0.03524017333984375,
+      -0.020551681518554688,
+      -0.01478719711303711,
+      -0.013951778411865234,
+      -0.01464700698852539,
+      -0.015613079071044922,
+      -0.013948440551757812,
+      -0.016277790069580078,
+      -0.010252952575683594,
+      -0.012321949005126953,
+      -0.008607149124145508,
+      -0.012037992477416992,
+      -0.008737802505493164,
+      -0.013904571533203125,
+      -0.010097980499267578,
+      -0.013910055160522461,
+      -0.010562896728515625,
+      -0.018952369689941406,
+      -0.013309478759765625,
+      -0.018421173095703125,
+      -0.025274276733398438,
+      -0.02254462242126465,
+      -0.01696610450744629,
+      -0.03021550178527832,
+      -0.03984832763671875,
+      -0.0401759147644043,
+      -0.030680418014526367,
+      -0.05649089813232422,
+      -0.07359719276428223,
+      -0.09176802635192871,
+      -0.07550787925720215,
+      -0.11891603469848633,
+      -0.10670852661132812,
+      -0.1276233196258545,
+      -0.14898061752319336,
+      -0.1840728521347046,
+      -0.19084644317626953,
+      -0.2173391580581665,
+      -0.2383211851119995,
+      -0.23623204231262207,
+      -0.27347636222839355,
+      -0.28460466861724854,
+      -0.32501447200775146,
+      -0.32063066959381104,
+      -0.34785377979278564,
+      -0.38096821308135986,
+      -0.3693283796310425,
+      -0.39820408821105957,
+      -0.4302417039871216,
+      -0.40181398391723633,
+      -0.4437370300292969,
+      -0.46821415424346924,
+      -0.4569128751754761,
+      -0.4830566644668579,
+      -0.5098972320556641,
+      -0.4775048494338989,
+      -0.4846118688583374,
+      -0.4952908754348755,
+      -0.5258979797363281,
+      -0.5176137685775757,
+      -0.518396258354187,
+      -0.5309580564498901,
+      -0.5087277889251709,
+      -0.5388277769088745
+    ],
+    "abs_loss": [
+      2.3025481700897217,
+      2.2900214195251465,
+      2.254666566848755,
+      2.190378189086914,
+      2.0967636108398438,
+      2.037475347518921,
+      1.9807038307189941,
+      1.8861985206604004,
+      1.8569782972335815,
+      1.824637532234192,
+      1.8228012323379517,
+      1.818422794342041,
+      1.8206357955932617,
+      1.822617530822754,
+      1.804533839225769,
+      1.8139501810073853,
+      1.793150544166565,
+      1.7828887701034546,
+      1.8064988851547241,
+      1.8026375770568848,
+      1.797040343284607,
+      1.8011388778686523,
+      1.8015633821487427,
+      1.8015815019607544,
+      1.799573302268982,
+      1.7933950424194336,
+      1.805014967918396,
+      1.8085817098617554,
+      1.7863329648971558,
+      1.7919312715530396,
+      1.783199429512024,
+      1.7947336435317993,
+      1.8047763109207153,
+      1.8011966943740845,
+      1.788076400756836,
+      1.7798686027526855,
+      1.7843881845474243,
+      1.7868894338607788,
+      1.7851332426071167,
+      1.7916141748428345,
+      1.7841373682022095,
+      1.7945789098739624,
+      1.7695688009262085,
+      1.8075765371322632,
+      1.8012295961380005,
+      1.8081436157226562,
+      1.7965914011001587,
+      1.7999669313430786,
+      1.7932742834091187,
+      1.7773023843765259,
+      1.7790831327438354,
+      1.7893775701522827,
+      1.7931427955627441,
+      1.8046531677246094,
+      1.7975425720214844,
+      1.7841027975082397,
+      1.7952299118041992,
+      1.8000589609146118,
+      1.7956489324569702,
+      1.7838338613510132,
+      1.7932010889053345,
+      1.785744547843933,
+      1.773536205291748,
+      1.7905513048171997,
+      1.792868971824646,
+      1.8099616765975952,
+      1.801303505897522,
+      1.800384521484375,
+      1.7764925956726074,
+      1.7878471612930298
+    ],
+    "zipf_loss": [
+      9.207036018371582,
+      8.74245548248291,
+      7.782087326049805,
+      6.414673805236816,
+      5.0151753425598145,
+      3.9485976696014404,
+      3.1862025260925293,
+      2.4078822135925293,
+      2.198904514312744,
+      2.067970037460327,
+      1.9859960079193115,
+      1.928288459777832,
+      1.8869855403900146,
+      1.853148341178894,
+      1.8104872703552246,
+      1.7938334941864014,
+      1.7794156074523926,
+      1.7680517435073853,
+      1.7584320306777954,
+      1.7499982118606567,
+      1.743680715560913,
+      1.7342076301574707,
+      1.730997920036316,
+      1.7279741764068604,
+      1.7251121997833252,
+      1.7227725982666016,
+      1.7214350700378418,
+      1.7198371887207031,
+      1.7171660661697388,
+      1.7158317565917969,
+      1.7149643898010254,
+      1.7142155170440674,
+      1.713447093963623,
+      1.7123608589172363,
+      1.7120893001556396,
+      1.7111916542053223,
+      1.7106807231903076,
+      1.7102984189987183,
+      1.7099019289016724,
+      1.7092549800872803,
+      1.7088669538497925,
+      1.7088780403137207,
+      1.7081632614135742,
+      1.707948923110962,
+      1.7072715759277344,
+      1.7079429626464844,
+      1.7079581022262573,
+      1.707815408706665,
+      1.7075835466384888,
+      1.7081646919250488,
+      1.707925796508789,
+      1.7083090543746948,
+      1.708240270614624,
+      1.7082902193069458,
+      1.708122968673706,
+      1.7083569765090942,
+      1.7080758810043335,
+      1.708160161972046,
+      1.7081109285354614,
+      1.7085528373718262,
+      1.7085543870925903,
+      1.70847487449646,
+      1.7086145877838135,
+      1.7087550163269043,
+      1.708471417427063,
+      1.708816409111023,
+      1.7089366912841797,
+      1.7089341878890991,
+      1.708909034729004,
+      1.7087242603302002
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.6665944457054138,
+      0.6689225435256958,
+      0.6665652394294739,
+      0.5638207197189331,
+      0.5186919569969177,
+      0.4937678277492523,
+      0.440208375453949,
+      0.44368138909339905,
+      0.4490883946418762,
+      0.46086809039115906,
+      0.47744476795196533,
+      0.4832924008369446,
+      0.5013949871063232,
+      0.49298223853111267,
+      0.48781320452690125,
+      0.4863438904285431,
+      0.4936782121658325,
+      0.49258145689964294,
+      0.49533185362815857,
+      0.507675290107727,
+      0.5198795795440674,
+      0.49237918853759766,
+      0.5015400648117065,
+      0.4975549578666687,
+      0.4957054555416107,
+      0.49398577213287354,
+      0.507213294506073,
+      0.5123801827430725,
+      0.5207350254058838,
+      0.5060194134712219,
+      0.5115739107131958,
+      0.5285096764564514,
+      0.5283535122871399,
+      0.524570643901825,
+      0.5327265858650208,
+      0.549440324306488,
+      0.5494626760482788,
+      0.558219313621521,
+      0.5633650422096252,
+      0.5702787637710571,
+      0.5571073293685913,
+      0.5566506385803223,
+      0.5707675814628601,
+      0.5843377113342285,
+      0.584276556968689,
+      0.5893025994300842,
+      0.6109833121299744,
+      0.6101059317588806,
+      0.6098511815071106,
+      0.6212247610092163,
+      0.6309865713119507,
+      0.6344567537307739,
+      0.6354795098304749,
+      0.647710919380188,
+      0.6540753841400146,
+      0.6593545079231262,
+      0.6664237976074219,
+      0.6748871803283691,
+      0.6771064400672913,
+      0.6822501420974731,
+      0.6884104013442993,
+      0.6926484704017639,
+      0.6983345150947571,
+      0.7035016417503357,
+      0.7052139639854431,
+      0.7094832062721252,
+      0.7137283682823181,
+      0.7142598628997803,
+      0.7140603065490723,
+      0.716733455657959
+    ],
+    "lr": [
+      8.376068376068378e-06,
+      1.6923076923076924e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      1.9973899288162407e-05,
+      1.9380701291853413e-05,
+      1.8787503295544426e-05,
+      1.8194305299235432e-05,
+      1.7601107302926442e-05,
+      1.7007909306617455e-05,
+      1.6414711310308464e-05,
+      1.5335090957026102e-05,
+      1.474189296071711e-05,
+      1.4148694964408121e-05,
+      1.355549696809913e-05,
+      1.2962298971790142e-05,
+      1.2369100975481152e-05,
+      1.1775902979172158e-05,
+      1.0696282625889797e-05,
+      1.0103084629580805e-05,
+      9.50988663327182e-06,
+      8.91668863696283e-06,
+      8.323490640653837e-06,
+      7.730292644344845e-06,
+      7.137094648035855e-06,
+      6.057474294753492e-06,
+      5.4642762984445065e-06,
+      4.871078302135514e-06,
+      4.277880305826523e-06,
+      3.684682309517532e-06,
+      3.091484313208541e-06,
+      2.4982863168995496e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      350,
+      741,
+      1132,
+      1523,
+      1914,
+      2305,
+      2696,
+      3087,
+      3478,
+      3869
+    ],
+    "eval_accuracy": [
+      0.01,
+      0.0,
+      0.0,
+      0.01,
+      0.0,
+      0.0,
+      0.01,
+      0.0,
+      0.0,
+      0.01
+    ]
+  },
+  "final_accuracy": 0.020416666666666666,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2066115702479339,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.2549019607843137,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.03550295857988166,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.8064516129032258,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.14527027027027026,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.4662576687116564,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.07692307692307693,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.5977011494252874,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.26108374384236455,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.4017094017094017,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.5206611570247934,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.024793388429752067,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.8571428571428571,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.24731182795698925,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.35874439461883406,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.5576923076923077,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.04716981132075472,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.8695652173913043,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.25,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.3583061889250814,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.45,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.11,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.04,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.26,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.26,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.27069351230425054,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.053125,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.7321428571428571,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.166351606805293,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.3333333333333333,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.24,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.02,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.06735751295336788,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.06542056074766354,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.335,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.01,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.04296875,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.06944444444444445,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.49,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.05,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.08169934640522876,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.23195876288659795,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.01,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.1912568306010929,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.6239316239316239,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.20465890183028287,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3835125448028674,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.09523809523809523,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.6150234741784038,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.16574585635359115,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.7597765363128491,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.12080536912751678,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.3,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.31,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.0,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3616666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.12072892938496584,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.3333333333333333,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.17766497461928935,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.145748987854251,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.11073825503355705,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.0,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.0,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.09,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.6776859504132231,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.8947368421052632,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.696078431372549,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.6804733727810651,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.8387096774193549,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.4831081081081081,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.05,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.7423312883435583,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.6615384615384615,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.7471264367816092,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.37438423645320196,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.49572649572649574,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.743801652892562,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.6611570247933884,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.8367346938775511,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.3709677419354839,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.3811659192825112,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.7307692307692307,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.7075471698113207,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.9565217391304348,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.325,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.3322475570032573,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.03,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.65,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.63,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.13,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.285,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.48,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.0,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.036,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.04,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.6487695749440716,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.70625,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.8571428571428571,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.46313799621928164,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.625,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8033333333333333,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.78,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.19170984455958548,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.5327102803738317,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.8,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.75,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.1875,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.5833333333333334,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.79,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.65,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.23529411764705882,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.5979381443298969,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.6,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.319672131147541,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.6282051282051282,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5391014975041597,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.9595959595959596,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.6666666666666666,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.38620689655172413,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 0.9583333333333334,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.5198412698412699,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.755868544600939,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.26548672566371684,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 0.9764705882352941,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.5524861878453039,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.2037037037037037,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.8547486033519553,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.3300970873786408,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5838926174496645,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.08450704225352113,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.65,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.21,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.67,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.06333333333333334,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.06,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.13,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.72,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.145,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.01,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.635,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.45318352059925093,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 0.9811320754716981,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.5444191343963554,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.1951219512195122,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.5266666666666666,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.41,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.49746192893401014,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.11650485436893204,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.67,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 0.35,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.4898785425101215,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.16993464052287582,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 0.15,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.4395973154362416,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.1782178217821782,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.020416666666666666,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.020416666666666666,
+  "sft_overall_accuracy": 0.0
+}

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:552c3c376c464a283388ace103d56b77840cec872425d931fcab7533aaebd6fd
+size 157702060

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 2e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 117,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 390,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs10_K1_25K_2L1H128d",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 1,
+  "n_embd": 128,
+  "ops": "add_sub",
+  "abs_vocab": 10,
+  "dataset_size": 25000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 39348864,
+  "run_name": "add_sub_sorl_v1_abs10_K1_25K_2L1H128d",
+  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
+  "timestamp": "2026-04-12T13:04:02.986587+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "r858hl5t",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/r858hl5t",
+  "final_accuracy": 0.020416666666666666,
+  "sft_accuracy": 0.0,
+  "eval_method": "ArithmeticEvaluator"
+}