Upload add_sub_sorl_v1_abs10_K1_10K

add_sub_sorl_v1_abs10_K1_10K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151654
+}

add_sub_sorl_v1_abs10_K1_10K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_sorl_v1_abs10_K1_10K/metrics.json

@@ -0,0 +1,1297 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      207,
+      257,
+      307,
+      364,
+      414,
+      464,
+      521,
+      571,
+      621,
+      678,
+      728,
+      778,
+      835,
+      885,
+      935,
+      992,
+      1042,
+      1092,
+      1149,
+      1199,
+      1249,
+      1306,
+      1356,
+      1406,
+      1463,
+      1513,
+      1563
+    ],
+    "loss": [
+      4.547272682189941,
+      7.030660629272461,
+      3.9523563385009766,
+      3.047678232192993,
+      3.085705280303955,
+      2.618412494659424,
+      2.579326629638672,
+      1.4848873615264893,
+      -0.47891831398010254,
+      -5.560091495513916,
+      -4.520766735076904,
+      -2.9085822105407715,
+      -2.024510383605957,
+      -1.1992690563201904,
+      -0.9796783924102783,
+      -0.5373831987380981,
+      -0.5736587047576904,
+      -0.7218008041381836,
+      -0.23317885398864746,
+      -0.339227557182312,
+      -0.03128504753112793,
+      -0.006059408187866211,
+      0.16202318668365479,
+      0.23520565032958984,
+      0.015602707862854004,
+      0.4141479730606079,
+      0.1808193325996399,
+      0.15000015497207642,
+      0.025604248046875,
+      0.30849093198776245
+    ],
+    "base_loss": [
+      8.916278839111328,
+      6.119595050811768,
+      3.673598527908325,
+      2.020040273666382,
+      1.944067120552063,
+      1.8892498016357422,
+      1.86012864112854,
+      1.8146944046020508,
+      1.869857907295227,
+      1.7928285598754883,
+      1.4957207441329956,
+      1.1758774518966675,
+      0.93010014295578,
+      0.6884778141975403,
+      0.6303120255470276,
+      0.5103054046630859,
+      0.4800644516944885,
+      0.431017130613327,
+      0.3630550801753998,
+      0.3632798194885254,
+      0.32823804020881653,
+      0.2981429398059845,
+      0.24132393300533295,
+      0.23764917254447937,
+      0.2843627333641052,
+      0.19818377494812012,
+      0.2125009298324585,
+      0.18415263295173645,
+      0.19887767732143402,
+      0.1734028160572052
+    ],
+    "info_loss": [
+      -1.2568669319152832,
+      -0.31183719635009766,
+      -0.1900501251220703,
+      -0.09034490585327148,
+      -0.07560265064239502,
+      -0.11611974239349365,
+      -0.11614418029785156,
+      -0.2208923101425171,
+      -0.42284250259399414,
+      -0.9240456819534302,
+      -0.7907073497772217,
+      -0.5979017615318298,
+      -0.4831601679325104,
+      -0.3740352690219879,
+      -0.34062066674232483,
+      -0.2785865366458893,
+      -0.27203837037086487,
+      -0.27023085951805115,
+      -0.20767968893051147,
+      -0.21481892466545105,
+      -0.16902248561382294,
+      -0.1613115668296814,
+      -0.1327231526374817,
+      -0.12193454056978226,
+      -0.14406734704971313,
+      -0.09546159207820892,
+      -0.1119590476155281,
+      -0.11057599633932114,
+      -0.12191784381866455,
+      -0.09515686333179474
+    ],
+    "abs_loss": [
+      2.28905987739563,
+      2.068211555480957,
+      1.8956918716430664,
+      1.865671157836914,
+      1.8401455879211426,
+      1.8589462041854858,
+      1.8197526931762695,
+      1.8445491790771484,
+      1.8481858968734741,
+      1.8446826934814453,
+      1.8715795278549194,
+      1.8394945859909058,
+      1.7485918998718262,
+      1.6203765869140625,
+      1.4851690530776978,
+      1.3853811025619507,
+      1.1656250953674316,
+      1.0630995035171509,
+      0.8788065910339355,
+      0.8097158074378967,
+      0.7307209968566895,
+      0.6055489182472229,
+      0.6438609957695007,
+      0.5721376538276672,
+      0.5374541878700256,
+      0.5297055840492249,
+      0.45829495787620544,
+      0.428969144821167,
+      0.47001174092292786,
+      0.4526701867580414
+    ],
+    "zipf_loss": [
+      7.970757007598877,
+      3.8226165771484375,
+      1.9896899461746216,
+      1.7445199489593506,
+      1.713650107383728,
+      1.704465627670288,
+      1.6986644268035889,
+      1.6946611404418945,
+      1.6948304176330566,
+      1.7030682563781738,
+      1.7034276723861694,
+      1.7106086015701294,
+      1.702131748199463,
+      1.690568208694458,
+      1.6476993560791016,
+      1.59963858127594,
+      1.550097942352295,
+      1.443180799484253,
+      1.392682433128357,
+      1.3647103309631348,
+      1.2576297521591187,
+      1.2483584880828857,
+      1.183544635772705,
+      1.159688115119934,
+      1.1181681156158447,
+      1.1176096200942993,
+      1.0420794486999512,
+      1.0287106037139893,
+      0.9989038109779358,
+      1.0413897037506104
+    ],
+    "denoise_loss": [],
+    "ortho_loss": [
+      0.5579876899719238,
+      0.252128541469574,
+      0.21211421489715576,
+      0.18933552503585815,
+      0.12901675701141357,
+      0.13211114704608917,
+      0.14735311269760132,
+      0.14610400795936584,
+      0.1557065099477768,
+      0.17582964897155762,
+      0.1939730942249298,
+      0.20376113057136536,
+      0.21460682153701782,
+      0.23509620130062103,
+      0.24628381431102753,
+      0.2590982913970947,
+      0.25852254033088684,
+      0.2667267620563507,
+      0.26971182227134705,
+      0.2820512354373932,
+      0.2821248769760132,
+      0.28232815861701965,
+      0.2819784879684448,
+      0.2866751551628113,
+      0.2907232642173767,
+      0.2921256422996521,
+      0.2953319251537323,
+      0.29460301995277405,
+      0.295316606760025,
+      0.2968752384185791
+    ],
+    "lr": [
+      1.9600000000000002e-05,
+      3.96e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      4e-05,
+      3.944897959183673e-05,
+      3.638775510204082e-05,
+      3.3326530612244897e-05,
+      2.983673469387755e-05,
+      2.6775510204081637e-05,
+      2.3714285714285713e-05,
+      2.022448979591837e-05,
+      1.7163265306122454e-05,
+      1.4102040816326535e-05,
+      1.0612244897959182e-05,
+      7.551020408163262e-06,
+      4.48979591836735e-06
+    ],
+    "emb_lr": [],
+    "eval_step": [
+      150,
+      307,
+      464,
+      621,
+      778,
+      935,
+      1092,
+      1199,
+      1356,
+      1513
+    ],
+    "eval_accuracy": [
+      0.01,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ]
+  },
+  "final_accuracy": 0.6116666666666667,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.89,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9834710743801653,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 0.9473684210526315,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.74,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9803921568627451,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 0.9467455621301775,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9391891891891891,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.5,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9631901840490797,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9230769230769231,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.8505747126436781,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.8325123152709359,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 0.9743589743589743,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.29,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9421487603305785,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 0.8979591836734694,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7204301075268817,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.8340807174887892,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.19,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9711538461538461,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 0.9622641509433962,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 0.9565217391304348,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.59375,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.6156351791530945,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.19,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.31,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.4775,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.53,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.65,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.672,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.73,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9776286353467561,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.975,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9338374291115312,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 0.9583333333333334,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.42,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 0.99,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7253886010362695,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.8504672897196262,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.43,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.96,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.76953125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.8055555555555556,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.31,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.7745098039215687,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.7783505154639175,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.44,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8306010928961749,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.8931623931623932,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.87,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9750415973377704,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.82,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.989247311827957,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.9586206896551724,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 0.9563492063492064,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.47,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9953051643192489,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9380530973451328,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.7237569060773481,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 0.9444444444444444,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.12,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.994413407821229,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9805825242718447,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.5302013422818792,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.7417840375586855,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.0,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.36,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.2966666666666667,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.01,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.3,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.215,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.765,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9783333333333334,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9700374531835206,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9384965831435079,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 0.9512195121951219,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.49,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 0.98,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.766497461928934,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.7572815533980582,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.4,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7975708502024291,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.6928104575163399,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.2,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.7348993288590604,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.6782178217821783,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.47041666666666665,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": 1,
+      "mode": "sorl",
+      "n_digits": 6,
+      "n_per_split": 100
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.97,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9950413223140496,
+            "count": 605
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 95
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.96,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9901960784313726,
+            "count": 204
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 169
+          },
+          "SS": {
+            "accuracy": 0.967741935483871,
+            "count": 31
+          },
+          "UC": {
+            "accuracy": 0.9932432432432432,
+            "count": 296
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.83,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9815950920245399,
+            "count": 163
+          },
+          "SC": {
+            "accuracy": 0.9769230769230769,
+            "count": 130
+          },
+          "SS": {
+            "accuracy": 0.9885057471264368,
+            "count": 87
+          },
+          "UC": {
+            "accuracy": 0.9408866995073891,
+            "count": 203
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 117
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.55,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SC": {
+            "accuracy": 0.9917355371900827,
+            "count": 121
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 49
+          },
+          "UC": {
+            "accuracy": 0.7903225806451613,
+            "count": 186
+          },
+          "US": {
+            "accuracy": 0.9461883408071748,
+            "count": 223
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.4,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 104
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 106
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 23
+          },
+          "UC": {
+            "accuracy": 0.7,
+            "count": 160
+          },
+          "US": {
+            "accuracy": 0.8045602605863192,
+            "count": 307
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.27,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.41,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.59,
+            "count": 400
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.32,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.52,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.586,
+            "count": 500
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.955,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9977628635346756,
+            "count": 447
+          },
+          "SC": {
+            "accuracy": 0.996875,
+            "count": 320
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UC": {
+            "accuracy": 0.9867674858223062,
+            "count": 529
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 48
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.67,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8341968911917098,
+            "count": 193
+          },
+          "US": {
+            "accuracy": 0.9065420560747663,
+            "count": 107
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.63,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.86328125,
+            "count": 256
+          },
+          "US": {
+            "accuracy": 0.9097222222222222,
+            "count": 144
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.57,
+        "n_examples": 100,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.8725490196078431,
+            "count": 306
+          },
+          "US": {
+            "accuracy": 0.9020618556701031,
+            "count": 194
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.66,
+        "n_examples": 100,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UC": {
+            "accuracy": 0.9098360655737705,
+            "count": 366
+          },
+          "US": {
+            "accuracy": 0.9487179487179487,
+            "count": 234
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.9,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9850249584026622,
+            "count": 601
+          },
+          "ME": {
+            "accuracy": 0.98989898989899,
+            "count": 99
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 0.99,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 279
+          },
+          "MB": {
+            "accuracy": 0.993103448275862,
+            "count": 145
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 24
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 252
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.6,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9953051643192489,
+            "count": 213
+          },
+          "MB": {
+            "accuracy": 0.9734513274336283,
+            "count": 113
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 85
+          },
+          "UB": {
+            "accuracy": 0.7845303867403315,
+            "count": 181
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 108
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 179
+          },
+          "MB": {
+            "accuracy": 0.9805825242718447,
+            "count": 103
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          },
+          "UB": {
+            "accuracy": 0.4563758389261745,
+            "count": 149
+          },
+          "UD": {
+            "accuracy": 0.7699530516431925,
+            "count": 213
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.07,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.38,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.48,
+            "count": 300
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.02,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.26,
+            "count": 100
+          },
+          "UD": {
+            "accuracy": 0.38,
+            "count": 400
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.89,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 600
+          },
+          "MB": {
+            "accuracy": 0.9962546816479401,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 53
+          },
+          "UB": {
+            "accuracy": 0.9567198177676538,
+            "count": 439
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 41
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.68,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 300
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8578680203045685,
+            "count": 197
+          },
+          "UD": {
+            "accuracy": 0.8932038834951457,
+            "count": 103
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.51,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8380566801619433,
+            "count": 247
+          },
+          "UD": {
+            "accuracy": 0.7647058823529411,
+            "count": 153
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.33,
+        "n_examples": 100,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "UB": {
+            "accuracy": 0.8187919463087249,
+            "count": 298
+          },
+          "UD": {
+            "accuracy": 0.7524752475247525,
+            "count": 202
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.6116666666666667,
+      "total_examples": 2400,
+      "n_splits": 22
+    }
+  },
+  "sorl_overall_accuracy": 0.6116666666666667,
+  "sft_overall_accuracy": 0.47041666666666665
+}

add_sub_sorl_v1_abs10_K1_10K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b512b0970baed990d0164c674044de1463fd819596795fb6a137261185804ce5
+size 650303660

add_sub_sorl_v1_abs10_K1_10K/train_config.json

@@ -0,0 +1,84 @@
+{
+  "num_rollouts": 4,
+  "K": 1,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
+  "batch_size": 64,
+  "gradient_accumulation_steps": 1,
+  "num_epochs": 10,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
+  "output_dir": "ckpt/sweep/as_sorl_abs10_K1_10K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 10,
+  "dataset_size": 10000,
+  "mode": "sorl",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162499262,
+  "run_name": "add_sub_sorl_v1_abs10_K1_10K",
+  "git_commit": "78d46f8665a87f4b44bd5894bd34f393f2dea51f",
+  "timestamp": "2026-04-12T08:59:10.916999+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "v1",
+  "wandb_run_id": "pdbywwrx",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/pdbywwrx",
+  "final_accuracy": 0.6116666666666667,
+  "sft_accuracy": 0.47041666666666665,
+  "eval_method": "ArithmeticEvaluator"
+}