Upload add_sub_baseline_10K

add_sub_baseline_10K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151645
+}

add_sub_baseline_10K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_baseline_10K/metrics.json

@@ -0,0 +1,831 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1600,
+      1650,
+      1700,
+      1750,
+      1800,
+      1850,
+      1900,
+      1950,
+      2000,
+      2050,
+      2100,
+      2150,
+      2200,
+      2250,
+      2300,
+      2350,
+      2400,
+      2450,
+      2500,
+      2550,
+      2600,
+      2650,
+      2700,
+      2750,
+      2800,
+      2850,
+      2900,
+      2950,
+      3000,
+      3050,
+      3100
+    ],
+    "loss": [
+      10.090532302856445,
+      7.41793155670166,
+      6.670701503753662,
+      5.4355902671813965,
+      4.486183166503906,
+      3.106839418411255,
+      2.181303024291992,
+      1.9472637176513672,
+      1.6879997253417969,
+      1.7242902517318726,
+      1.3950544595718384,
+      0.9311097860336304,
+      0.593478798866272,
+      0.4981410801410675,
+      0.24719169735908508,
+      0.10993476957082748,
+      0.09566618502140045,
+      0.08340698480606079,
+      0.07292894273996353,
+      0.05232112854719162,
+      0.04302288219332695,
+      0.04004356637597084,
+      0.04495292156934738,
+      0.048470307141542435,
+      0.033653225749731064,
+      0.030521346256136894,
+      0.06012653931975365,
+      0.05565373972058296,
+      0.04759713634848595,
+      0.054727476090192795,
+      0.034593965858221054,
+      0.027987422421574593,
+      0.034813184291124344,
+      0.03215349465608597,
+      0.02478446625173092,
+      0.022883405908942223,
+      0.02932637929916382,
+      0.017739301547408104,
+      0.024523159489035606,
+      0.02478833682835102,
+      0.01739186979830265,
+      0.01981291174888611,
+      0.038495369255542755,
+      0.02020339109003544,
+      0.035459987819194794,
+      0.033973682671785355,
+      0.03820496052503586,
+      0.01859247498214245,
+      0.02102138102054596,
+      0.021527009084820747,
+      0.02982836402952671,
+      0.026523862034082413,
+      0.0283384770154953,
+      0.023189542815089226,
+      0.022154804319143295,
+      0.02203175611793995,
+      0.01161875855177641,
+      0.01782594434916973,
+      0.014171228744089603,
+      0.014905942603945732,
+      0.019558599218726158,
+      0.012723379768431187
+    ],
+    "base_loss": [
+      10.090532302856445,
+      7.41793155670166,
+      6.670701503753662,
+      5.4355902671813965,
+      4.486183166503906,
+      3.106839418411255,
+      2.181303024291992,
+      1.9472637176513672,
+      1.6879997253417969,
+      1.7242902517318726,
+      1.3950544595718384,
+      0.9311097860336304,
+      0.593478798866272,
+      0.4981410801410675,
+      0.24719169735908508,
+      0.10993476957082748,
+      0.09566618502140045,
+      0.08340698480606079,
+      0.07292894273996353,
+      0.05232112854719162,
+      0.04302288219332695,
+      0.04004356637597084,
+      0.04495292156934738,
+      0.048470307141542435,
+      0.033653225749731064,
+      0.030521346256136894,
+      0.06012653931975365,
+      0.05565373972058296,
+      0.04759713634848595,
+      0.054727476090192795,
+      0.034593965858221054,
+      0.027987422421574593,
+      0.034813184291124344,
+      0.03215349465608597,
+      0.02478446625173092,
+      0.022883405908942223,
+      0.02932637929916382,
+      0.017739301547408104,
+      0.024523159489035606,
+      0.02478833682835102,
+      0.01739186979830265,
+      0.01981291174888611,
+      0.038495369255542755,
+      0.02020339109003544,
+      0.035459987819194794,
+      0.033973682671785355,
+      0.03820496052503586,
+      0.01859247498214245,
+      0.02102138102054596,
+      0.021527009084820747,
+      0.02982836402952671,
+      0.026523862034082413,
+      0.0283384770154953,
+      0.023189542815089226,
+      0.022154804319143295,
+      0.02203175611793995,
+      0.01161875855177641,
+      0.01782594434916973,
+      0.014171228744089603,
+      0.014905942603945732,
+      0.019558599218726158,
+      0.012723379768431187
+    ],
+    "lr": [
+      6.242038216560511e-06,
+      1.2611464968152866e-05,
+      1.8980891719745225e-05,
+      2.5350318471337578e-05,
+      3.1719745222929934e-05,
+      3.808917197452229e-05,
+      4.445859872611465e-05,
+      5.082802547770701e-05,
+      5.7197452229299365e-05,
+      6.356687898089173e-05,
+      6.993630573248408e-05,
+      7.630573248407644e-05,
+      7.99862055592881e-05,
+      7.984241248831029e-05,
+      7.954287783192742e-05,
+      7.90887724530305e-05,
+      7.848187142213441e-05,
+      7.772454707873448e-05,
+      7.681975975797462e-05,
+      7.57710462188759e-05,
+      7.458250581935905e-05,
+      7.325878449210182e-05,
+      7.180505658386849e-05,
+      7.022700462930083e-05,
+      6.853079713823312e-05,
+      6.672306448335957e-05,
+      6.481087298250779e-05,
+      6.280169727682872e-05,
+      6.070339111287581e-05,
+      5.8524156642783655e-05,
+      5.627251236255051e-05,
+      5.3957259813751526e-05,
+      5.1587449178844164e-05,
+      4.917234390455111e-05,
+      4.672138449160635e-05,
+      4.424415159240753e-05,
+      4.175032856082417e-05,
+      3.924966360055181e-05,
+      3.675193165997228e-05,
+      3.42668962224704e-05,
+      3.180427114156694e-05,
+      2.9373682670051437e-05,
+      2.6984631831541183e-05,
+      2.4646457281553407e-05,
+      2.2368298803264487e-05,
+      2.0159061580649347e-05,
+      1.8027381388654794e-05,
+      1.5981590836476463e-05,
+      1.4029686795892575e-05,
+      1.2179299141974771e-05,
+      1.0437660928367057e-05,
+      8.811580113715755e-06,
+      7.307412949770034e-06,
+      5.9310391351775455e-06,
+      4.687838832097362e-06,
+      3.5826716354707645e-06,
+      2.6198575771580583e-06,
+      1.8031602391947344e-06,
+      1.1357720421765062e-06,
+      6.203017662798872e-07,
+      2.5876435369797334e-07,
+      5.257303235302935e-08
+    ],
+    "eval_step": [
+      156,
+      312,
+      468,
+      624,
+      780,
+      936,
+      1092,
+      1248,
+      1404,
+      1560,
+      1716,
+      1872,
+      2028,
+      2184,
+      2340,
+      2496,
+      2652,
+      2808,
+      2964,
+      3120
+    ],
+    "eval_epoch": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20
+    ],
+    "eval_accuracy": [
+      0.0,
+      0.0064285714285714285,
+      0.007142857142857143,
+      0.04,
+      0.6357142857142857,
+      0.7271428571428571,
+      0.7485714285714286,
+      0.7471428571428571,
+      0.75,
+      0.7878571428571428,
+      0.7842857142857143,
+      0.7714285714285715,
+      0.8028571428571428,
+      0.8085714285714286,
+      0.8228571428571428,
+      0.8342857142857143,
+      0.8307142857142857,
+      0.8635714285714285,
+      0.8814285714285715,
+      0.8628571428571429
+    ]
+  },
+  "final_accuracy": 0.8628571428571429,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 50
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 295
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 55
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 126
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 79
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 21
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 124
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.96,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 75
+          },
+          "SC": {
+            "accuracy": 0.967741935483871,
+            "count": 62
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 39
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 111
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 63
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.88,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 60
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 57
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 19
+          },
+          "UC": {
+            "accuracy": 0.9423076923076923,
+            "count": 104
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 110
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.7,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 48
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 52
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 7
+          },
+          "UC": {
+            "accuracy": 0.8314606741573034,
+            "count": 89
+          },
+          "US": {
+            "accuracy": 0.9025974025974026,
+            "count": 154
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.58,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.6,
+            "count": 50
+          },
+          "US": {
+            "accuracy": 0.79,
+            "count": 200
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 250
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.99,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 431
+          },
+          "SC": {
+            "accuracy": 0.9968354430379747,
+            "count": 316
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 39
+          },
+          "UC": {
+            "accuracy": 0.9982142857142857,
+            "count": 560
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 54
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.88,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 150
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9423076923076923,
+            "count": 104
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 46
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.86,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.943089430894309,
+            "count": 123
+          },
+          "US": {
+            "accuracy": 0.948051948051948,
+            "count": 77
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.86,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9545454545454546,
+            "count": 154
+          },
+          "US": {
+            "accuracy": 0.9895833333333334,
+            "count": 96
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.88,
+        "n_examples": 50,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.967032967032967,
+            "count": 182
+          },
+          "US": {
+            "accuracy": 0.940677966101695,
+            "count": 118
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 294
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 56
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 143
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 69
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 15
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 123
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 108
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 52
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 52
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 87
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 51
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.46,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 94
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 51
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 25
+          },
+          "UB": {
+            "accuracy": 0.6538461538461539,
+            "count": 78
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 102
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.48,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UB": {
+            "accuracy": 0.48,
+            "count": 50
+          },
+          "UD": {
+            "accuracy": 0.8333333333333334,
+            "count": 150
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.36,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UB": {
+            "accuracy": 0.44,
+            "count": 50
+          },
+          "UD": {
+            "accuracy": 0.69,
+            "count": 200
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.995,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 588
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 268
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 60
+          },
+          "UB": {
+            "accuracy": 0.9977628635346756,
+            "count": 447
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 37
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.92,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 150
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UB": {
+            "accuracy": 0.9626168224299065,
+            "count": 107
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 43
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.72,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 100
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UB": {
+            "accuracy": 0.8771929824561403,
+            "count": 114
+          },
+          "UD": {
+            "accuracy": 0.9302325581395349,
+            "count": 86
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.7,
+        "n_examples": 50,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UB": {
+            "accuracy": 0.9019607843137255,
+            "count": 153
+          },
+          "UD": {
+            "accuracy": 0.865979381443299,
+            "count": 97
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.8628571428571429,
+      "total_examples": 1400,
+      "n_splits": 22
+    }
+  }
+}

add_sub_baseline_10K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc2ceca0901e7a6c1a4fdcedf14bf615dda905c45631f688fe16182a3dabbdcd
+size 650266922

add_sub_baseline_10K/train_config.json

@@ -0,0 +1,35 @@
+{
+  "mode": "baseline",
+  "ops": "add_sub",
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "abs_vocab": 0,
+  "K": 4,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "batch_size": 64,
+  "num_epochs": 20,
+  "dataset_size": 10000,
+  "lr": 8e-05,
+  "output_dir": "ckpt/sweep/add_sub_baseline_10K",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162490082,
+  "run_name": "add_sub_baseline_10K",
+  "git_commit": "800625019270114adcda289bbd550c4f1109a514",
+  "timestamp": "2026-04-12T01:58:10.796905+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "sft",
+  "wandb_run_id": "04vy23fy",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/04vy23fy",
+  "final_accuracy": 0.8628571428571429,
+  "sft_accuracy": 0.8628571428571429,
+  "eval_method": "ArithmeticEvaluator"
+}