Upload add_sub_baseline_10K

add_sub_baseline_10K/config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "SorlModelWrapper"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 510,
+  "initializer_range": 0.02,
+  "intermediate_size": 2040,
+  "layer_types": [
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 3,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151645
+}

add_sub_baseline_10K/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true
+}

add_sub_baseline_10K/metrics.json

@@ -0,0 +1,831 @@
+{
+  "history": {
+    "step": [
+      50,
+      100,
+      150,
+      200,
+      250,
+      300,
+      350,
+      400,
+      450,
+      500,
+      550,
+      600,
+      650,
+      700,
+      750,
+      800,
+      850,
+      900,
+      950,
+      1000,
+      1050,
+      1100,
+      1150,
+      1200,
+      1250,
+      1300,
+      1350,
+      1400,
+      1450,
+      1500,
+      1550,
+      1600,
+      1650,
+      1700,
+      1750,
+      1800,
+      1850,
+      1900,
+      1950,
+      2000,
+      2050,
+      2100,
+      2150,
+      2200,
+      2250,
+      2300,
+      2350,
+      2400,
+      2450,
+      2500,
+      2550,
+      2600,
+      2650,
+      2700,
+      2750,
+      2800,
+      2850,
+      2900,
+      2950,
+      3000,
+      3050,
+      3100
+    ],
+    "loss": [
+      10.006766319274902,
+      7.251689434051514,
+      6.329094886779785,
+      5.457243919372559,
+      4.436343669891357,
+      3.0709240436553955,
+      2.1984894275665283,
+      1.9786336421966553,
+      1.8310412168502808,
+      1.7493754625320435,
+      1.7336280345916748,
+      1.668373942375183,
+      1.656901478767395,
+      1.5222007036209106,
+      1.1166151762008667,
+      0.8200947642326355,
+      0.5962659120559692,
+      0.4486698508262634,
+      0.3298042118549347,
+      0.21894291043281555,
+      0.1570301353931427,
+      0.12834718823432922,
+      0.09281289577484131,
+      0.0865481048822403,
+      0.10090114921331406,
+      0.05000796541571617,
+      0.05903910472989082,
+      0.041255030781030655,
+      0.03712769225239754,
+      0.05814942717552185,
+      0.04444698244333267,
+      0.03615949675440788,
+      0.027622858062386513,
+      0.04681888222694397,
+      0.06002098321914673,
+      0.049318160861730576,
+      0.059921007603406906,
+      0.032836705446243286,
+      0.061555344611406326,
+      0.03171056881546974,
+      0.04210656136274338,
+      0.04413192719221115,
+      0.047493066638708115,
+      0.050301071256399155,
+      0.02659599296748638,
+      0.022891702130436897,
+      0.05459100008010864,
+      0.017184091731905937,
+      0.013001530431210995,
+      0.03498239442706108,
+      0.02405695430934429,
+      0.03834375739097595,
+      0.03403574228286743,
+      0.014610671438276768,
+      0.044524967670440674,
+      0.017222251743078232,
+      0.03136735409498215,
+      0.02747531048953533,
+      0.030222097411751747,
+      0.02000907063484192,
+      0.02265077270567417,
+      0.026362445205450058
+    ],
+    "base_loss": [
+      10.006766319274902,
+      7.251689434051514,
+      6.329094886779785,
+      5.457243919372559,
+      4.436343669891357,
+      3.0709240436553955,
+      2.1984894275665283,
+      1.9786336421966553,
+      1.8310412168502808,
+      1.7493754625320435,
+      1.7336280345916748,
+      1.668373942375183,
+      1.656901478767395,
+      1.5222007036209106,
+      1.1166151762008667,
+      0.8200947642326355,
+      0.5962659120559692,
+      0.4486698508262634,
+      0.3298042118549347,
+      0.21894291043281555,
+      0.1570301353931427,
+      0.12834718823432922,
+      0.09281289577484131,
+      0.0865481048822403,
+      0.10090114921331406,
+      0.05000796541571617,
+      0.05903910472989082,
+      0.041255030781030655,
+      0.03712769225239754,
+      0.05814942717552185,
+      0.04444698244333267,
+      0.03615949675440788,
+      0.027622858062386513,
+      0.04681888222694397,
+      0.06002098321914673,
+      0.049318160861730576,
+      0.059921007603406906,
+      0.032836705446243286,
+      0.061555344611406326,
+      0.03171056881546974,
+      0.04210656136274338,
+      0.04413192719221115,
+      0.047493066638708115,
+      0.050301071256399155,
+      0.02659599296748638,
+      0.022891702130436897,
+      0.05459100008010864,
+      0.017184091731905937,
+      0.013001530431210995,
+      0.03498239442706108,
+      0.02405695430934429,
+      0.03834375739097595,
+      0.03403574228286743,
+      0.014610671438276768,
+      0.044524967670440674,
+      0.017222251743078232,
+      0.03136735409498215,
+      0.02747531048953533,
+      0.030222097411751747,
+      0.02000907063484192,
+      0.02265077270567417,
+      0.026362445205450058
+    ],
+    "lr": [
+      6.242038216560511e-06,
+      1.2611464968152866e-05,
+      1.8980891719745225e-05,
+      2.5350318471337578e-05,
+      3.1719745222929934e-05,
+      3.808917197452229e-05,
+      4.445859872611465e-05,
+      5.082802547770701e-05,
+      5.7197452229299365e-05,
+      6.356687898089173e-05,
+      6.993630573248408e-05,
+      7.630573248407644e-05,
+      7.99862055592881e-05,
+      7.984241248831029e-05,
+      7.954287783192742e-05,
+      7.90887724530305e-05,
+      7.848187142213441e-05,
+      7.772454707873448e-05,
+      7.681975975797462e-05,
+      7.57710462188759e-05,
+      7.458250581935905e-05,
+      7.325878449210182e-05,
+      7.180505658386849e-05,
+      7.022700462930083e-05,
+      6.853079713823312e-05,
+      6.672306448335957e-05,
+      6.481087298250779e-05,
+      6.280169727682872e-05,
+      6.070339111287581e-05,
+      5.8524156642783655e-05,
+      5.627251236255051e-05,
+      5.3957259813751526e-05,
+      5.1587449178844164e-05,
+      4.917234390455111e-05,
+      4.672138449160635e-05,
+      4.424415159240753e-05,
+      4.175032856082417e-05,
+      3.924966360055181e-05,
+      3.675193165997228e-05,
+      3.42668962224704e-05,
+      3.180427114156694e-05,
+      2.9373682670051437e-05,
+      2.6984631831541183e-05,
+      2.4646457281553407e-05,
+      2.2368298803264487e-05,
+      2.0159061580649347e-05,
+      1.8027381388654794e-05,
+      1.5981590836476463e-05,
+      1.4029686795892575e-05,
+      1.2179299141974771e-05,
+      1.0437660928367057e-05,
+      8.811580113715755e-06,
+      7.307412949770034e-06,
+      5.9310391351775455e-06,
+      4.687838832097362e-06,
+      3.5826716354707645e-06,
+      2.6198575771580583e-06,
+      1.8031602391947344e-06,
+      1.1357720421765062e-06,
+      6.203017662798872e-07,
+      2.5876435369797334e-07,
+      5.257303235302935e-08
+    ],
+    "eval_step": [
+      156,
+      312,
+      468,
+      624,
+      780,
+      936,
+      1092,
+      1248,
+      1404,
+      1560,
+      1716,
+      1872,
+      2028,
+      2184,
+      2340,
+      2496,
+      2652,
+      2808,
+      2964,
+      3120
+    ],
+    "eval_epoch": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20
+    ],
+    "eval_accuracy": [
+      0.0014285714285714286,
+      0.012142857142857143,
+      0.004285714285714286,
+      0.007142857142857143,
+      0.035,
+      0.26285714285714284,
+      0.6278571428571429,
+      0.6864285714285714,
+      0.6885714285714286,
+      0.7314285714285714,
+      0.7442857142857143,
+      0.73,
+      0.735,
+      0.75,
+      0.7807142857142857,
+      0.7964285714285714,
+      0.8185714285714286,
+      0.8185714285714286,
+      0.8221428571428572,
+      0.8235714285714286
+    ]
+  },
+  "final_accuracy": 0.7609259259259259,
+  "sft_eval": {
+    "config": {
+      "ops": "add_sub",
+      "K": null,
+      "mode": "sft",
+      "n_digits": 6,
+      "n_per_split": 250
+    },
+    "splits": {
+      "add_S0": {
+        "full_accuracy": 0.988,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9980302035456337,
+            "count": 1523
+          },
+          "SS": {
+            "accuracy": 0.9955947136563876,
+            "count": 227
+          }
+        }
+      },
+      "add_S1": {
+        "full_accuracy": 0.996,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 542
+          },
+          "SC": {
+            "accuracy": 0.9976019184652278,
+            "count": 417
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 70
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 721
+          }
+        }
+      },
+      "add_S2": {
+        "full_accuracy": 0.968,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 368
+          },
+          "SC": {
+            "accuracy": 0.9750778816199377,
+            "count": 321
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 228
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 531
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 302
+          }
+        }
+      },
+      "add_S3": {
+        "full_accuracy": 0.74,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 307
+          },
+          "SC": {
+            "accuracy": 0.9965635738831615,
+            "count": 291
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 113
+          },
+          "UC": {
+            "accuracy": 0.8674948240165632,
+            "count": 483
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 556
+          }
+        }
+      },
+      "add_S4": {
+        "full_accuracy": 0.704,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 238
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 271
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 59
+          },
+          "UC": {
+            "accuracy": 0.8395061728395061,
+            "count": 405
+          },
+          "US": {
+            "accuracy": 0.9317889317889317,
+            "count": 777
+          }
+        }
+      },
+      "add_S5": {
+        "full_accuracy": 0.5,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UC": {
+            "accuracy": 0.644,
+            "count": 250
+          },
+          "US": {
+            "accuracy": 0.795,
+            "count": 1000
+          }
+        }
+      },
+      "add_S6": {
+        "full_accuracy": 0.916,
+        "n_examples": 250,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "US": {
+            "accuracy": 0.9824,
+            "count": 1250
+          }
+        }
+      },
+      "add_random": {
+        "full_accuracy": 0.985,
+        "n_examples": 200,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.9977324263038548,
+            "count": 441
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 317
+          },
+          "SS": {
+            "accuracy": 1.0,
+            "count": 54
+          },
+          "UC": {
+            "accuracy": 0.9962406015037594,
+            "count": 532
+          },
+          "US": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
+          }
+        }
+      },
+      "add_C3": {
+        "full_accuracy": 0.812,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 750
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UC": {
+            "accuracy": 0.9012605042016807,
+            "count": 476
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 274
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.888,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 500
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UC": {
+            "accuracy": 0.9606656580937972,
+            "count": 661
+          },
+          "US": {
+            "accuracy": 0.967551622418879,
+            "count": 339
+          }
+        }
+      },
+      "add_C5": {
+        "full_accuracy": 0.812,
+        "n_examples": 250,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UC": {
+            "accuracy": 0.9385245901639344,
+            "count": 732
+          },
+          "US": {
+            "accuracy": 0.9498069498069498,
+            "count": 518
+          }
+        }
+      },
+      "add_C6": {
+        "full_accuracy": 0.824,
+        "n_examples": 250,
+        "per_subtask": {
+          "SC": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UC": {
+            "accuracy": 0.954954954954955,
+            "count": 888
+          },
+          "US": {
+            "accuracy": 0.9624183006535948,
+            "count": 612
+          }
+        }
+      },
+      "sub_M0": {
+        "full_accuracy": 0.968,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9946843853820598,
+            "count": 1505
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 245
+          }
+        }
+      },
+      "sub_M1": {
+        "full_accuracy": 1.0,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 714
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 374
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 75
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 587
+          }
+        }
+      },
+      "sub_M2": {
+        "full_accuracy": 0.996,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 0.9981949458483754,
+            "count": 554
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 273
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 219
+          },
+          "UB": {
+            "accuracy": 1.0,
+            "count": 430
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 274
+          }
+        }
+      },
+      "sub_M3": {
+        "full_accuracy": 0.34,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 458
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 261
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 124
+          },
+          "UB": {
+            "accuracy": 0.5747422680412371,
+            "count": 388
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 519
+          }
+        }
+      },
+      "sub_M4": {
+        "full_accuracy": 0.116,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 500
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UB": {
+            "accuracy": 0.312,
+            "count": 250
+          },
+          "UD": {
+            "accuracy": 0.7466666666666667,
+            "count": 750
+          }
+        }
+      },
+      "sub_M5": {
+        "full_accuracy": 0.052,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UB": {
+            "accuracy": 0.368,
+            "count": 250
+          },
+          "UD": {
+            "accuracy": 0.547,
+            "count": 1000
+          }
+        }
+      },
+      "sub_random": {
+        "full_accuracy": 0.99,
+        "n_examples": 200,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 580
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 267
+          },
+          "ME": {
+            "accuracy": 1.0,
+            "count": 63
+          },
+          "UB": {
+            "accuracy": 0.9955357142857143,
+            "count": 448
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 42
+          }
+        }
+      },
+      "sub_B3": {
+        "full_accuracy": 0.868,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 750
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UB": {
+            "accuracy": 0.9350393700787402,
+            "count": 508
+          },
+          "UD": {
+            "accuracy": 1.0,
+            "count": 242
+          }
+        }
+      },
+      "sub_B4": {
+        "full_accuracy": 0.716,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 500
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UB": {
+            "accuracy": 0.8973941368078175,
+            "count": 614
+          },
+          "UD": {
+            "accuracy": 0.9300518134715026,
+            "count": 386
+          }
+        }
+      },
+      "sub_B5": {
+        "full_accuracy": 0.652,
+        "n_examples": 250,
+        "per_subtask": {
+          "MD": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "MB": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "UB": {
+            "accuracy": 0.8957528957528957,
+            "count": 777
+          },
+          "UD": {
+            "accuracy": 0.9090909090909091,
+            "count": 473
+          }
+        }
+      }
+    },
+    "summary": {
+      "overall_accuracy": 0.7609259259259259,
+      "total_examples": 5400,
+      "n_splits": 22
+    }
+  }
+}

add_sub_baseline_10K/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd67ff426b2d74cb962e75cb59ccc6e89358ef5a3fa76bfe35640b18cc9c69ee
+size 650266922

add_sub_baseline_10K/train_config.json

@@ -0,0 +1,35 @@
+{
+  "mode": "baseline",
+  "ops": "add_sub",
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "abs_vocab": 0,
+  "K": 4,
+  "alpha_info_gain": 10.0,
+  "alpha_abs": 0.1,
+  "alpha_soft_zipf": 1.0,
+  "batch_size": 64,
+  "num_epochs": 20,
+  "dataset_size": 10000,
+  "lr": 8e-05,
+  "output_dir": "ckpt/sweep/add_sub_baseline_10K",
+  "device": "cuda",
+  "push_to_hub": true,
+  "no_wandb": false,
+  "n_params": 162490082,
+  "run_name": "add_sub_baseline_10K",
+  "git_commit": "800625019270114adcda289bbd550c4f1109a514",
+  "timestamp": "2026-04-11T21:33:55.773262+00:00",
+  "tokenizer": "Qwen/Qwen3-0.6B",
+  "dataset_repo": "thoughtworks/arithmetic-sorl-data",
+  "dataset_config": "add_sub_6digit",
+  "model_repo": "thoughtworks/arithmetic-sorl",
+  "trainer_version": "sft",
+  "wandb_run_id": "1e2dttjb",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/1e2dttjb",
+  "final_accuracy": 0.7609259259259259,
+  "sft_accuracy": 0.7609259259259259,
+  "eval_method": "ArithmeticEvaluator"
+}