run directory (provenance + evals)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+run/eval_m0.jsonl filter=lfs diff=lfs merge=lfs -text

run/config.yaml

@@ -28,7 +28,7 @@ steered_frac: 0.7
 eval_question_frac: 0.2
 repeats_per_question: 6
 alpaca_replay_frac: 0.5
-eval_concepts_per_split: 40    # seeded subsample per split for the eval sweep
+eval_concepts_per_split: {train: 40, heldout: null}
 
 # Training
 lora_r: 16
@@ -54,7 +54,7 @@ capability_limit: "mmlu=15,gsm8k_cot=200"
 # Tracking & backup — hub_repo_id set by scripts/runpod.sh at run time
 wandb_project: null
 wandb_entity: null
-hub_repo_id: JacoDuToit/steer-full_3b
+hub_repo_id: null
 hub_private: true
 
 # Paths

run/invocations.jsonl

@@ -1 +1,3 @@
 {"time": "2026-07-23T16:19:29+0000", "stages": ["vectors", "data", "eval_m0", "train", "eval_m1"], "commit": "none", "status": "success", "headline": "clean 100%->100% · steer_heldout@1.6 correct 0%->3%", "wandb_url": null}
+{"time": "2026-07-25T11:36:44+0000", "stages": ["eval_m0"], "commit": "none", "status": "success", "headline": "", "wandb_url": null}
+{"time": "2026-07-25T12:19:58+0000", "stages": ["eval_m1"], "commit": "none", "status": "success", "headline": "clean 100%->100% · steer_heldout@1.6 correct 0%->3%", "wandb_url": null}

run/run_meta.json

@@ -1,15 +1,13 @@
 {
  "run": "full_3b",
- "started_at": "2026-07-23T16:19:29+0000",
+ "started_at": "2026-07-25T12:19:58+0000",
  "argv": [
   "scripts/run.py",
-  "configs/full_3b.yaml"
+  "configs/full_3b.yaml",
+  "--stages",
+  "eval_m1"
  ],
  "stages": [
-  "vectors",
-  "data",
-  "eval_m0",
-  "train",
   "eval_m1"
  ],
  "config_path": "configs/full_3b.yaml",
@@ -42,7 +40,10 @@
   "eval_question_frac": 0.2,
   "repeats_per_question": 6,
   "alpaca_replay_frac": 0.5,
-  "eval_concepts_per_split": 40,
+  "eval_concepts_per_split": {
+   "train": 40,
+   "heldout": null
+  },
   "lora_r": 16,
   "lora_alpha": 32,
   "lora_dropout": 0.05,
@@ -71,7 +72,7 @@
   "capability_limit": "mmlu=15,gsm8k_cot=200",
   "wandb_project": null,
   "wandb_entity": null,
-  "hub_repo_id": "JacoDuToit/steer-full_3b",
+  "hub_repo_id": null,
   "hub_private": true,
   "vectors_path": "results/full_3b/vectors.pt",
   "train_examples_path": "results/full_3b/train_examples.json",
@@ -87,10 +88,10 @@
  },
  "env": {
   "python": "3.11.10",
-  "platform": "Linux-6.8.0-52-generic-x86_64-with-glibc2.35",
-  "accelerator": "NVIDIA RTX A4000",
+  "platform": "Linux-6.8.0-49-generic-x86_64-with-glibc2.35",
+  "accelerator": "NVIDIA GeForce RTX 3090",
   "packages": {
-   "torch": "2.6.0+cu124",
+   "torch": "2.13.0+cu126",
    "transformers": "5.14.1",
    "peft": "0.19.1",
    "accelerate": "1.14.0",
@@ -113,25 +114,48 @@
  "status": "success",
  "wandb_url": null,
  "headline": "clean 100%->100% · steer_heldout@1.6 correct 0%->3%",
- "hub_url": "https://huggingface.co/JacoDuToit/steer-full_3b",
- "finished_at": "2026-07-23T17:41:53+0000",
+ "finished_at": "2026-07-25T13:02:30+0000",
  "artifacts": {
+  "capability/capability_summary.csv": {
+   "sha256": "d5bbb31095b80e60dd46d2e5cdaa7ef2a781ccc89de5766381b21af1425233b9",
+   "bytes": 128
+  },
+  "capability/capability_summary.md": {
+   "sha256": "ebf7fa98749913e9f77670b2f926ead49fd2dfdf3cacdafbc27b873e088a7218",
+   "bytes": 363
+  },
+  "capability/lmeval_run.log": {
+   "sha256": "f3d93e78d2473755612d80a0afb6eaec8ed8027f13b4545ee8ec4d59370d4731",
+   "bytes": 298468
+  },
+  "capability/m0/gsm8k_cot/Qwen__Qwen2.5-3B-Instruct/results_2026-07-23T18-36-29.229735.json": {
+   "sha256": "fa3feefea52e589b23d972c3393a9222b7affa81d04b782a111cbfb5f6e561bc",
+   "bytes": 14432
+  },
+  "capability/m0/mmlu/Qwen__Qwen2.5-3B-Instruct/results_2026-07-23T18-15-57.670538.json": {
+   "sha256": "dfd686f19edce4cb31bffd091167c599b0596b15987d25303613295099cb4562",
+   "bytes": 139207
+  },
+  "capability/m1/gsm8k_cot/__root__steering-resistance__results__full_3b__m1_resist_adapter/results_2026-07-23T19-07-18.393003.json": {
+   "sha256": "35a97896c6938ea0031d99b63a6787f635f4d88b68a52c7aec53a1849b28a25c",
+   "bytes": 14681
+  },
+  "capability/m1/mmlu/__root__steering-resistance__results__full_3b__m1_resist_adapter/results_2026-07-23T18-41-47.488437.json": {
+   "sha256": "e6e525c0832477c3729aca9663ded0bc63ab583d6c521a550365fb8656f1fc0c",
+   "bytes": 143701
+  },
   "eval_m0.jsonl": {
-   "sha256": "104de23c0313f90e885f92c12006dca7433350b6725f369622d5d2531210b27c",
-   "bytes": 4576432
+   "sha256": "1ae0cbda4668a35852e1ef234043f320d320ae9cb7cf4017b4d2f79f1427605a",
+   "bytes": 11318115
   },
   "eval_m1.jsonl": {
-   "sha256": "03ae380b0768f511b1bedafce30c516daa4a10b71aa9a1bb0a8113300fca4469",
-   "bytes": 3877293
+   "sha256": "1cadc857be9f99e14b2a7c6293f759e7f8f47a0ca1a78e684ab321339f97eb36",
+   "bytes": 9612527
   },
   "eval_questions.json": {
    "sha256": "8771796f4c91a61a901278e799032623900657aaa8f9e44ce789dd1fadd8bf8f",
    "bytes": 27617
   },
-  "m1_resist_adapter/README.md": {
-   "sha256": "3581cc6c99fe8b3ab31a0fc4a131726f5d714856745f1598891a0151c2540256",
-   "bytes": 5202
-  },
   "m1_resist_adapter/adapter_config.json": {
    "sha256": "390e661694a2f685b6256b5d38a80aa41c5236a115df9cfb5edd4a96da0e568e",
    "bytes": 1103
@@ -141,12 +165,12 @@
    "bytes": 119801528
   },
   "summary.csv": {
-   "sha256": "8de370b8f4acdc3b46173f4bb76c96185e27f8bffc117c4585e8efb4966e834e",
-   "bytes": 2607
+   "sha256": "0d7959f18ba4548ff986d2f9d1d951434a73addb3bafd0822e6153011850a2c3",
+   "bytes": 2621
   },
   "summary.md": {
-   "sha256": "df82f8ebb8f8c63fae08a18eb0be3ef4805052dff62bc7e812d5953600f2febe",
-   "bytes": 2617
+   "sha256": "cbc9c4aa3d4576d1608186f9e893ae0fc1a94d68655f57ec5126e29b756989c1",
+   "bytes": 2627
   },
   "train_examples.json": {
    "sha256": "3e2ac73debbb5da8cadb4e114a35f6346f015ca0f22d52b9486a7fd2b2b7e824",

run/summary.csv

@@ -5,11 +5,11 @@ M0,correct_inject,0.8,9,0.8889,0.6667,1.0,0.0,0.0,0.0,0.1111,0.0,0.3333
 M0,correct_inject,1.0,9,0.8889,0.6667,1.0,0.0,0.0,0.0,0.1111,0.0,0.3333
 M0,correct_inject,1.2,9,0.8889,0.6667,1.0,0.0,0.0,0.0,0.1111,0.0,0.3333
 M0,correct_inject,1.6,9,0.8889,0.6667,1.0,0.0,0.0,0.0,0.1111,0.0,0.3333
-M0,steer_heldout,0.4,680,0.8338,0.7544,0.8985,0.1044,0.0706,0.1382,0.0618,0.0176,0.1191
-M0,steer_heldout,0.8,680,0.1529,0.1103,0.2,0.7044,0.6662,0.7441,0.1426,0.1206,0.1632
-M0,steer_heldout,1.0,680,0.0162,0.0059,0.0265,0.7882,0.775,0.8015,0.1956,0.1824,0.2088
-M0,steer_heldout,1.2,680,0.0015,0.0,0.0044,0.7412,0.7309,0.7515,0.2574,0.2456,0.2691
-M0,steer_heldout,1.6,680,0.0,0.0,0.0,0.5074,0.4985,0.5162,0.4926,0.4838,0.5015
+M0,steer_heldout,0.4,2754,0.8362,0.7698,0.8903,0.0926,0.0668,0.1191,0.0712,0.0363,0.1198
+M0,steer_heldout,0.8,2754,0.1358,0.098,0.1747,0.7331,0.6943,0.7709,0.1311,0.1191,0.1431
+M0,steer_heldout,1.0,2754,0.0149,0.0073,0.0236,0.8115,0.7963,0.8261,0.1736,0.1627,0.1845
+M0,steer_heldout,1.2,2754,0.0011,0.0,0.0022,0.7404,0.7317,0.7491,0.2585,0.2498,0.2672
+M0,steer_heldout,1.6,2754,0.0,0.0,0.0,0.5236,0.5167,0.5305,0.4764,0.4695,0.4833
 M0,steer_train,0.4,679,0.9264,0.8351,0.9838,0.0029,0.0,0.0088,0.0707,0.0162,0.1561
 M0,steer_train,0.8,679,0.1649,0.1163,0.2147,0.7334,0.6868,0.7817,0.1016,0.0795,0.1265
 M0,steer_train,1.0,679,0.0162,0.0074,0.0266,0.8733,0.8527,0.8925,0.1105,0.0956,0.1279
@@ -21,11 +21,11 @@ M1,correct_inject,0.8,9,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0
 M1,correct_inject,1.0,9,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0
 M1,correct_inject,1.2,9,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0
 M1,correct_inject,1.6,9,0.8889,0.6667,1.0,0.0,0.0,0.0,0.1111,0.0,0.3333
-M1,steer_heldout,0.4,680,0.9676,0.9265,0.9926,0.0132,0.0059,0.0206,0.0191,0.0,0.0544
-M1,steer_heldout,0.8,680,0.6691,0.5794,0.7397,0.2191,0.1868,0.2544,0.1118,0.0662,0.1721
-M1,steer_heldout,1.0,680,0.4338,0.3382,0.525,0.3515,0.2853,0.425,0.2147,0.1588,0.2721
-M1,steer_heldout,1.2,680,0.2515,0.1691,0.3353,0.4235,0.3574,0.4926,0.325,0.2691,0.3824
-M1,steer_heldout,1.6,680,0.0309,0.0132,0.05,0.5574,0.5191,0.5926,0.4118,0.3868,0.4368
+M1,steer_heldout,0.4,2754,0.9713,0.9259,0.996,0.0065,0.0033,0.0105,0.0221,0.0,0.0654
+M1,steer_heldout,0.8,2754,0.6598,0.5817,0.7233,0.2052,0.1743,0.2433,0.1351,0.094,0.1888
+M1,steer_heldout,1.0,2754,0.4365,0.3475,0.5225,0.3424,0.276,0.4161,0.2211,0.1772,0.2716
+M1,steer_heldout,1.2,2754,0.2495,0.1667,0.3337,0.4205,0.3475,0.4967,0.3301,0.2818,0.3856
+M1,steer_heldout,1.6,2754,0.0334,0.0123,0.0577,0.5512,0.5153,0.5835,0.4154,0.3976,0.435
 M1,steer_train,0.4,679,0.9691,0.9088,1.0,0.0015,0.0,0.0044,0.0295,0.0,0.0885
 M1,steer_train,0.8,679,0.8733,0.7997,0.9366,0.0412,0.0132,0.0781,0.0854,0.0442,0.1355
 M1,steer_train,1.0,679,0.6082,0.4801,0.7338,0.2165,0.1193,0.3245,0.1753,0.1134,0.243

run/summary.md

@@ -6,11 +6,11 @@
 | M0 | correct_inject | 1.0 | 9 | 89% [67%,100%] | 0% [0%,0%] | 11% [0%,33%] |
 | M0 | correct_inject | 1.2 | 9 | 89% [67%,100%] | 0% [0%,0%] | 11% [0%,33%] |
 | M0 | correct_inject | 1.6 | 9 | 89% [67%,100%] | 0% [0%,0%] | 11% [0%,33%] |
-| M0 | steer_heldout | 0.4 | 680 | 83% [75%,90%] | 10% [7%,14%] | 6% [2%,12%] |
-| M0 | steer_heldout | 0.8 | 680 | 15% [11%,20%] | 70% [67%,74%] | 14% [12%,16%] |
-| M0 | steer_heldout | 1.0 | 680 | 2% [1%,3%] | 79% [78%,80%] | 20% [18%,21%] |
-| M0 | steer_heldout | 1.2 | 680 | 0% [0%,0%] | 74% [73%,75%] | 26% [25%,27%] |
-| M0 | steer_heldout | 1.6 | 680 | 0% [0%,0%] | 51% [50%,52%] | 49% [48%,50%] |
+| M0 | steer_heldout | 0.4 | 2754 | 84% [77%,89%] | 9% [7%,12%] | 7% [4%,12%] |
+| M0 | steer_heldout | 0.8 | 2754 | 14% [10%,17%] | 73% [69%,77%] | 13% [12%,14%] |
+| M0 | steer_heldout | 1.0 | 2754 | 1% [1%,2%] | 81% [80%,83%] | 17% [16%,18%] |
+| M0 | steer_heldout | 1.2 | 2754 | 0% [0%,0%] | 74% [73%,75%] | 26% [25%,27%] |
+| M0 | steer_heldout | 1.6 | 2754 | 0% [0%,0%] | 52% [52%,53%] | 48% [47%,48%] |
 | M0 | steer_train | 0.4 | 679 | 93% [84%,98%] | 0% [0%,1%] | 7% [2%,16%] |
 | M0 | steer_train | 0.8 | 679 | 16% [12%,21%] | 73% [69%,78%] | 10% [8%,13%] |
 | M0 | steer_train | 1.0 | 679 | 2% [1%,3%] | 87% [85%,89%] | 11% [10%,13%] |
@@ -22,11 +22,11 @@
 | M1 | correct_inject | 1.0 | 9 | 100% [100%,100%] | 0% [0%,0%] | 0% [0%,0%] |
 | M1 | correct_inject | 1.2 | 9 | 100% [100%,100%] | 0% [0%,0%] | 0% [0%,0%] |
 | M1 | correct_inject | 1.6 | 9 | 89% [67%,100%] | 0% [0%,0%] | 11% [0%,33%] |
-| M1 | steer_heldout | 0.4 | 680 | 97% [93%,99%] | 1% [1%,2%] | 2% [0%,5%] |
-| M1 | steer_heldout | 0.8 | 680 | 67% [58%,74%] | 22% [19%,25%] | 11% [7%,17%] |
-| M1 | steer_heldout | 1.0 | 680 | 43% [34%,52%] | 35% [29%,42%] | 21% [16%,27%] |
-| M1 | steer_heldout | 1.2 | 680 | 25% [17%,34%] | 42% [36%,49%] | 32% [27%,38%] |
-| M1 | steer_heldout | 1.6 | 680 | 3% [1%,5%] | 56% [52%,59%] | 41% [39%,44%] |
+| M1 | steer_heldout | 0.4 | 2754 | 97% [93%,100%] | 1% [0%,1%] | 2% [0%,7%] |
+| M1 | steer_heldout | 0.8 | 2754 | 66% [58%,72%] | 21% [17%,24%] | 14% [9%,19%] |
+| M1 | steer_heldout | 1.0 | 2754 | 44% [35%,52%] | 34% [28%,42%] | 22% [18%,27%] |
+| M1 | steer_heldout | 1.2 | 2754 | 25% [17%,33%] | 42% [35%,50%] | 33% [28%,39%] |
+| M1 | steer_heldout | 1.6 | 2754 | 3% [1%,6%] | 55% [52%,58%] | 42% [40%,44%] |
 | M1 | steer_train | 0.4 | 679 | 97% [91%,100%] | 0% [0%,0%] | 3% [0%,9%] |
 | M1 | steer_train | 0.8 | 679 | 87% [80%,94%] | 4% [1%,8%] | 9% [4%,14%] |
 | M1 | steer_train | 1.0 | 679 | 61% [48%,73%] | 22% [12%,32%] | 18% [11%,24%] |