diff --git a/run/capability/capability_summary.csv b/run/capability/capability_summary.csv
new file mode 100644
index 0000000000000000000000000000000000000000..c88f1a3ea03784497d381d671321b3132d2434a3
--- /dev/null
+++ b/run/capability/capability_summary.csv
@@ -0,0 +1,3 @@
+benchmark,metric,m0,m1,delta_pp
+mmlu,"acc,none",75.09,76.49,+1.40
+gsm8k_cot,"exact_match,flexible-extract",78.00,79.50,+1.50
diff --git a/run/capability/capability_summary.md b/run/capability/capability_summary.md
new file mode 100644
index 0000000000000000000000000000000000000000..9f75c2611181bd786fa930a95092bbcd9d9ff37b
--- /dev/null
+++ b/run/capability/capability_summary.md
@@ -0,0 +1,8 @@
+# Capability retention: M0 (base) vs M1 (resist SFT)
+
+arXiv:2511.21399 App. E.4 methodology — lm-eval-harness, MMLU 5-shot MC, GSM8K 8-shot CoT, greedy, accuracy on the test split.
+
+| benchmark | metric | M0 | M1 | delta (pp) |
+|---|---|---|---|---|
+| mmlu | acc,none | 75.1% | 76.5% | +1.4 |
+| gsm8k_cot | exact_match,flexible-extract | 78.0% | 79.5% | +1.5 |
diff --git a/run/capability/lmeval_run.log b/run/capability/lmeval_run.log
new file mode 100644
index 0000000000000000000000000000000000000000..d332576943b6df1b2856624bff450d01945e14ab
--- /dev/null
+++ b/run/capability/lmeval_run.log
@@ -0,0 +1,1796 @@
+
+
+########## m0 / mmlu ##########
+/usr/bin/python -m lm_eval --model hf --model_args pretrained=Qwen/Qwen2.5-7B-Instruct,dtype=bfloat16 --tasks mmlu --num_fewshot 5 --batch_size auto --seed 0 --output_path /root/steering-resistance/results/full_7b/capability/m0/mmlu --apply_chat_template --fewshot_as_multiturn --limit 15
+/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
+  warnings.warn(
+2026-07-24:06:59:32 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-24:06:59:40 INFO     [_cli.run:388] Selected Tasks: ['mmlu']
+2026-07-24:06:59:42 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 0 | Setting torch manual seed to 0 | Setting fewshot manual seed to 0
+2026-07-24:06:59:42 INFO     [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16'}
+2026-07-24:06:59:45 INFO     [models.huggingface:286] Using device 'cuda:0'
+2026-07-24:06:59:47 INFO     [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights:   1%|          | 2/339 [00:00<00:34,  9.90it/s]
Loading weights:   5%|▌         | 18/339 [00:00<00:04, 67.44it/s]
Loading weights:  12%|█▏        | 39/339 [00:00<00:02, 117.55it/s]
Loading weights:  16%|█▌        | 54/339 [00:00<00:02, 122.71it/s]
Loading weights:  22%|██▏       | 76/339 [00:00<00:01, 152.23it/s]
Loading weights:  27%|██▋       | 93/339 [00:00<00:01, 150.55it/s]
Loading weights:  33%|███▎      | 112/339 [00:00<00:01, 157.73it/s]
Loading weights:  38%|███▊      | 129/339 [00:00<00:01, 154.81it/s]
Loading weights:  43%|████▎     | 147/339 [00:01<00:01, 150.27it/s]
Loading weights:  49%|████▉     | 166/339 [00:01<00:01, 161.08it/s]
Loading weights:  55%|█████▍    | 185/339 [00:01<00:00, 160.72it/s]
Loading weights:  60%|██████    | 204/339 [00:01<00:00, 166.55it/s]
Loading weights:  65%|██████▌   | 221/339 [00:01<00:00, 138.27it/s]
Loading weights:  71%|███████   | 239/339 [00:01<00:00, 148.06it/s]
Loading weights:  76%|███████▌  | 256/339 [00:01<00:00, 134.18it/s]
Loading weights:  80%|███████▉  | 271/339 [00:01<00:00, 132.58it/s]
Loading weights:  86%|████████▌ | 291/339 [00:02<00:00, 148.35it/s]
Loading weights:  91%|█████████ | 307/339 [00:02<00:00, 145.24it/s]
Loading weights:  96%|█████████▌| 326/339 [00:02<00:00, 156.08it/s]
Loading weights: 100%|██████████| 339/339 [00:02<00:00, 144.24it/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 16430.85 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3259.21 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1705.83 examples/s]
+
Generating test split:   0%|          | 0/135 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 135/135 [00:00<00:00, 28436.67 examples/s]
+
Generating validation split:   0%|          | 0/14 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 14/14 [00:00<00:00, 4171.36 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1593.70 examples/s]
+
Generating test split:   0%|          | 0/152 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 152/152 [00:00<00:00, 31927.79 examples/s]
+
Generating validation split:   0%|          | 0/16 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 16/16 [00:00<00:00, 6026.84 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 2310.15 examples/s]
+
Generating test split:   0%|          | 0/144 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 144/144 [00:00<00:00, 37398.13 examples/s]
+
Generating validation split:   0%|          | 0/16 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 16/16 [00:00<00:00, 8749.53 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3191.04 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 41128.69 examples/s]
+
Generating validation split:   0%|          | 0/8 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 8/8 [00:00<00:00, 5571.05 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3763.73 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 27960.16 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 5566.76 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3098.17 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 39074.94 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 7359.60 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3454.95 examples/s]
+
Generating test split:   0%|          | 0/102 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 102/102 [00:00<00:00, 22906.20 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3457.02 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1632.66 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 24230.53 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3496.84 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1594.19 examples/s]
+
Generating test split:   0%|          | 0/235 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 235/235 [00:00<00:00, 54716.41 examples/s]
+
Generating validation split:   0%|          | 0/26 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 26/26 [00:00<00:00, 8424.25 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1623.31 examples/s]
+
Generating test split:   0%|          | 0/145 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 145/145 [00:00<00:00, 37727.92 examples/s]
+
Generating validation split:   0%|          | 0/16 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 16/16 [00:00<00:00, 5072.48 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1641.99 examples/s]
+
Generating test split:   0%|          | 0/378 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 378/378 [00:00<00:00, 85363.00 examples/s]
+
Generating validation split:   0%|          | 0/41 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 41/41 [00:00<00:00, 12884.28 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1631.77 examples/s]
+
Generating test split:   0%|          | 0/310 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 310/310 [00:00<00:00, 64591.86 examples/s]
+
Generating validation split:   0%|          | 0/32 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 32/32 [00:00<00:00, 9884.94 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1631.39 examples/s]
+
Generating test split:   0%|          | 0/203 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 203/203 [00:00<00:00, 44775.12 examples/s]
+
Generating validation split:   0%|          | 0/22 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 6978.35 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1621.55 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 23079.87 examples/s]
+
Generating validation split:   0%|          | 0/9 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 9/9 [00:00<00:00, 2890.63 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1614.68 examples/s]
+
Generating test split:   0%|          | 0/270 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 270/270 [00:00<00:00, 63262.50 examples/s]
+
Generating validation split:   0%|          | 0/29 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 29/29 [00:00<00:00, 9260.36 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1622.43 examples/s]
+
Generating test split:   0%|          | 0/151 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 151/151 [00:00<00:00, 37651.74 examples/s]
+
Generating validation split:   0%|          | 0/17 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 17/17 [00:00<00:00, 5464.26 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1653.51 examples/s]
+
Generating test split:   0%|          | 0/216 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 216/216 [00:00<00:00, 49710.27 examples/s]
+
Generating validation split:   0%|          | 0/23 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 23/23 [00:00<00:00, 7244.59 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1664.67 examples/s]
+
Generating test split:   0%|          | 0/112 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 112/112 [00:00<00:00, 25343.23 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3452.10 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1625.95 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 22135.87 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3524.36 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1612.08 examples/s]
+
Generating test split:   0%|          | 0/265 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 265/265 [00:00<00:00, 59355.47 examples/s]
+
Generating validation split:   0%|          | 0/29 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 29/29 [00:00<00:00, 8975.41 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1662.16 examples/s]
+
Generating test split:   0%|          | 0/173 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 173/173 [00:00<00:00, 40222.54 examples/s]
+
Generating validation split:   0%|          | 0/22 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 7028.31 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1627.34 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 24508.03 examples/s]
+
Generating validation split:   0%|          | 0/10 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 10/10 [00:00<00:00, 3222.67 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1647.28 examples/s]
+
Generating test split:   0%|          | 0/223 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 223/223 [00:00<00:00, 46150.38 examples/s]
+
Generating validation split:   0%|          | 0/23 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 23/23 [00:00<00:00, 7188.99 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1609.23 examples/s]
+
Generating test split:   0%|          | 0/103 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 103/103 [00:00<00:00, 26798.17 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3534.89 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1657.30 examples/s]
+
Generating test split:   0%|          | 0/234 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 234/234 [00:00<00:00, 51855.40 examples/s]
+
Generating validation split:   0%|          | 0/25 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 25/25 [00:00<00:00, 7519.37 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1631.64 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 21949.36 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3314.46 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1591.89 examples/s]
+
Generating test split:   0%|          | 0/783 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 783/783 [00:00<00:00, 168685.60 examples/s]
+
Generating validation split:   0%|          | 0/86 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 86/86 [00:00<00:00, 26446.96 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1681.76 examples/s]
+
Generating test split:   0%|          | 0/306 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 306/306 [00:00<00:00, 62252.37 examples/s]
+
Generating validation split:   0%|          | 0/33 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 33/33 [00:00<00:00, 10545.68 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1639.30 examples/s]
+
Generating test split:   0%|          | 0/282 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 282/282 [00:00<00:00, 54797.02 examples/s]
+
Generating validation split:   0%|          | 0/31 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 31/31 [00:00<00:00, 8868.06 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1631.52 examples/s]
+
Generating test split:   0%|          | 0/272 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 272/272 [00:00<00:00, 71276.44 examples/s]
+
Generating validation split:   0%|          | 0/31 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 31/31 [00:00<00:00, 18469.24 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3331.99 examples/s]
+
Generating test split:   0%|          | 0/166 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 166/166 [00:00<00:00, 60877.37 examples/s]
+
Generating validation split:   0%|          | 0/18 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 18/18 [00:00<00:00, 11910.00 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3718.35 examples/s]
+
Generating test split:   0%|          | 0/114 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 114/114 [00:00<00:00, 29466.36 examples/s]
+
Generating validation split:   0%|          | 0/12 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 12/12 [00:00<00:00, 6281.25 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 2940.48 examples/s]
+
Generating test split:   0%|          | 0/198 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 198/198 [00:00<00:00, 78198.89 examples/s]
+
Generating validation split:   0%|          | 0/22 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 14375.24 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3510.47 examples/s]
+
Generating test split:   0%|          | 0/193 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 193/193 [00:00<00:00, 73384.16 examples/s]
+
Generating validation split:   0%|          | 0/21 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 21/21 [00:00<00:00, 13601.05 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3430.64 examples/s]
+
Generating test split:   0%|          | 0/390 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 390/390 [00:00<00:00, 93542.55 examples/s]
+
Generating validation split:   0%|          | 0/43 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 43/43 [00:00<00:00, 23404.50 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 2460.00 examples/s]
+
Generating test split:   0%|          | 0/238 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 238/238 [00:00<00:00, 80386.89 examples/s]
+
Generating validation split:   0%|          | 0/26 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 26/26 [00:00<00:00, 16810.84 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3360.28 examples/s]
+
Generating test split:   0%|          | 0/545 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 545/545 [00:00<00:00, 118772.51 examples/s]
+
Generating validation split:   0%|          | 0/60 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 60/60 [00:00<00:00, 31903.94 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3125.41 examples/s]
+
Generating test split:   0%|          | 0/131 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 131/131 [00:00<00:00, 46650.86 examples/s]
+
Generating validation split:   0%|          | 0/12 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 12/12 [00:00<00:00, 7881.56 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3006.67 examples/s]
+
Generating test split:   0%|          | 0/612 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 612/612 [00:00<00:00, 145822.53 examples/s]
+
Generating validation split:   0%|          | 0/69 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 69/69 [00:00<00:00, 36463.02 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3421.69 examples/s]
+
Generating test split:   0%|          | 0/110 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 110/110 [00:00<00:00, 34807.50 examples/s]
+
Generating validation split:   0%|          | 0/12 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 12/12 [00:00<00:00, 7150.40 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3381.41 examples/s]
+
Generating test split:   0%|          | 0/245 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 245/245 [00:00<00:00, 74447.91 examples/s]
+
Generating validation split:   0%|          | 0/27 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 27/27 [00:00<00:00, 16634.28 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3286.04 examples/s]
+
Generating test split:   0%|          | 0/201 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 201/201 [00:00<00:00, 59070.57 examples/s]
+
Generating validation split:   0%|          | 0/22 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 13301.81 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 2998.07 examples/s]
+
Generating test split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 42781.56 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 7086.06 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3570.84 examples/s]
+
Generating test split:   0%|          | 0/126 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 126/126 [00:00<00:00, 46867.89 examples/s]
+
Generating validation split:   0%|          | 0/14 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 14/14 [00:00<00:00, 9067.37 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3365.67 examples/s]
+
Generating test split:   0%|          | 0/165 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 165/165 [00:00<00:00, 45407.79 examples/s]
+
Generating validation split:   0%|          | 0/18 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 18/18 [00:00<00:00, 10614.01 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3355.44 examples/s]
+
Generating test split:   0%|          | 0/204 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 204/204 [00:00<00:00, 54350.38 examples/s]
+
Generating validation split:   0%|          | 0/22 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 22/22 [00:00<00:00, 9625.98 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 2989.10 examples/s]
+
Generating test split:   0%|          | 0/237 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 237/237 [00:00<00:00, 49877.07 examples/s]
+
Generating validation split:   0%|          | 0/26 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 26/26 [00:00<00:00, 9236.99 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3063.77 examples/s]
+
Generating test split:   0%|          | 0/121 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 121/121 [00:00<00:00, 32153.50 examples/s]
+
Generating validation split:   0%|          | 0/13 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 13/13 [00:00<00:00, 4383.82 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1734.47 examples/s]
+
Generating test split:   0%|          | 0/108 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 108/108 [00:00<00:00, 22462.80 examples/s]
+
Generating validation split:   0%|          | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3137.31 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1612.57 examples/s]
+
Generating test split:   0%|          | 0/163 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 163/163 [00:00<00:00, 56632.83 examples/s]
+
Generating validation split:   0%|          | 0/18 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 18/18 [00:00<00:00, 11112.37 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3323.01 examples/s]
+
Generating test split:   0%|          | 0/346 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 346/346 [00:00<00:00, 84773.01 examples/s]
+
Generating validation split:   0%|          | 0/38 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 38/38 [00:00<00:00, 13389.08 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1793.82 examples/s]
+
Generating test split:   0%|          | 0/895 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 895/895 [00:00<00:00, 198325.34 examples/s]
+
Generating validation split:   0%|          | 0/100 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 100/100 [00:00<00:00, 44544.44 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 2613.27 examples/s]
+
Generating test split:   0%|          | 0/311 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 311/311 [00:00<00:00, 119540.74 examples/s]
+
Generating validation split:   0%|          | 0/34 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 34/34 [00:00<00:00, 22359.10 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3291.20 examples/s]
+
Generating test split:   0%|          | 0/324 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 324/324 [00:00<00:00, 113048.37 examples/s]
+
Generating validation split:   0%|          | 0/35 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 35/35 [00:00<00:00, 22208.87 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3415.56 examples/s]
+
Generating test split:   0%|          | 0/1534 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 1534/1534 [00:00<00:00, 166405.34 examples/s]
+
Generating validation split:   0%|          | 0/170 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 170/170 [00:00<00:00, 56215.05 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 2611.32 examples/s]
+
Generating test split:   0%|          | 0/171 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 171/171 [00:00<00:00, 47438.72 examples/s]
+
Generating validation split:   0%|          | 0/19 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 19/19 [00:00<00:00, 9720.88 examples/s]
+
Generating dev split:   0%|          | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 3386.87 examples/s]
+2026-07-24:07:03:30 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-24:07:03:30 INFO     [evaluator_utils:462] Group: mmlu
+2026-07-24:07:03:30 INFO     [evaluator_utils:462]   Group: mmlu_stem
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_abstract_algebra (mmlu/default/mmlu_abstract_algebra.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_anatomy (mmlu/default/mmlu_anatomy.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_astronomy (mmlu/default/mmlu_astronomy.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_college_biology (mmlu/default/mmlu_college_biology.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_college_chemistry (mmlu/default/mmlu_college_chemistry.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_college_computer_science (mmlu/default/mmlu_college_computer_science.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_college_mathematics (mmlu/default/mmlu_college_mathematics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_college_physics (mmlu/default/mmlu_college_physics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_computer_security (mmlu/default/mmlu_computer_security.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_conceptual_physics (mmlu/default/mmlu_conceptual_physics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_electrical_engineering (mmlu/default/mmlu_electrical_engineering.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_elementary_mathematics (mmlu/default/mmlu_elementary_mathematics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_biology (mmlu/default/mmlu_high_school_biology.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_chemistry (mmlu/default/mmlu_high_school_chemistry.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_computer_science (mmlu/default/mmlu_high_school_computer_science.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_mathematics (mmlu/default/mmlu_high_school_mathematics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_physics (mmlu/default/mmlu_high_school_physics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_statistics (mmlu/default/mmlu_high_school_statistics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_machine_learning (mmlu/default/mmlu_machine_learning.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:462]   Group: mmlu_other
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_business_ethics (mmlu/default/mmlu_business_ethics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_clinical_knowledge (mmlu/default/mmlu_clinical_knowledge.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_college_medicine (mmlu/default/mmlu_college_medicine.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_global_facts (mmlu/default/mmlu_global_facts.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_human_aging (mmlu/default/mmlu_human_aging.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_management (mmlu/default/mmlu_management.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_marketing (mmlu/default/mmlu_marketing.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_medical_genetics (mmlu/default/mmlu_medical_genetics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_miscellaneous (mmlu/default/mmlu_miscellaneous.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_nutrition (mmlu/default/mmlu_nutrition.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_professional_accounting (mmlu/default/mmlu_professional_accounting.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_professional_medicine (mmlu/default/mmlu_professional_medicine.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_virology (mmlu/default/mmlu_virology.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:462]   Group: mmlu_social_sciences
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_econometrics (mmlu/default/mmlu_econometrics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_geography (mmlu/default/mmlu_high_school_geography.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_government_and_politics (mmlu/default/mmlu_high_school_government_and_politics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_macroeconomics (mmlu/default/mmlu_high_school_macroeconomics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_microeconomics (mmlu/default/mmlu_high_school_microeconomics.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_psychology (mmlu/default/mmlu_high_school_psychology.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_human_sexuality (mmlu/default/mmlu_human_sexuality.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_professional_psychology (mmlu/default/mmlu_professional_psychology.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_public_relations (mmlu/default/mmlu_public_relations.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_security_studies (mmlu/default/mmlu_security_studies.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_sociology (mmlu/default/mmlu_sociology.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_us_foreign_policy (mmlu/default/mmlu_us_foreign_policy.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:462]   Group: mmlu_humanities
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_formal_logic (mmlu/default/mmlu_formal_logic.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_european_history (mmlu/default/mmlu_high_school_european_history.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_us_history (mmlu/default/mmlu_high_school_us_history.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_high_school_world_history (mmlu/default/mmlu_high_school_world_history.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_international_law (mmlu/default/mmlu_international_law.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_jurisprudence (mmlu/default/mmlu_jurisprudence.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_logical_fallacies (mmlu/default/mmlu_logical_fallacies.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_moral_disputes (mmlu/default/mmlu_moral_disputes.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_moral_scenarios (mmlu/default/mmlu_moral_scenarios.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_philosophy (mmlu/default/mmlu_philosophy.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_prehistory (mmlu/default/mmlu_prehistory.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_professional_law (mmlu/default/mmlu_professional_law.yaml)
+2026-07-24:07:03:30 INFO     [evaluator_utils:470]     Task: mmlu_world_religions (mmlu/default/mmlu_world_religions.yaml)
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_abstract_algebra from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_anatomy from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_astronomy from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_biology from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_chemistry from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_computer_science from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_mathematics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_physics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_computer_security from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_conceptual_physics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_electrical_engineering from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_elementary_mathematics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_biology from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_chemistry from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_computer_science from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_mathematics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_physics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_statistics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_machine_learning from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_business_ethics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_clinical_knowledge from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_medicine from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_global_facts from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_aging from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_management from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_marketing from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_medical_genetics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_miscellaneous from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_nutrition from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_accounting from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_medicine from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_virology from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_econometrics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_geography from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_government_and_politics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_macroeconomics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_microeconomics from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_psychology from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_sexuality from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_psychology from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_public_relations from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_security_studies from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_sociology from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_us_foreign_policy from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_formal_logic from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_european_history from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_us_history from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_world_history from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_international_law from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_jurisprudence from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_logical_fallacies from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_disputes from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_scenarios from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_philosophy from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_prehistory from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_law from None to 5
+2026-07-24:07:03:30 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_world_religions from None to 5
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_abstract_algebra on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 155.15it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_anatomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 178.73it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_astronomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 175.53it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_college_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 174.59it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_college_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 175.39it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_college_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 183.82it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_college_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.21it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_college_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 189.37it/s]
+2026-07-24:07:03:30 INFO     [api.task:312] Building contexts for mmlu_computer_security on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 189.93it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_conceptual_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 189.57it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_electrical_engineering on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.00it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_elementary_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.56it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_high_school_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.70it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_high_school_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.83it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_high_school_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.12it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_high_school_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.17it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_high_school_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.21it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_high_school_statistics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.71it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_machine_learning on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.46it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_business_ethics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.23it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_clinical_knowledge on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.03it/s]
+2026-07-24:07:03:31 INFO     [api.task:312] Building contexts for mmlu_college_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.95it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_global_facts on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.75it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_human_aging on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.14it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_management on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.86it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_marketing on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.73it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_medical_genetics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.32it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_miscellaneous on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.48it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_nutrition on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.25it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_professional_accounting on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.26it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_professional_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.58it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_virology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.78it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_econometrics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.31it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_high_school_geography on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.41it/s]
+2026-07-24:07:03:32 INFO     [api.task:312] Building contexts for mmlu_high_school_government_and_politics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 187.55it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_high_school_macroeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.12it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_high_school_microeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.03it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_high_school_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.52it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_human_sexuality on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.34it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_professional_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 174.80it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_public_relations on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 174.56it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_security_studies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 174.08it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_sociology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 164.39it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_us_foreign_policy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.55it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_formal_logic on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.50it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_high_school_european_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.52it/s]
+2026-07-24:07:03:33 INFO     [api.task:312] Building contexts for mmlu_high_school_us_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.47it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_high_school_world_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.61it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_international_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.41it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_jurisprudence on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.13it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_logical_fallacies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.99it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_moral_disputes on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.80it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_moral_scenarios on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.20it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_philosophy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.73it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_prehistory on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.56it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_professional_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.15it/s]
+2026-07-24:07:03:34 INFO     [api.task:312] Building contexts for mmlu_world_religions on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.83it/s]
+2026-07-24:07:03:34 INFO     [evaluator:585] Running loglikelihood requests
+
Tokenizing inputs:   0%|          | 0/3420 [00:00<?, ?it/s]
Tokenizing inputs:   2%|▏         | 69/3420 [00:00<00:04, 688.52it/s]
Tokenizing inputs:   4%|▍         | 138/3420 [00:00<00:05, 642.25it/s]
Tokenizing inputs:   6%|▌         | 203/3420 [00:00<00:05, 547.89it/s]
Tokenizing inputs:   8%|▊         | 260/3420 [00:00<00:05, 540.59it/s]
Tokenizing inputs:   9%|▉         | 315/3420 [00:00<00:06, 495.90it/s]
Tokenizing inputs:  11%|█         | 366/3420 [00:00<00:07, 431.15it/s]
Tokenizing inputs:  12%|█▏        | 413/3420 [00:00<00:06, 440.16it/s]
Tokenizing inputs:  14%|█▎        | 465/3420 [00:00<00:06, 460.78it/s]
Tokenizing inputs:  15%|█▌        | 524/3420 [00:01<00:05, 496.98it/s]
Tokenizing inputs:  18%|█▊        | 599/3420 [00:01<00:04, 568.34it/s]
Tokenizing inputs:  19%|█▉        | 658/3420 [00:01<00:04, 569.08it/s]
Tokenizing inputs:  21%|██        | 716/3420 [00:01<00:05, 531.18it/s]
Tokenizing inputs:  23%|██▎       | 771/3420 [00:01<00:05, 521.78it/s]
Tokenizing inputs:  24%|██▍       | 826/3420 [00:01<00:04, 529.20it/s]
Tokenizing inputs:  26%|██▌       | 880/3420 [00:01<00:05, 450.84it/s]
Tokenizing inputs:  27%|██▋       | 928/3420 [00:01<00:05, 429.83it/s]
Tokenizing inputs:  29%|██▊       | 978/3420 [00:01<00:05, 447.03it/s]
Tokenizing inputs:  30%|██▉       | 1025/3420 [00:02<00:05, 440.73it/s]
Tokenizing inputs:  31%|███▏      | 1071/3420 [00:02<00:05, 404.41it/s]
Tokenizing inputs:  33%|███▎      | 1113/3420 [00:02<00:05, 399.57it/s]
Tokenizing inputs:  34%|███▍      | 1155/3420 [00:02<00:05, 404.04it/s]
Tokenizing inputs:  35%|███▌      | 1202/3420 [00:02<00:05, 422.10it/s]
Tokenizing inputs:  37%|███▋      | 1265/3420 [00:02<00:04, 479.22it/s]
Tokenizing inputs:  38%|███▊      | 1314/3420 [00:02<00:04, 472.77it/s]
Tokenizing inputs:  40%|████      | 1374/3420 [00:02<00:04, 508.72it/s]
Tokenizing inputs:  42%|████▏     | 1449/3420 [00:02<00:03, 577.81it/s]
Tokenizing inputs:  45%|████▍     | 1523/3420 [00:03<00:03, 624.64it/s]
Tokenizing inputs:  46%|████▋     | 1589/3420 [00:03<00:02, 632.67it/s]
Tokenizing inputs:  49%|████▉     | 1670/3420 [00:03<00:02, 684.23it/s]
Tokenizing inputs:  51%|█████     | 1739/3420 [00:03<00:02, 606.21it/s]
Tokenizing inputs:  53%|█████▎    | 1802/3420 [00:03<00:03, 516.91it/s]
Tokenizing inputs:  54%|█████▍    | 1857/3420 [00:03<00:03, 407.04it/s]
Tokenizing inputs:  56%|█████▋    | 1925/3420 [00:03<00:03, 463.62it/s]
Tokenizing inputs:  58%|█████▊    | 1978/3420 [00:03<00:03, 459.74it/s]
Tokenizing inputs:  60%|█████▉    | 2040/3420 [00:04<00:02, 497.00it/s]
Tokenizing inputs:  61%|██████    | 2094/3420 [00:04<00:02, 499.56it/s]
Tokenizing inputs:  63%|██████▎   | 2155/3420 [00:04<00:02, 527.80it/s]
Tokenizing inputs:  65%|██████▍   | 2210/3420 [00:04<00:02, 527.72it/s]
Tokenizing inputs:  66%|██████▌   | 2265/3420 [00:04<00:02, 523.55it/s]
Tokenizing inputs:  68%|██████▊   | 2330/3420 [00:04<00:01, 558.50it/s]
Tokenizing inputs:  70%|██████▉   | 2387/3420 [00:04<00:01, 522.94it/s]
Tokenizing inputs:  71%|███████▏  | 2441/3420 [00:04<00:01, 521.05it/s]
Tokenizing inputs:  73%|███████▎  | 2494/3420 [00:05<00:02, 417.89it/s]
Tokenizing inputs:  74%|███████▍  | 2540/3420 [00:05<00:02, 376.86it/s]
Tokenizing inputs:  76%|███████▌  | 2594/3420 [00:05<00:01, 413.97it/s]
Tokenizing inputs:  77%|███████▋  | 2648/3420 [00:05<00:01, 444.15it/s]
Tokenizing inputs:  79%|███████▉  | 2696/3420 [00:05<00:01, 448.40it/s]
Tokenizing inputs:  80%|████████  | 2743/3420 [00:05<00:02, 233.68it/s]
Tokenizing inputs:  81%|████████▏ | 2779/3420 [00:06<00:03, 183.09it/s]
Tokenizing inputs:  82%|████████▏ | 2808/3420 [00:06<00:03, 161.78it/s]
Tokenizing inputs:  83%|████████▎ | 2832/3420 [00:06<00:03, 156.97it/s]
Tokenizing inputs:  83%|████████▎ | 2853/3420 [00:06<00:03, 159.82it/s]
Tokenizing inputs:  84%|████████▍ | 2873/3420 [00:06<00:03, 161.85it/s]
Tokenizing inputs:  85%|████████▍ | 2901/3420 [00:07<00:02, 184.73it/s]
Tokenizing inputs:  86%|████████▌ | 2937/3420 [00:07<00:02, 223.33it/s]
Tokenizing inputs:  87%|████████▋ | 2991/3420 [00:07<00:01, 298.85it/s]
Tokenizing inputs:  89%|████████▉ | 3041/3420 [00:07<00:01, 349.84it/s]
Tokenizing inputs:  90%|█████████ | 3090/3420 [00:07<00:00, 386.20it/s]
Tokenizing inputs:  92%|█████████▏| 3134/3420 [00:07<00:00, 398.80it/s]
Tokenizing inputs:  93%|█████████▎| 3177/3420 [00:07<00:00, 389.36it/s]
Tokenizing inputs:  95%|█████████▍| 3243/3420 [00:07<00:00, 463.62it/s]
Tokenizing inputs:  96%|█████████▋| 3292/3420 [00:07<00:00, 451.66it/s]
Tokenizing inputs:  98%|█████████▊| 3339/3420 [00:08<00:00, 297.20it/s]
Tokenizing inputs:  99%|█████████▊| 3377/3420 [00:08<00:00, 271.75it/s]
Tokenizing inputs: 100%|██████████| 3420/3420 [00:08<00:00, 406.60it/s]
+
Running loglikelihood requests:   0%|          | 0/3420 [00:00<?, ?it/s][W724 07:03:44.682734209 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 1398800384 bytes (free: 1334902784, total: 25323503616).
+[W724 07:03:44.692117298 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 1398800384 bytes (free: 1336999936, total: 25323503616).
+[W724 07:03:45.225058967 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2491416576 bytes (free: 1487994880, total: 25323503616).
+[W724 07:03:45.350124050 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2491416576 bytes (free: 1490092032, total: 25323503616).
+[W724 07:03:45.763971953 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2229272576 bytes (free: 1487994880, total: 25323503616).
+[W724 07:03:45.015232763 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2229272576 bytes (free: 659619840, total: 25323503616).
+[W724 07:03:45.015443062 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2229272576 bytes (free: 659619840, total: 25323503616).
+[W724 07:03:46.429121736 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 5196742656 bytes (free: 657522688, total: 25323503616).
+[W724 07:03:46.706313938 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 5196742656 bytes (free: 3457220608, total: 25323503616).
+[W724 07:03:46.119323845 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 4620025856 bytes (free: 3455123456, total: 25323503616).
+[W724 07:03:46.119594404 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 4620025856 bytes (free: 3455123456, total: 25323503616).
+[W724 07:03:47.519073863 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 4158652416 bytes (free: 3455123456, total: 25323503616).
+[W724 07:03:47.746748559 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 4158652416 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:47.747117978 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 4158652416 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:47.152750694 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3697278976 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:47.152949563 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3697278976 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:48.553544097 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3233808384 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:48.553752476 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3233808384 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:48.951723194 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2887778304 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:48.951923873 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2887778304 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:49.347026236 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2541748224 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:49.347203045 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2541748224 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:49.934171845 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2195718144 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:49.934344804 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2195718144 bytes (free: 1525743616, total: 25323503616).
+[W724 07:03:50.431509363 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 1965031424 bytes (free: 1754333184, total: 25323503616).
+[W724 07:03:50.968855997 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 1965031424 bytes (free: 1756430336, total: 25323503616).
+[W724 07:03:51.418108671 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 1732247552 bytes (free: 256966656, total: 25323503616).
+[W724 07:03:51.418341660 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 1732247552 bytes (free: 256966656, total: 25323503616).
+[W724 07:03:54.498813424 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 12052332544 bytes (free: 514916352, total: 25323503616).
+[W724 07:04:01.876636141 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 12052332544 bytes (free: 5483069440, total: 25323503616).
+[W724 07:04:02.795794779 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 10198450176 bytes (free: 1668349952, total: 25323503616).
+[W724 07:04:08.074967975 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 10198450176 bytes (free: 5944442880, total: 25323503616).
+[W724 07:04:09.899865376 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 8344567808 bytes (free: 2821783552, total: 25323503616).
+[W724 07:04:14.043266229 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 8344567808 bytes (free: 6405816320, total: 25323503616).
+[W724 07:04:15.821188720 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 7417626624 bytes (free: 3629187072, total: 25323503616).
+[W724 07:04:20.403925052 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 7417626624 bytes (free: 6842023936, total: 25323503616).
+[W724 07:04:20.137264992 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 6490685440 bytes (free: 4411424768, total: 25323503616).
+[W724 07:04:24.191626811 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 6490685440 bytes (free: 1506869248, total: 25323503616).
+[W724 07:04:25.553668901 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 6490685440 bytes (free: 3241213952, total: 25323503616).
+[W724 07:04:26.234742399 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 5563744256 bytes (free: 3239116800, total: 25323503616).
+[W724 07:04:29.641853200 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 5563744256 bytes (free: 3241213952, total: 25323503616).
+[W724 07:04:30.278318216 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 4636803072 bytes (free: 3239116800, total: 25323503616).
+[W724 07:04:33.368086070 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 4636803072 bytes (free: 3241213952, total: 25323503616).
+[W724 07:04:33.953711357 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3709861888 bytes (free: 3239116800, total: 25323503616).
+[W724 07:04:36.440505036 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3709861888 bytes (free: 3241213952, total: 25323503616).
+
Running loglikelihood requests:   0%|          | 1/3420 [01:05<62:17:01, 65.58s/it][W724 07:04:48.203727316 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2720006144 bytes (free: 1825636352, total: 25323503616).
+
Running loglikelihood requests:   0%|          | 13/3420 [01:07<3:34:22,  3.78s/it]
Running loglikelihood requests:   1%|          | 25/3420 [01:09<1:36:05,  1.70s/it]
Running loglikelihood requests:   1%|          | 37/3420 [01:11<56:33,  1.00s/it]  
Running loglikelihood requests:   1%|▏         | 49/3420 [01:13<37:35,  1.49it/s]
Running loglikelihood requests:   2%|▏         | 61/3420 [01:14<26:41,  2.10it/s]
Running loglikelihood requests:   2%|▏         | 73/3420 [01:16<20:01,  2.79it/s]
Running loglikelihood requests:   2%|▏         | 85/3420 [01:17<15:45,  3.53it/s]
Running loglikelihood requests:   3%|▎         | 97/3420 [01:19<12:57,  4.27it/s]
Running loglikelihood requests:   3%|▎         | 109/3420 [01:20<11:04,  4.98it/s]
Running loglikelihood requests:   4%|▎         | 121/3420 [01:22<09:28,  5.80it/s]
Running loglikelihood requests:   4%|▍         | 133/3420 [01:23<08:14,  6.65it/s]
Running loglikelihood requests:   4%|▍         | 145/3420 [01:24<07:20,  7.44it/s]
Running loglikelihood requests:   5%|▍         | 157/3420 [01:25<06:37,  8.21it/s]
Running loglikelihood requests:   5%|▍         | 169/3420 [01:26<06:07,  8.86it/s]
Running loglikelihood requests:   5%|▌         | 181/3420 [01:27<05:42,  9.47it/s]
Running loglikelihood requests:   6%|▌         | 193/3420 [01:28<05:20, 10.08it/s]
Running loglikelihood requests:   6%|▌         | 205/3420 [01:29<05:00, 10.71it/s]
Running loglikelihood requests:   6%|▋         | 217/3420 [01:30<04:43, 11.29it/s]
+
+########## m0 / mmlu ##########
+/usr/bin/python -m lm_eval --model hf --model_args pretrained=Qwen/Qwen2.5-7B-Instruct,dtype=bfloat16 --tasks mmlu --num_fewshot 5 --batch_size 4 --seed 0 --output_path /root/steering-resistance/results/full_7b/capability/m0/mmlu --apply_chat_template --fewshot_as_multiturn --limit 15
+/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
+  warnings.warn(
+2026-07-24:07:05:35 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-24:07:05:44 INFO     [_cli.run:388] Selected Tasks: ['mmlu']
+2026-07-24:07:05:46 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 0 | Setting torch manual seed to 0 | Setting fewshot manual seed to 0
+2026-07-24:07:05:46 INFO     [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16'}
+2026-07-24:07:05:49 INFO     [models.huggingface:286] Using device 'cuda:0'
+2026-07-24:07:05:50 INFO     [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights:   1%|          | 2/339 [00:00<00:34,  9.87it/s]
Loading weights:   5%|▍         | 16/339 [00:00<00:05, 62.84it/s]
Loading weights:  12%|█▏        | 41/339 [00:00<00:02, 131.42it/s]
Loading weights:  19%|█▉        | 65/339 [00:00<00:01, 151.64it/s]
Loading weights:  27%|██▋       | 90/339 [00:00<00:01, 172.99it/s]
Loading weights:  33%|███▎      | 113/339 [00:00<00:01, 184.11it/s]
Loading weights:  40%|███▉      | 135/339 [00:00<00:01, 193.85it/s]
Loading weights:  47%|████▋     | 160/339 [00:00<00:00, 193.63it/s]
Loading weights:  55%|█████▍    | 186/339 [00:01<00:00, 203.77it/s]
Loading weights:  61%|██████▏   | 208/339 [00:01<00:00, 203.09it/s]
Loading weights:  68%|██████▊   | 229/339 [00:01<00:00, 188.49it/s]
Loading weights:  73%|███████▎  | 249/339 [00:01<00:00, 184.38it/s]
Loading weights:  79%|███████▉  | 268/339 [00:01<00:00, 166.73it/s]
Loading weights:  84%|████████▍ | 286/339 [00:01<00:00, 170.02it/s]
Loading weights:  90%|████████▉ | 305/339 [00:01<00:00, 169.09it/s]
Loading weights:  97%|█████████▋| 329/339 [00:01<00:00, 177.91it/s]
Loading weights: 100%|██████████| 339/339 [00:01<00:00, 173.60it/s]
+2026-07-24:07:06:52 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-24:07:06:52 INFO     [evaluator_utils:462] Group: mmlu
+2026-07-24:07:06:52 INFO     [evaluator_utils:462]   Group: mmlu_stem
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_abstract_algebra (mmlu/default/mmlu_abstract_algebra.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_anatomy (mmlu/default/mmlu_anatomy.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_astronomy (mmlu/default/mmlu_astronomy.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_college_biology (mmlu/default/mmlu_college_biology.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_college_chemistry (mmlu/default/mmlu_college_chemistry.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_college_computer_science (mmlu/default/mmlu_college_computer_science.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_college_mathematics (mmlu/default/mmlu_college_mathematics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_college_physics (mmlu/default/mmlu_college_physics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_computer_security (mmlu/default/mmlu_computer_security.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_conceptual_physics (mmlu/default/mmlu_conceptual_physics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_electrical_engineering (mmlu/default/mmlu_electrical_engineering.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_elementary_mathematics (mmlu/default/mmlu_elementary_mathematics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_biology (mmlu/default/mmlu_high_school_biology.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_chemistry (mmlu/default/mmlu_high_school_chemistry.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_computer_science (mmlu/default/mmlu_high_school_computer_science.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_mathematics (mmlu/default/mmlu_high_school_mathematics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_physics (mmlu/default/mmlu_high_school_physics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_statistics (mmlu/default/mmlu_high_school_statistics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_machine_learning (mmlu/default/mmlu_machine_learning.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:462]   Group: mmlu_other
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_business_ethics (mmlu/default/mmlu_business_ethics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_clinical_knowledge (mmlu/default/mmlu_clinical_knowledge.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_college_medicine (mmlu/default/mmlu_college_medicine.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_global_facts (mmlu/default/mmlu_global_facts.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_human_aging (mmlu/default/mmlu_human_aging.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_management (mmlu/default/mmlu_management.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_marketing (mmlu/default/mmlu_marketing.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_medical_genetics (mmlu/default/mmlu_medical_genetics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_miscellaneous (mmlu/default/mmlu_miscellaneous.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_nutrition (mmlu/default/mmlu_nutrition.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_professional_accounting (mmlu/default/mmlu_professional_accounting.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_professional_medicine (mmlu/default/mmlu_professional_medicine.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_virology (mmlu/default/mmlu_virology.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:462]   Group: mmlu_social_sciences
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_econometrics (mmlu/default/mmlu_econometrics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_geography (mmlu/default/mmlu_high_school_geography.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_government_and_politics (mmlu/default/mmlu_high_school_government_and_politics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_macroeconomics (mmlu/default/mmlu_high_school_macroeconomics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_microeconomics (mmlu/default/mmlu_high_school_microeconomics.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_psychology (mmlu/default/mmlu_high_school_psychology.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_human_sexuality (mmlu/default/mmlu_human_sexuality.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_professional_psychology (mmlu/default/mmlu_professional_psychology.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_public_relations (mmlu/default/mmlu_public_relations.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_security_studies (mmlu/default/mmlu_security_studies.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_sociology (mmlu/default/mmlu_sociology.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_us_foreign_policy (mmlu/default/mmlu_us_foreign_policy.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:462]   Group: mmlu_humanities
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_formal_logic (mmlu/default/mmlu_formal_logic.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_european_history (mmlu/default/mmlu_high_school_european_history.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_us_history (mmlu/default/mmlu_high_school_us_history.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_high_school_world_history (mmlu/default/mmlu_high_school_world_history.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_international_law (mmlu/default/mmlu_international_law.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_jurisprudence (mmlu/default/mmlu_jurisprudence.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_logical_fallacies (mmlu/default/mmlu_logical_fallacies.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_moral_disputes (mmlu/default/mmlu_moral_disputes.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_moral_scenarios (mmlu/default/mmlu_moral_scenarios.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_philosophy (mmlu/default/mmlu_philosophy.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_prehistory (mmlu/default/mmlu_prehistory.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_professional_law (mmlu/default/mmlu_professional_law.yaml)
+2026-07-24:07:06:52 INFO     [evaluator_utils:470]     Task: mmlu_world_religions (mmlu/default/mmlu_world_religions.yaml)
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_abstract_algebra from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_anatomy from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_astronomy from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_biology from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_chemistry from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_computer_science from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_mathematics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_physics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_computer_security from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_conceptual_physics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_electrical_engineering from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_elementary_mathematics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_biology from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_chemistry from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_computer_science from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_mathematics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_physics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_statistics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_machine_learning from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_business_ethics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_clinical_knowledge from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_medicine from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_global_facts from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_aging from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_management from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_marketing from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_medical_genetics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_miscellaneous from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_nutrition from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_accounting from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_medicine from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_virology from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_econometrics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_geography from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_government_and_politics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_macroeconomics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_microeconomics from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_psychology from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_sexuality from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_psychology from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_public_relations from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_security_studies from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_sociology from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_us_foreign_policy from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_formal_logic from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_european_history from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_us_history from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_world_history from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_international_law from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_jurisprudence from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_logical_fallacies from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_disputes from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_scenarios from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_philosophy from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_prehistory from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_law from None to 5
+2026-07-24:07:06:52 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_world_religions from None to 5
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_abstract_algebra on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
 87%|████████▋ | 13/15 [00:00<00:00, 127.35it/s]
100%|██████████| 15/15 [00:00<00:00, 133.15it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_anatomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.18it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_astronomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.29it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_college_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.32it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_college_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.44it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_college_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.26it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_college_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.22it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_college_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.26it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_computer_security on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.82it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_conceptual_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.51it/s]
+2026-07-24:07:06:52 INFO     [api.task:312] Building contexts for mmlu_electrical_engineering on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.02it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_elementary_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.28it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_high_school_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.83it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_high_school_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.29it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_high_school_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.41it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_high_school_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.50it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_high_school_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.86it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_high_school_statistics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.61it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_machine_learning on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.88it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_business_ethics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.49it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_clinical_knowledge on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.34it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_college_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.97it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_global_facts on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.42it/s]
+2026-07-24:07:06:53 INFO     [api.task:312] Building contexts for mmlu_human_aging on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.50it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_management on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.60it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_marketing on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.71it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_medical_genetics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.32it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_miscellaneous on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.17it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_nutrition on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.36it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_professional_accounting on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.89it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_professional_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.02it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_virology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.34it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_econometrics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.63it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_high_school_geography on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.90it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_high_school_government_and_politics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.71it/s]
+2026-07-24:07:06:54 INFO     [api.task:312] Building contexts for mmlu_high_school_macroeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.25it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_high_school_microeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.10it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_high_school_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.59it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_human_sexuality on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.88it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_professional_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.32it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_public_relations on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.47it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_security_studies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.75it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_sociology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.27it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_us_foreign_policy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.39it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_formal_logic on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.38it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_high_school_european_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.31it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_high_school_us_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.43it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_high_school_world_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.26it/s]
+2026-07-24:07:06:55 INFO     [api.task:312] Building contexts for mmlu_international_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.33it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_jurisprudence on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.21it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_logical_fallacies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.42it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_moral_disputes on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.74it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_moral_scenarios on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.07it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_philosophy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.75it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_prehistory on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.94it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_professional_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.25it/s]
+2026-07-24:07:06:56 INFO     [api.task:312] Building contexts for mmlu_world_religions on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.53it/s]
+2026-07-24:07:06:56 INFO     [evaluator:585] Running loglikelihood requests
+
Tokenizing inputs:   0%|          | 0/3420 [00:00<?, ?it/s]
Tokenizing inputs:   2%|▏         | 69/3420 [00:00<00:04, 689.67it/s]
Tokenizing inputs:   4%|▍         | 138/3420 [00:00<00:05, 644.59it/s]
Tokenizing inputs:   6%|▌         | 203/3420 [00:00<00:05, 548.36it/s]
Tokenizing inputs:   8%|▊         | 260/3420 [00:00<00:05, 540.88it/s]
Tokenizing inputs:   9%|▉         | 315/3420 [00:00<00:06, 497.46it/s]
Tokenizing inputs:  11%|█         | 366/3420 [00:00<00:07, 433.79it/s]
Tokenizing inputs:  12%|█▏        | 413/3420 [00:00<00:06, 441.42it/s]
Tokenizing inputs:  14%|█▎        | 465/3420 [00:00<00:06, 461.89it/s]
Tokenizing inputs:  15%|█▌        | 524/3420 [00:01<00:05, 496.96it/s]
Tokenizing inputs:  18%|█▊        | 599/3420 [00:01<00:04, 568.22it/s]
Tokenizing inputs:  19%|█▉        | 661/3420 [00:01<00:04, 579.82it/s]
Tokenizing inputs:  21%|██        | 720/3420 [00:01<00:04, 549.77it/s]
Tokenizing inputs:  23%|██▎       | 776/3420 [00:01<00:04, 539.40it/s]
Tokenizing inputs:  24%|██▍       | 831/3420 [00:01<00:04, 541.98it/s]
Tokenizing inputs:  26%|██▌       | 886/3420 [00:01<00:05, 450.52it/s]
Tokenizing inputs:  27%|██▋       | 934/3420 [00:01<00:05, 439.02it/s]
Tokenizing inputs:  29%|██▊       | 983/3420 [00:01<00:05, 451.03it/s]
Tokenizing inputs:  30%|███       | 1030/3420 [00:02<00:05, 441.15it/s]
Tokenizing inputs:  31%|███▏      | 1076/3420 [00:02<00:05, 405.30it/s]
Tokenizing inputs:  33%|███▎      | 1118/3420 [00:02<00:05, 402.10it/s]
Tokenizing inputs:  34%|███▍      | 1161/3420 [00:02<00:05, 408.87it/s]
Tokenizing inputs:  35%|███▌      | 1207/3420 [00:02<00:05, 422.45it/s]
Tokenizing inputs:  37%|███▋      | 1269/3420 [00:02<00:04, 476.15it/s]
Tokenizing inputs:  39%|███▊      | 1318/3420 [00:02<00:04, 469.70it/s]
Tokenizing inputs:  40%|████      | 1380/3420 [00:02<00:03, 511.28it/s]
Tokenizing inputs:  43%|████▎     | 1457/3420 [00:02<00:03, 584.21it/s]
Tokenizing inputs:  45%|████▍     | 1529/3420 [00:03<00:03, 620.77it/s]
Tokenizing inputs:  47%|████▋     | 1595/3420 [00:03<00:02, 631.25it/s]
Tokenizing inputs:  49%|████▉     | 1676/3420 [00:03<00:02, 682.89it/s]
Tokenizing inputs:  51%|█████     | 1745/3420 [00:03<00:02, 590.84it/s]
Tokenizing inputs:  53%|█████▎    | 1807/3420 [00:03<00:03, 500.68it/s]
Tokenizing inputs:  54%|█████▍    | 1861/3420 [00:03<00:03, 402.34it/s]
Tokenizing inputs:  56%|█████▋    | 1929/3420 [00:03<00:03, 461.08it/s]
Tokenizing inputs:  58%|█████▊    | 1981/3420 [00:03<00:03, 457.92it/s]
Tokenizing inputs:  60%|█████▉    | 2043/3420 [00:04<00:02, 494.94it/s]
Tokenizing inputs:  61%|██████▏   | 2097/3420 [00:04<00:02, 498.11it/s]
Tokenizing inputs:  63%|██████▎   | 2158/3420 [00:04<00:02, 526.65it/s]
Tokenizing inputs:  65%|██████▍   | 2217/3420 [00:04<00:02, 543.73it/s]
Tokenizing inputs:  66%|██████▋   | 2274/3420 [00:04<00:02, 529.85it/s]
Tokenizing inputs:  68%|██████▊   | 2342/3420 [00:04<00:01, 568.71it/s]
Tokenizing inputs:  70%|███████   | 2400/3420 [00:04<00:01, 513.39it/s]
Tokenizing inputs:  72%|███████▏  | 2457/3420 [00:04<00:01, 526.81it/s]
Tokenizing inputs:  73%|███████▎  | 2511/3420 [00:05<00:02, 380.90it/s]
Tokenizing inputs:  75%|███████▍  | 2556/3420 [00:05<00:02, 388.48it/s]
Tokenizing inputs:  76%|███████▋  | 2610/3420 [00:05<00:01, 422.75it/s]
Tokenizing inputs:  78%|███████▊  | 2660/3420 [00:05<00:01, 442.05it/s]
Tokenizing inputs:  79%|███████▉  | 2708/3420 [00:05<00:01, 385.03it/s]
Tokenizing inputs:  80%|████████  | 2750/3420 [00:06<00:03, 217.31it/s]
Tokenizing inputs:  81%|████████▏ | 2783/3420 [00:06<00:03, 179.22it/s]
Tokenizing inputs:  82%|████████▏ | 2809/3420 [00:06<00:03, 163.04it/s]
Tokenizing inputs:  83%|████████▎ | 2831/3420 [00:06<00:03, 158.13it/s]
Tokenizing inputs:  83%|████████▎ | 2851/3420 [00:06<00:03, 161.34it/s]
Tokenizing inputs:  84%|████████▍ | 2870/3420 [00:06<00:03, 162.84it/s]
Tokenizing inputs:  85%|████████▍ | 2895/3420 [00:07<00:02, 180.63it/s]
Tokenizing inputs:  86%|████████▌ | 2931/3420 [00:07<00:02, 221.80it/s]
Tokenizing inputs:  87%|████████▋ | 2981/3420 [00:07<00:01, 289.84it/s]
Tokenizing inputs:  89%|████████▊ | 3033/3420 [00:07<00:01, 348.34it/s]
Tokenizing inputs:  90%|█████████ | 3082/3420 [00:07<00:00, 385.29it/s]
Tokenizing inputs:  91%|█████████▏| 3127/3420 [00:07<00:00, 403.01it/s]
Tokenizing inputs:  93%|█████████▎| 3170/3420 [00:07<00:00, 391.85it/s]
Tokenizing inputs:  94%|█████████▍| 3231/3420 [00:07<00:00, 452.34it/s]
Tokenizing inputs:  96%|█████████▌| 3278/3420 [00:07<00:00, 456.84it/s]
Tokenizing inputs:  97%|█████████▋| 3325/3420 [00:08<00:00, 334.94it/s]
Tokenizing inputs:  98%|█████████▊| 3364/3420 [00:08<00:00, 254.47it/s]
Tokenizing inputs: 100%|██████████| 3420/3420 [00:08<00:00, 407.74it/s]
+
Running loglikelihood requests:   0%|          | 0/3420 [00:00<?, ?it/s][W724 07:07:06.273798252 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3709861888 bytes (free: 3373334528, total: 25323503616).
+
Running loglikelihood requests:   0%|          | 1/3420 [00:03<3:25:09,  3.60s/it][W724 07:07:08.102283239 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3590324224 bytes (free: 2310078464, total: 25323503616).
+[W724 07:07:11.318059393 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3590324224 bytes (free: 2312175616, total: 25323503616).
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/__main__.py", line 14, in <module>
+    cli_evaluate()
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/__main__.py", line 10, in cli_evaluate
+    parser.execute(args)
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/_cli/harness.py", line 60, in execute
+    args.func(args)
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/_cli/run.py", line 391, in _execute
+    results = simple_evaluate(
+              ^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/utils.py", line 575, in _wrapper
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/evaluator.py", line 358, in simple_evaluate
+    results = evaluate(
+              ^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/utils.py", line 575, in _wrapper
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/evaluator.py", line 596, in evaluate
+    resps = getattr(lm, reqtype)(cloned_reqs)
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/models/huggingface.py", line 1227, in loglikelihood
+    return super().loglikelihood(requests, disable_tqdm=disable_tqdm)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/api/model.py", line 446, in loglikelihood
+    return self._loglikelihood_tokens(new_reqs, disable_tqdm=disable_tqdm)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/models/huggingface.py", line 1507, in _loglikelihood_tokens
+    self._model_call(batched_inps, **call_kwargs),
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/models/huggingface.py", line 1154, in _model_call
+    return self.model(inps).logits
+           ^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/transformers/utils/generic.py", line 911, in wrapper
+    output = func(self, *args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/transformers/models/qwen2/modeling_qwen2.py", line 475, in forward
+    logits = self.lm_head(hidden_states[:, slice_indices, :])
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1789, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/linear.py", line 134, in forward
+    return F.linear(input, self.weight, self.bias)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 3.34 GiB. GPU 0 has a total capacity of 23.58 GiB of which 2.15 GiB is free. Process 1398907 has 21.42 GiB memory in use. Of the allocated memory 18.36 GiB is allocated by PyTorch, and 2.76 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
+
Running loglikelihood requests:   0%|          | 16/3420 [00:06<22:48,  2.49it/s] 
+
+
+########## m0 / gsm8k_cot ##########
+/usr/bin/python -m lm_eval --model hf --model_args pretrained=Qwen/Qwen2.5-7B-Instruct,dtype=bfloat16 --tasks gsm8k_cot --num_fewshot 8 --batch_size 4 --seed 0 --output_path /root/steering-resistance/results/full_7b/capability/m0/gsm8k_cot --gen_kwargs do_sample=False --apply_chat_template --fewshot_as_multiturn --limit 200
+/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
+  warnings.warn(
+2026-07-24:07:07:13 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-24:07:07:21 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot']
+2026-07-24:07:07:23 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 0 | Setting torch manual seed to 0 | Setting fewshot manual seed to 0
+2026-07-24:07:07:23 WARNING  [evaluator:226] generation_kwargs: {'do_sample': False} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-24:07:07:23 INFO     [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16'}
+2026-07-24:07:07:26 INFO     [models.huggingface:286] Using device 'cuda:0'
+2026-07-24:07:07:27 INFO     [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights:   1%|          | 2/339 [00:00<00:34,  9.85it/s]
Loading weights:   5%|▌         | 18/339 [00:00<00:04, 66.14it/s]
Loading weights:  12%|█▏        | 39/339 [00:00<00:02, 115.57it/s]
Loading weights:  16%|█▌        | 54/339 [00:00<00:02, 121.92it/s]
Loading weights:  22%|██▏       | 76/339 [00:00<00:01, 150.89it/s]
Loading weights:  27%|██▋       | 93/339 [00:00<00:01, 150.53it/s]
Loading weights:  33%|███▎      | 111/339 [00:00<00:01, 155.32it/s]
Loading weights:  38%|███▊      | 128/339 [00:00<00:01, 152.75it/s]
Loading weights:  43%|████▎     | 147/339 [00:01<00:01, 161.08it/s]
Loading weights:  48%|████▊     | 164/339 [00:01<00:01, 155.54it/s]
Loading weights:  54%|█████▍    | 184/339 [00:01<00:00, 167.37it/s]
Loading weights:  60%|█████▉    | 202/339 [00:01<00:00, 162.65it/s]
Loading weights:  65%|██████▍   | 219/339 [00:01<00:00, 147.37it/s]
Loading weights:  69%|██████▉   | 235/339 [00:01<00:00, 138.48it/s]
Loading weights:  76%|███████▌  | 256/339 [00:01<00:00, 134.67it/s]
Loading weights:  80%|███████▉  | 270/339 [00:01<00:00, 130.64it/s]
Loading weights:  86%|████████▌ | 292/339 [00:02<00:00, 143.60it/s]
Loading weights:  92%|█████████▏| 312/339 [00:02<00:00, 156.82it/s]
Loading weights:  97%|█████████▋| 329/339 [00:02<00:00, 145.99it/s]
Loading weights: 100%|██████████| 339/339 [00:02<00:00, 143.99it/s]
+
Generating train split:   0%|          | 0/7473 [00:00<?, ? examples/s]
Generating train split: 100%|██████████| 7473/7473 [00:00<00:00, 325078.14 examples/s]
+
Generating test split:   0%|          | 0/1319 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 1319/1319 [00:00<00:00, 249775.93 examples/s]
+2026-07-24:07:07:35 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-24:07:07:35 INFO     [evaluator_utils:480] Task: gsm8k_cot (gsm8k/gsm8k-cot.yaml)
+2026-07-24:07:07:35 INFO     [evaluator:314] gsm8k_cot: Using gen_kwargs: {'do_sample': False, 'until': ['Q:', '</s>', '<|im_end|>']}
+2026-07-24:07:07:35 WARNING  [evaluator:333] Overwriting default num_fewshot of gsm8k_cot from 8 to 8
+2026-07-24:07:07:35 INFO     [api.task:312] Building contexts for gsm8k_cot on rank 0...
+
  0%|          | 0/200 [00:00<?, ?it/s]
  4%|▎         | 7/200 [00:00<00:02, 67.89it/s]
  9%|▉         | 18/200 [00:00<00:02, 88.55it/s]
 14%|█▍        | 29/200 [00:00<00:01, 95.06it/s]
 20%|██        | 40/200 [00:00<00:01, 98.12it/s]
 26%|██▌       | 51/200 [00:00<00:01, 99.85it/s]
 31%|███       | 62/200 [00:00<00:01, 100.96it/s]
 36%|███▋      | 73/200 [00:00<00:01, 101.60it/s]
 42%|████▏     | 84/200 [00:00<00:01, 102.07it/s]
 48%|████▊     | 95/200 [00:00<00:01, 102.39it/s]
 53%|█████▎    | 106/200 [00:01<00:00, 102.62it/s]
 58%|█████▊    | 117/200 [00:01<00:00, 102.76it/s]
 64%|██████▍   | 128/200 [00:01<00:00, 102.89it/s]
 70%|██████▉   | 139/200 [00:01<00:00, 103.03it/s]
 75%|███████▌  | 150/200 [00:01<00:00, 103.07it/s]
 80%|████████  | 161/200 [00:01<00:00, 103.08it/s]
 86%|████████▌ | 172/200 [00:01<00:00, 103.16it/s]
 92%|█████████▏| 183/200 [00:01<00:00, 103.17it/s]
 97%|█████████▋| 194/200 [00:01<00:00, 103.22it/s]
100%|██████████| 200/200 [00:01<00:00, 101.22it/s]
+2026-07-24:07:07:37 INFO     [evaluator:585] Running generate_until requests
+
Running generate_until requests:   0%|          | 0/200 [00:00<?, ?it/s]
Running generate_until requests:   0%|          | 1/200 [00:09<32:36,  9.83s/it]
Running generate_until requests:   2%|▎         | 5/200 [00:17<10:13,  3.15s/it]
Running generate_until requests:   4%|▍         | 9/200 [00:26<08:13,  2.58s/it]
Running generate_until requests:   6%|▋         | 13/200 [00:34<07:24,  2.38s/it]
Running generate_until requests:   8%|▊         | 17/200 [00:43<06:56,  2.28s/it]
Running generate_until requests:  10%|█         | 21/200 [00:51<06:37,  2.22s/it]
Running generate_until requests:  12%|█▎        | 25/200 [01:00<06:22,  2.19s/it]
Running generate_until requests:  14%|█▍        | 29/200 [01:08<06:09,  2.16s/it]
Running generate_until requests:  16%|█▋        | 33/200 [01:17<05:58,  2.15s/it]
Running generate_until requests:  18%|█▊        | 37/200 [01:22<05:11,  1.91s/it]
Running generate_until requests:  20%|██        | 41/200 [01:30<05:13,  1.97s/it]
Running generate_until requests:  22%|██▎       | 45/200 [01:39<05:12,  2.02s/it]
Running generate_until requests:  24%|██▍       | 49/200 [01:47<05:09,  2.05s/it]
Running generate_until requests:  26%|██▋       | 53/200 [01:56<05:04,  2.07s/it]
Running generate_until requests:  28%|██▊       | 57/200 [02:04<04:58,  2.09s/it]
Running generate_until requests:  30%|███       | 61/200 [02:13<04:51,  2.10s/it]
Running generate_until requests:  32%|███▎      | 65/200 [02:21<04:43,  2.10s/it]
Running generate_until requests:  34%|███▍      | 69/200 [02:29<04:29,  2.06s/it]
Running generate_until requests:  36%|███▋      | 73/200 [02:38<04:23,  2.07s/it]
Running generate_until requests:  38%|███▊      | 77/200 [02:46<04:16,  2.08s/it]
Running generate_until requests:  40%|████      | 81/200 [02:54<04:08,  2.09s/it]
Running generate_until requests:  42%|████▎     | 85/200 [03:01<03:47,  1.98s/it]
Running generate_until requests:  44%|████▍     | 89/200 [03:08<03:26,  1.86s/it]
Running generate_until requests:  46%|████▋     | 93/200 [03:14<03:12,  1.79s/it]
Running generate_until requests:  48%|████▊     | 97/200 [03:21<03:01,  1.76s/it]
Running generate_until requests:  50%|█████     | 101/200 [03:29<03:04,  1.87s/it]
Running generate_until requests:  52%|█████▎    | 105/200 [03:36<02:53,  1.82s/it]
Running generate_until requests:  55%|█████▍    | 109/200 [03:41<02:27,  1.62s/it]
Running generate_until requests:  56%|█████▋    | 113/200 [03:49<02:32,  1.75s/it]
Running generate_until requests:  58%|█████▊    | 117/200 [03:57<02:28,  1.79s/it]
Running generate_until requests:  60%|██████    | 121/200 [04:05<02:28,  1.88s/it]
Running generate_until requests:  62%|██████▎   | 125/200 [04:12<02:20,  1.87s/it]
Running generate_until requests:  64%|██████▍   | 129/200 [04:19<02:05,  1.77s/it]
Running generate_until requests:  66%|██████▋   | 133/200 [04:24<01:50,  1.65s/it]
Running generate_until requests:  68%|██████▊   | 137/200 [04:29<01:36,  1.54s/it]
Running generate_until requests:  70%|███████   | 141/200 [04:37<01:37,  1.65s/it]
Running generate_until requests:  72%|███████▎  | 145/200 [04:43<01:30,  1.64s/it]
Running generate_until requests:  74%|███████▍  | 149/200 [04:51<01:28,  1.74s/it]
Running generate_until requests:  76%|███████▋  | 153/200 [04:58<01:21,  1.73s/it]
Running generate_until requests:  78%|███████▊  | 157/200 [05:05<01:13,  1.72s/it]
Running generate_until requests:  80%|████████  | 161/200 [05:12<01:07,  1.74s/it]
Running generate_until requests:  82%|████████▎ | 165/200 [05:20<01:03,  1.81s/it]
Running generate_until requests:  84%|████████▍ | 169/200 [05:26<00:54,  1.76s/it]
Running generate_until requests:  86%|████████▋ | 173/200 [05:32<00:45,  1.67s/it]
Running generate_until requests:  88%|████████▊ | 177/200 [05:38<00:37,  1.63s/it]
Running generate_until requests:  90%|█████████ | 181/200 [05:44<00:30,  1.60s/it]
Running generate_until requests:  92%|█████████▎| 185/200 [05:49<00:22,  1.47s/it]
Running generate_until requests:  94%|█████████▍| 189/200 [05:56<00:16,  1.52s/it]
Running generate_until requests:  96%|█████████▋| 193/200 [06:04<00:11,  1.69s/it]
Running generate_until requests:  98%|█████████▊| 197/200 [06:11<00:05,  1.70s/it]
Running generate_until requests: 100%|██████████| 200/200 [06:11<00:00,  1.86s/it]
+fatal: detected dubious ownership in repository at '/root/steering-resistance'
+To add an exception for this directory, call:
+
+	git config --global --add safe.directory /root/steering-resistance
+2026-07-24:07:13:50 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+hf ({'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16'}), gen_kwargs: ({'do_sample': False}), limit: 200.0, num_fewshot: 8, batch_size: 4
+|  Tasks  |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|---------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot|      3|flexible-extract|     8|exact_match|↑  | 0.78|±  |0.0294|
+|         |       |strict-match    |     8|exact_match|↑  | 0.33|±  |0.0333|
+
+
+
+########## m1 / mmlu ##########
+/usr/bin/python -m lm_eval --model hf --model_args pretrained=Qwen/Qwen2.5-7B-Instruct,dtype=bfloat16,peft=/root/steering-resistance/results/full_7b/m1_resist_adapter --tasks mmlu --num_fewshot 5 --batch_size 4 --seed 0 --output_path /root/steering-resistance/results/full_7b/capability/m1/mmlu --apply_chat_template --fewshot_as_multiturn --limit 15
+/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
+  warnings.warn(
+2026-07-24:07:13:52 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-24:07:14:00 INFO     [_cli.run:388] Selected Tasks: ['mmlu']
+2026-07-24:07:14:02 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 0 | Setting torch manual seed to 0 | Setting fewshot manual seed to 0
+2026-07-24:07:14:02 INFO     [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16', 'peft': '/root/steering-resistance/results/full_7b/m1_resist_adapter'}
+2026-07-24:07:14:05 INFO     [models.huggingface:286] Using device 'cuda:0'
+2026-07-24:07:14:06 INFO     [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights:   1%|          | 2/339 [00:00<00:34,  9.85it/s]
Loading weights:   5%|▌         | 18/339 [00:00<00:04, 69.13it/s]
Loading weights:  12%|█▏        | 41/339 [00:00<00:02, 123.65it/s]
Loading weights:  17%|█▋        | 57/339 [00:00<00:02, 135.31it/s]
Loading weights:  23%|██▎       | 77/339 [00:00<00:01, 147.27it/s]
Loading weights:  28%|██▊       | 96/339 [00:00<00:01, 158.38it/s]
Loading weights:  33%|███▎      | 113/339 [00:00<00:01, 143.25it/s]
Loading weights:  40%|████      | 136/339 [00:00<00:01, 165.49it/s]
Loading weights:  45%|████▌     | 154/339 [00:01<00:01, 157.15it/s]
Loading weights:  51%|█████     | 172/339 [00:01<00:01, 155.25it/s]
Loading weights:  57%|█████▋    | 192/339 [00:01<00:00, 166.30it/s]
Loading weights:  62%|██████▏   | 210/339 [00:01<00:00, 135.28it/s]
Loading weights:  68%|██████▊   | 232/339 [00:01<00:00, 153.58it/s]
Loading weights:  73%|███████▎  | 249/339 [00:01<00:00, 152.49it/s]
Loading weights:  78%|███████▊  | 266/339 [00:01<00:00, 140.85it/s]
Loading weights:  83%|████████▎ | 281/339 [00:02<00:00, 138.29it/s]
Loading weights:  88%|████████▊ | 298/339 [00:02<00:00, 144.97it/s]
Loading weights:  94%|█████████▎| 317/339 [00:02<00:00, 150.66it/s]
Loading weights:  99%|█████████▉| 336/339 [00:02<00:00, 160.03it/s]
Loading weights: 100%|██████████| 339/339 [00:02<00:00, 145.21it/s]
+2026-07-24:07:14:11 INFO     [models.huggingface:885] Model config indicates vocab_size='152064', but found tokenizer with vocab size '151665'. Resizing model embedding layer...
+2026-07-24:07:15:10 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-24:07:15:10 INFO     [evaluator_utils:462] Group: mmlu
+2026-07-24:07:15:10 INFO     [evaluator_utils:462]   Group: mmlu_stem
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_abstract_algebra (mmlu/default/mmlu_abstract_algebra.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_anatomy (mmlu/default/mmlu_anatomy.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_astronomy (mmlu/default/mmlu_astronomy.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_college_biology (mmlu/default/mmlu_college_biology.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_college_chemistry (mmlu/default/mmlu_college_chemistry.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_college_computer_science (mmlu/default/mmlu_college_computer_science.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_college_mathematics (mmlu/default/mmlu_college_mathematics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_college_physics (mmlu/default/mmlu_college_physics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_computer_security (mmlu/default/mmlu_computer_security.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_conceptual_physics (mmlu/default/mmlu_conceptual_physics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_electrical_engineering (mmlu/default/mmlu_electrical_engineering.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_elementary_mathematics (mmlu/default/mmlu_elementary_mathematics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_biology (mmlu/default/mmlu_high_school_biology.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_chemistry (mmlu/default/mmlu_high_school_chemistry.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_computer_science (mmlu/default/mmlu_high_school_computer_science.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_mathematics (mmlu/default/mmlu_high_school_mathematics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_physics (mmlu/default/mmlu_high_school_physics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_statistics (mmlu/default/mmlu_high_school_statistics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_machine_learning (mmlu/default/mmlu_machine_learning.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:462]   Group: mmlu_other
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_business_ethics (mmlu/default/mmlu_business_ethics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_clinical_knowledge (mmlu/default/mmlu_clinical_knowledge.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_college_medicine (mmlu/default/mmlu_college_medicine.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_global_facts (mmlu/default/mmlu_global_facts.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_human_aging (mmlu/default/mmlu_human_aging.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_management (mmlu/default/mmlu_management.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_marketing (mmlu/default/mmlu_marketing.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_medical_genetics (mmlu/default/mmlu_medical_genetics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_miscellaneous (mmlu/default/mmlu_miscellaneous.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_nutrition (mmlu/default/mmlu_nutrition.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_professional_accounting (mmlu/default/mmlu_professional_accounting.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_professional_medicine (mmlu/default/mmlu_professional_medicine.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_virology (mmlu/default/mmlu_virology.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:462]   Group: mmlu_social_sciences
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_econometrics (mmlu/default/mmlu_econometrics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_geography (mmlu/default/mmlu_high_school_geography.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_government_and_politics (mmlu/default/mmlu_high_school_government_and_politics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_macroeconomics (mmlu/default/mmlu_high_school_macroeconomics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_microeconomics (mmlu/default/mmlu_high_school_microeconomics.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_psychology (mmlu/default/mmlu_high_school_psychology.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_human_sexuality (mmlu/default/mmlu_human_sexuality.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_professional_psychology (mmlu/default/mmlu_professional_psychology.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_public_relations (mmlu/default/mmlu_public_relations.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_security_studies (mmlu/default/mmlu_security_studies.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_sociology (mmlu/default/mmlu_sociology.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_us_foreign_policy (mmlu/default/mmlu_us_foreign_policy.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:462]   Group: mmlu_humanities
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_formal_logic (mmlu/default/mmlu_formal_logic.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_european_history (mmlu/default/mmlu_high_school_european_history.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_us_history (mmlu/default/mmlu_high_school_us_history.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_high_school_world_history (mmlu/default/mmlu_high_school_world_history.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_international_law (mmlu/default/mmlu_international_law.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_jurisprudence (mmlu/default/mmlu_jurisprudence.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_logical_fallacies (mmlu/default/mmlu_logical_fallacies.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_moral_disputes (mmlu/default/mmlu_moral_disputes.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_moral_scenarios (mmlu/default/mmlu_moral_scenarios.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_philosophy (mmlu/default/mmlu_philosophy.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_prehistory (mmlu/default/mmlu_prehistory.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_professional_law (mmlu/default/mmlu_professional_law.yaml)
+2026-07-24:07:15:10 INFO     [evaluator_utils:470]     Task: mmlu_world_religions (mmlu/default/mmlu_world_religions.yaml)
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_abstract_algebra from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_anatomy from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_astronomy from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_biology from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_chemistry from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_computer_science from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_mathematics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_physics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_computer_security from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_conceptual_physics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_electrical_engineering from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_elementary_mathematics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_biology from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_chemistry from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_computer_science from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_mathematics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_physics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_statistics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_machine_learning from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_business_ethics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_clinical_knowledge from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_medicine from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_global_facts from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_aging from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_management from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_marketing from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_medical_genetics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_miscellaneous from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_nutrition from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_accounting from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_medicine from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_virology from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_econometrics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_geography from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_government_and_politics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_macroeconomics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_microeconomics from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_psychology from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_sexuality from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_psychology from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_public_relations from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_security_studies from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_sociology from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_us_foreign_policy from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_formal_logic from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_european_history from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_us_history from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_world_history from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_international_law from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_jurisprudence from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_logical_fallacies from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_disputes from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_scenarios from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_philosophy from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_prehistory from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_law from None to 5
+2026-07-24:07:15:10 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_world_religions from None to 5
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_abstract_algebra on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
 87%|████████▋ | 13/15 [00:00<00:00, 127.92it/s]
100%|██████████| 15/15 [00:00<00:00, 133.70it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_anatomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.46it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_astronomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.13it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_college_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.70it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_college_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.03it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_college_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.49it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_college_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.39it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_college_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.62it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_computer_security on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.29it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_conceptual_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.47it/s]
+2026-07-24:07:15:10 INFO     [api.task:312] Building contexts for mmlu_electrical_engineering on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.53it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_elementary_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.25it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_high_school_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.36it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_high_school_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 185.51it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_high_school_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.53it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_high_school_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.31it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_high_school_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.02it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_high_school_statistics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.32it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_machine_learning on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.76it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_business_ethics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.06it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_clinical_knowledge on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.86it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_college_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.44it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_global_facts on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.54it/s]
+2026-07-24:07:15:11 INFO     [api.task:312] Building contexts for mmlu_human_aging on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.93it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_management on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.09it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_marketing on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.53it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_medical_genetics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.09it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_miscellaneous on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.79it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_nutrition on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 187.00it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_professional_accounting on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 189.48it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_professional_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.81it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_virology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.63it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_econometrics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.46it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_high_school_geography on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.57it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_high_school_government_and_politics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.17it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_high_school_macroeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.86it/s]
+2026-07-24:07:15:12 INFO     [api.task:312] Building contexts for mmlu_high_school_microeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.29it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_high_school_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.61it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_human_sexuality on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.39it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_professional_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.26it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_public_relations on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.43it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_security_studies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.44it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_sociology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.22it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_us_foreign_policy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.48it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_formal_logic on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.20it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_high_school_european_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.45it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_high_school_us_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.43it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_high_school_world_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.70it/s]
+2026-07-24:07:15:13 INFO     [api.task:312] Building contexts for mmlu_international_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.88it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_jurisprudence on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.23it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_logical_fallacies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.81it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_moral_disputes on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.04it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_moral_scenarios on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.97it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_philosophy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.98it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_prehistory on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.91it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_professional_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.92it/s]
+2026-07-24:07:15:14 INFO     [api.task:312] Building contexts for mmlu_world_religions on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.54it/s]
+2026-07-24:07:15:14 INFO     [evaluator:585] Running loglikelihood requests
+
Tokenizing inputs:   0%|          | 0/3420 [00:00<?, ?it/s]
Tokenizing inputs:   2%|▏         | 69/3420 [00:00<00:04, 684.29it/s]
Tokenizing inputs:   4%|▍         | 138/3420 [00:00<00:05, 632.03it/s]
Tokenizing inputs:   6%|▌         | 202/3420 [00:00<00:06, 534.67it/s]
Tokenizing inputs:   8%|▊         | 257/3420 [00:00<00:05, 529.92it/s]
Tokenizing inputs:   9%|▉         | 311/3420 [00:00<00:06, 498.28it/s]
Tokenizing inputs:  11%|█         | 362/3420 [00:00<00:07, 422.05it/s]
Tokenizing inputs:  12%|█▏        | 407/3420 [00:00<00:07, 428.79it/s]
Tokenizing inputs:  13%|█▎        | 456/3420 [00:00<00:06, 443.27it/s]
Tokenizing inputs:  15%|█▌        | 513/3420 [00:01<00:06, 477.96it/s]
Tokenizing inputs:  17%|█▋        | 585/3420 [00:01<00:05, 546.83it/s]
Tokenizing inputs:  19%|█▉        | 648/3420 [00:01<00:04, 570.79it/s]
Tokenizing inputs:  21%|██        | 707/3420 [00:01<00:04, 547.74it/s]
Tokenizing inputs:  22%|██▏       | 763/3420 [00:01<00:04, 535.27it/s]
Tokenizing inputs:  24%|██▍       | 818/3420 [00:01<00:04, 532.92it/s]
Tokenizing inputs:  25%|██▌       | 872/3420 [00:01<00:05, 467.14it/s]
Tokenizing inputs:  27%|██▋       | 921/3420 [00:01<00:05, 427.96it/s]
Tokenizing inputs:  28%|██▊       | 971/3420 [00:01<00:05, 446.07it/s]
Tokenizing inputs:  30%|██▉       | 1017/3420 [00:02<00:05, 443.01it/s]
Tokenizing inputs:  31%|███       | 1063/3420 [00:02<00:05, 405.05it/s]
Tokenizing inputs:  32%|███▏      | 1105/3420 [00:02<00:05, 395.71it/s]
Tokenizing inputs:  34%|███▎      | 1146/3420 [00:02<00:05, 397.23it/s]
Tokenizing inputs:  35%|███▍      | 1192/3420 [00:02<00:05, 413.12it/s]
Tokenizing inputs:  37%|███▋      | 1253/3420 [00:02<00:04, 466.27it/s]
Tokenizing inputs:  38%|███▊      | 1301/3420 [00:02<00:04, 463.04it/s]
Tokenizing inputs:  40%|███▉      | 1358/3420 [00:02<00:04, 492.68it/s]
Tokenizing inputs:  42%|████▏     | 1428/3420 [00:02<00:03, 551.93it/s]
Tokenizing inputs:  44%|████▍     | 1507/3420 [00:03<00:03, 618.37it/s]
Tokenizing inputs:  46%|████▌     | 1570/3420 [00:03<00:03, 611.34it/s]
Tokenizing inputs:  48%|████▊     | 1647/3420 [00:03<00:02, 656.28it/s]
Tokenizing inputs:  50%|█████     | 1713/3420 [00:03<00:02, 634.28it/s]
Tokenizing inputs:  52%|█████▏    | 1777/3420 [00:03<00:03, 540.53it/s]
Tokenizing inputs:  54%|█████▎    | 1834/3420 [00:03<00:03, 427.02it/s]
Tokenizing inputs:  55%|█████▌    | 1882/3420 [00:03<00:03, 409.12it/s]
Tokenizing inputs:  57%|█████▋    | 1942/3420 [00:03<00:03, 451.16it/s]
Tokenizing inputs:  58%|█████▊    | 1991/3420 [00:04<00:03, 456.76it/s]
Tokenizing inputs:  60%|█████▉    | 2050/3420 [00:04<00:02, 489.96it/s]
Tokenizing inputs:  61%|██████▏   | 2102/3420 [00:04<00:02, 493.99it/s]
Tokenizing inputs:  63%|██████▎   | 2163/3420 [00:04<00:02, 524.43it/s]
Tokenizing inputs:  65%|██████▍   | 2221/3420 [00:04<00:02, 539.94it/s]
Tokenizing inputs:  67%|██████▋   | 2277/3420 [00:04<00:02, 523.06it/s]
Tokenizing inputs:  69%|██████▊   | 2344/3420 [00:04<00:01, 561.93it/s]
Tokenizing inputs:  70%|███████   | 2401/3420 [00:04<00:02, 509.39it/s]
Tokenizing inputs:  72%|███████▏  | 2457/3420 [00:04<00:01, 522.74it/s]
Tokenizing inputs:  73%|███████▎  | 2511/3420 [00:05<00:02, 376.15it/s]
Tokenizing inputs:  75%|███████▍  | 2555/3420 [00:05<00:02, 383.17it/s]
Tokenizing inputs:  76%|███████▋  | 2608/3420 [00:05<00:01, 416.90it/s]
Tokenizing inputs:  78%|███████▊  | 2659/3420 [00:05<00:01, 437.89it/s]
Tokenizing inputs:  79%|███████▉  | 2707/3420 [00:05<00:01, 387.65it/s]
Tokenizing inputs:  80%|████████  | 2749/3420 [00:06<00:03, 216.00it/s]
Tokenizing inputs:  81%|████████▏ | 2782/3420 [00:06<00:03, 177.64it/s]
Tokenizing inputs:  82%|████████▏ | 2808/3420 [00:06<00:03, 161.39it/s]
Tokenizing inputs:  83%|████████▎ | 2830/3420 [00:06<00:03, 156.07it/s]
Tokenizing inputs:  83%|████████▎ | 2850/3420 [00:06<00:03, 159.01it/s]
Tokenizing inputs:  84%|████████▍ | 2869/3420 [00:06<00:03, 160.63it/s]
Tokenizing inputs:  85%|████████▍ | 2893/3420 [00:07<00:02, 176.52it/s]
Tokenizing inputs:  86%|████████▌ | 2928/3420 [00:07<00:02, 215.65it/s]
Tokenizing inputs:  87%|████████▋ | 2976/3420 [00:07<00:01, 280.20it/s]
Tokenizing inputs:  89%|████████▊ | 3028/3420 [00:07<00:01, 341.13it/s]
Tokenizing inputs:  90%|████████▉ | 3077/3420 [00:07<00:00, 380.71it/s]
Tokenizing inputs:  91%|█████████▏| 3123/3420 [00:07<00:00, 402.15it/s]
Tokenizing inputs:  93%|█████████▎| 3166/3420 [00:07<00:00, 391.12it/s]
Tokenizing inputs:  94%|█████████▍| 3223/3420 [00:07<00:00, 440.30it/s]
Tokenizing inputs:  96%|█████████▌| 3274/3420 [00:07<00:00, 456.95it/s]
Tokenizing inputs:  97%|█████████▋| 3321/3420 [00:08<00:00, 347.58it/s]
Tokenizing inputs:  98%|█████████▊| 3361/3420 [00:08<00:00, 248.95it/s]
Tokenizing inputs: 100%|██████████| 3420/3420 [00:08<00:00, 402.54it/s]
+
Running loglikelihood requests:   0%|          | 0/3420 [00:00<?, ?it/s][W724 07:15:27.387443972 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3699376128 bytes (free: 967901184, total: 25323503616).
+
Running loglikelihood requests:   0%|          | 1/3420 [00:04<4:12:38,  4.43s/it][W724 07:15:29.640460842 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3581935616 bytes (free: 1183907840, total: 25323503616).
+[W724 07:15:29.640639731 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 3581935616 bytes (free: 1183907840, total: 25323503616).
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/__main__.py", line 14, in <module>
+    cli_evaluate()
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/__main__.py", line 10, in cli_evaluate
+    parser.execute(args)
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/_cli/harness.py", line 60, in execute
+    args.func(args)
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/_cli/run.py", line 391, in _execute
+    results = simple_evaluate(
+              ^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/utils.py", line 575, in _wrapper
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/evaluator.py", line 358, in simple_evaluate
+    results = evaluate(
+              ^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/utils.py", line 575, in _wrapper
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/evaluator.py", line 596, in evaluate
+    resps = getattr(lm, reqtype)(cloned_reqs)
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/models/huggingface.py", line 1227, in loglikelihood
+    return super().loglikelihood(requests, disable_tqdm=disable_tqdm)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/api/model.py", line 446, in loglikelihood
+    return self._loglikelihood_tokens(new_reqs, disable_tqdm=disable_tqdm)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/lm_eval/models/huggingface.py", line 1506, in _loglikelihood_tokens
+    multi_logits = F.log_softmax(
+                   ^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/functional.py", line 2315, in log_softmax
+    ret = input.log_softmax(dim)
+          ^^^^^^^^^^^^^^^^^^^^^^
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 3.34 GiB. GPU 0 has a total capacity of 23.58 GiB of which 1.10 GiB is free. Process 1401096 has 22.47 GiB memory in use. Of the allocated memory 21.12 GiB is allocated by PyTorch, and 1.05 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
+
Running loglikelihood requests:   0%|          | 16/3420 [00:06<23:42,  2.39it/s] 
+
+
+########## m1 / gsm8k_cot ##########
+/usr/bin/python -m lm_eval --model hf --model_args pretrained=Qwen/Qwen2.5-7B-Instruct,dtype=bfloat16,peft=/root/steering-resistance/results/full_7b/m1_resist_adapter --tasks gsm8k_cot --num_fewshot 8 --batch_size 4 --seed 0 --output_path /root/steering-resistance/results/full_7b/capability/m1/gsm8k_cot --gen_kwargs do_sample=False --apply_chat_template --fewshot_as_multiturn --limit 200
+/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
+  warnings.warn(
+2026-07-24:07:15:31 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-24:07:15:40 INFO     [_cli.run:388] Selected Tasks: ['gsm8k_cot']
+2026-07-24:07:15:41 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 0 | Setting torch manual seed to 0 | Setting fewshot manual seed to 0
+2026-07-24:07:15:41 WARNING  [evaluator:226] generation_kwargs: {'do_sample': False} specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!
+2026-07-24:07:15:41 INFO     [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16', 'peft': '/root/steering-resistance/results/full_7b/m1_resist_adapter'}
+2026-07-24:07:15:44 INFO     [models.huggingface:286] Using device 'cuda:0'
+2026-07-24:07:15:46 INFO     [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights:   1%|          | 2/339 [00:00<00:35,  9.39it/s]
Loading weights:   6%|▋         | 22/339 [00:00<00:03, 85.59it/s]
Loading weights:  14%|█▎        | 46/339 [00:00<00:02, 140.14it/s]
Loading weights:  20%|█▉        | 67/339 [00:00<00:01, 163.61it/s]
Loading weights:  26%|██▋       | 89/339 [00:00<00:01, 179.88it/s]
Loading weights:  34%|███▎      | 114/339 [00:00<00:01, 198.53it/s]
Loading weights:  41%|████      | 138/339 [00:00<00:00, 208.84it/s]
Loading weights:  48%|████▊     | 162/339 [00:00<00:00, 216.03it/s]
Loading weights:  55%|█████▍    | 185/339 [00:01<00:00, 218.61it/s]
Loading weights:  62%|██████▏   | 210/339 [00:01<00:00, 211.47it/s]
Loading weights:  69%|██████▉   | 234/339 [00:01<00:00, 217.43it/s]
Loading weights:  76%|███████▌  | 258/339 [00:01<00:00, 212.48it/s]
Loading weights:  83%|████████▎ | 282/339 [00:01<00:00, 216.97it/s]
Loading weights:  90%|████████▉ | 305/339 [00:01<00:00, 218.79it/s]
Loading weights:  97%|█████████▋| 330/339 [00:01<00:00, 224.03it/s]
Loading weights: 100%|██████████| 339/339 [00:01<00:00, 198.62it/s]
+2026-07-24:07:15:49 INFO     [models.huggingface:885] Model config indicates vocab_size='152064', but found tokenizer with vocab size '151665'. Resizing model embedding layer...
+2026-07-24:07:15:53 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-24:07:15:53 INFO     [evaluator_utils:480] Task: gsm8k_cot (gsm8k/gsm8k-cot.yaml)
+2026-07-24:07:15:53 INFO     [evaluator:314] gsm8k_cot: Using gen_kwargs: {'do_sample': False, 'until': ['Q:', '</s>', '<|im_end|>']}
+2026-07-24:07:15:53 WARNING  [evaluator:333] Overwriting default num_fewshot of gsm8k_cot from 8 to 8
+2026-07-24:07:15:53 INFO     [api.task:312] Building contexts for gsm8k_cot on rank 0...
+
  0%|          | 0/200 [00:00<?, ?it/s]
  4%|▍         | 9/200 [00:00<00:02, 84.88it/s]
 10%|▉         | 19/200 [00:00<00:01, 91.24it/s]
 14%|█▍        | 29/200 [00:00<00:01, 93.17it/s]
 20%|█▉        | 39/200 [00:00<00:01, 94.01it/s]
 24%|██▍       | 49/200 [00:00<00:01, 94.88it/s]
 30%|██▉       | 59/200 [00:00<00:01, 95.39it/s]
 34%|███▍      | 69/200 [00:00<00:01, 95.94it/s]
 40%|███▉      | 79/200 [00:00<00:01, 96.17it/s]
 44%|████▍     | 89/200 [00:00<00:01, 96.53it/s]
 50%|████▉     | 99/200 [00:01<00:01, 96.84it/s]
 55%|█████▍    | 109/200 [00:01<00:00, 96.98it/s]
 60%|█████▉    | 119/200 [00:01<00:00, 97.16it/s]
 64%|██████▍   | 129/200 [00:01<00:00, 97.38it/s]
 70%|██████▉   | 139/200 [00:01<00:00, 97.47it/s]
 74%|███████▍  | 149/200 [00:01<00:00, 97.66it/s]
 80%|███████▉  | 159/200 [00:01<00:00, 97.56it/s]
 84%|████████▍ | 169/200 [00:01<00:00, 97.41it/s]
 90%|████████▉ | 179/200 [00:01<00:00, 97.57it/s]
 94%|█████████▍| 189/200 [00:01<00:00, 97.72it/s]
100%|█████████▉| 199/200 [00:02<00:00, 97.54it/s]
100%|██████████| 200/200 [00:02<00:00, 96.46it/s]
+2026-07-24:07:15:55 INFO     [evaluator:585] Running generate_until requests
+
Running generate_until requests:   0%|          | 0/200 [00:00<?, ?it/s]
Running generate_until requests:   0%|          | 1/200 [00:14<47:36, 14.35s/it]
Running generate_until requests:   2%|▎         | 5/200 [00:23<12:56,  3.98s/it]
Running generate_until requests:   4%|▍         | 9/200 [00:35<11:05,  3.49s/it]
Running generate_until requests:   6%|▋         | 13/200 [00:46<10:00,  3.21s/it]
Running generate_until requests:   8%|▊         | 17/200 [00:54<08:22,  2.75s/it]
Running generate_until requests:  10%|█         | 21/200 [01:08<08:54,  2.99s/it]
Running generate_until requests:  12%|█▎        | 25/200 [01:20<08:37,  2.96s/it]
Running generate_until requests:  14%|█▍        | 29/200 [01:28<07:43,  2.71s/it]
Running generate_until requests:  16%|█▋        | 33/200 [01:38<07:22,  2.65s/it]
Running generate_until requests:  18%|█▊        | 37/200 [01:45<06:25,  2.37s/it]
Running generate_until requests:  20%|██        | 41/200 [01:53<05:47,  2.19s/it]
Running generate_until requests:  22%|██▎       | 45/200 [02:02<05:41,  2.21s/it]
Running generate_until requests:  24%|██▍       | 49/200 [02:14<06:09,  2.45s/it]
Running generate_until requests:  26%|██▋       | 53/200 [02:27<06:38,  2.71s/it]
Running generate_until requests:  28%|██▊       | 57/200 [02:37<06:22,  2.67s/it]
Running generate_until requests:  30%|███       | 61/200 [02:47<06:03,  2.62s/it]
Running generate_until requests:  32%|███▎      | 65/200 [02:55<05:23,  2.40s/it]
Running generate_until requests:  34%|███▍      | 69/200 [03:04<05:14,  2.40s/it]
Running generate_until requests:  36%|███▋      | 73/200 [03:12<04:44,  2.24s/it]
Running generate_until requests:  38%|███▊      | 77/200 [03:22<04:48,  2.34s/it]
Running generate_until requests:  40%|████      | 81/200 [03:30<04:22,  2.21s/it]
Running generate_until requests:  42%|████▎     | 85/200 [03:39<04:21,  2.27s/it]
Running generate_until requests:  44%|████▍     | 89/200 [03:46<03:53,  2.11s/it]
Running generate_until requests:  46%|████▋     | 93/200 [03:55<03:45,  2.11s/it]
Running generate_until requests:  48%|████▊     | 97/200 [04:06<03:56,  2.29s/it]
Running generate_until requests:  50%|█████     | 101/200 [04:13<03:31,  2.14s/it]
Running generate_until requests:  52%|█████▎    | 105/200 [04:21<03:20,  2.11s/it]
Running generate_until requests:  55%|█████▍    | 109/200 [04:28<03:06,  2.05s/it]
Running generate_until requests:  56%|█████▋    | 113/200 [04:40<03:21,  2.32s/it]
Running generate_until requests:  58%|█████▊    | 117/200 [04:50<03:13,  2.33s/it]
Running generate_until requests:  60%|██████    | 121/200 [04:59<03:03,  2.33s/it]
Running generate_until requests:  62%|██████▎   | 125/200 [05:05<02:38,  2.11s/it]
Running generate_until requests:  64%|██████▍   | 129/200 [05:12<02:21,  2.00s/it]
Running generate_until requests:  66%|██████▋   | 133/200 [05:18<02:04,  1.85s/it]
Running generate_until requests:  68%|██████▊   | 137/200 [05:24<01:47,  1.71s/it]
Running generate_until requests:  70%|███████   | 141/200 [05:32<01:45,  1.79s/it]
Running generate_until requests:  72%|███████▎  | 145/200 [05:38<01:36,  1.75s/it]
Running generate_until requests:  74%|███████▍  | 149/200 [05:52<01:54,  2.25s/it]
Running generate_until requests:  76%|███████▋  | 153/200 [05:59<01:39,  2.11s/it]
Running generate_until requests:  78%|███████▊  | 157/200 [06:05<01:22,  1.92s/it]
Running generate_until requests:  80%|████████  | 161/200 [06:15<01:21,  2.10s/it]
Running generate_until requests:  82%|████████▎ | 165/200 [06:26<01:19,  2.28s/it]
Running generate_until requests:  84%|████████▍ | 169/200 [06:32<01:03,  2.04s/it]
Running generate_until requests:  86%|████████▋ | 173/200 [06:42<00:58,  2.16s/it]
Running generate_until requests:  88%|████████▊ | 177/200 [06:48<00:45,  1.97s/it]
Running generate_until requests:  90%|█████████ | 181/200 [06:54<00:35,  1.85s/it]
Running generate_until requests:  92%|█████████▎| 185/200 [06:59<00:25,  1.69s/it]
Running generate_until requests:  94%|█████████▍| 189/200 [07:11<00:22,  2.07s/it]
Running generate_until requests:  96%|█████████▋| 193/200 [07:25<00:17,  2.47s/it]
Running generate_until requests:  98%|█████████▊| 197/200 [07:35<00:07,  2.53s/it]
Running generate_until requests: 100%|██████████| 200/200 [07:35<00:00,  2.28s/it]
+fatal: detected dubious ownership in repository at '/root/steering-resistance'
+To add an exception for this directory, call:
+
+	git config --global --add safe.directory /root/steering-resistance
+2026-07-24:07:23:32 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+hf ({'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16', 'peft': '/root/steering-resistance/results/full_7b/m1_resist_adapter'}), gen_kwargs: ({'do_sample': False}), limit: 200.0, num_fewshot: 8, batch_size: 4
+|  Tasks  |Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
+|---------|------:|----------------|-----:|-----------|---|----:|---|-----:|
+|gsm8k_cot|      3|flexible-extract|     8|exact_match|↑  |0.795|±  |0.0286|
+|         |       |strict-match    |     8|exact_match|↑  |0.365|±  |0.0341|
+
+
+
+########## m0 / mmlu ##########
+/usr/bin/python -m lm_eval --model hf --model_args pretrained=Qwen/Qwen2.5-7B-Instruct,dtype=bfloat16 --tasks mmlu --num_fewshot 5 --batch_size 2 --seed 0 --output_path /root/steering-resistance/results/full_7b/capability/m0/mmlu --apply_chat_template --fewshot_as_multiturn --limit 15
+/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
+  warnings.warn(
+2026-07-24:07:25:10 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-24:07:25:19 INFO     [_cli.run:388] Selected Tasks: ['mmlu']
+2026-07-24:07:25:20 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 0 | Setting torch manual seed to 0 | Setting fewshot manual seed to 0
+2026-07-24:07:25:20 INFO     [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16'}
+2026-07-24:07:25:24 INFO     [models.huggingface:286] Using device 'cuda:0'
+2026-07-24:07:25:25 INFO     [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights:   1%|          | 2/339 [00:00<00:34,  9.85it/s]
Loading weights:   5%|▌         | 18/339 [00:00<00:04, 66.34it/s]
Loading weights:  12%|█▏        | 40/339 [00:00<00:02, 110.69it/s]
Loading weights:  17%|█▋        | 59/339 [00:00<00:02, 134.54it/s]
Loading weights:  23%|██▎       | 77/339 [00:00<00:01, 142.25it/s]
Loading weights:  28%|██▊       | 95/339 [00:00<00:01, 152.21it/s]
Loading weights:  33%|███▎      | 112/339 [00:00<00:01, 157.23it/s]
Loading weights:  38%|███▊      | 129/339 [00:00<00:01, 154.27it/s]
Loading weights:  44%|████▎     | 148/339 [00:01<00:01, 161.95it/s]
Loading weights:  49%|████▊     | 165/339 [00:01<00:01, 154.23it/s]
Loading weights:  55%|█████▍    | 185/339 [00:01<00:00, 158.32it/s]
Loading weights:  60%|█████▉    | 202/339 [00:01<00:00, 159.19it/s]
Loading weights:  65%|██████▍   | 219/339 [00:01<00:00, 148.40it/s]
Loading weights:  69%|██████▉   | 235/339 [00:01<00:00, 139.28it/s]
Loading weights:  76%|███████▌  | 256/339 [00:01<00:00, 149.89it/s]
Loading weights:  80%|████████  | 272/339 [00:01<00:00, 132.04it/s]
Loading weights:  86%|████████▌ | 292/339 [00:02<00:00, 147.52it/s]
Loading weights:  91%|█████████ | 308/339 [00:02<00:00, 145.22it/s]
Loading weights:  97%|█████████▋| 328/339 [00:02<00:00, 158.58it/s]
Loading weights: 100%|██████████| 339/339 [00:02<00:00, 144.01it/s]
+2026-07-24:07:26:27 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-24:07:26:27 INFO     [evaluator_utils:462] Group: mmlu
+2026-07-24:07:26:27 INFO     [evaluator_utils:462]   Group: mmlu_stem
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_abstract_algebra (mmlu/default/mmlu_abstract_algebra.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_anatomy (mmlu/default/mmlu_anatomy.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_astronomy (mmlu/default/mmlu_astronomy.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_college_biology (mmlu/default/mmlu_college_biology.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_college_chemistry (mmlu/default/mmlu_college_chemistry.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_college_computer_science (mmlu/default/mmlu_college_computer_science.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_college_mathematics (mmlu/default/mmlu_college_mathematics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_college_physics (mmlu/default/mmlu_college_physics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_computer_security (mmlu/default/mmlu_computer_security.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_conceptual_physics (mmlu/default/mmlu_conceptual_physics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_electrical_engineering (mmlu/default/mmlu_electrical_engineering.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_elementary_mathematics (mmlu/default/mmlu_elementary_mathematics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_biology (mmlu/default/mmlu_high_school_biology.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_chemistry (mmlu/default/mmlu_high_school_chemistry.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_computer_science (mmlu/default/mmlu_high_school_computer_science.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_mathematics (mmlu/default/mmlu_high_school_mathematics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_physics (mmlu/default/mmlu_high_school_physics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_statistics (mmlu/default/mmlu_high_school_statistics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_machine_learning (mmlu/default/mmlu_machine_learning.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:462]   Group: mmlu_other
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_business_ethics (mmlu/default/mmlu_business_ethics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_clinical_knowledge (mmlu/default/mmlu_clinical_knowledge.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_college_medicine (mmlu/default/mmlu_college_medicine.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_global_facts (mmlu/default/mmlu_global_facts.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_human_aging (mmlu/default/mmlu_human_aging.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_management (mmlu/default/mmlu_management.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_marketing (mmlu/default/mmlu_marketing.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_medical_genetics (mmlu/default/mmlu_medical_genetics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_miscellaneous (mmlu/default/mmlu_miscellaneous.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_nutrition (mmlu/default/mmlu_nutrition.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_professional_accounting (mmlu/default/mmlu_professional_accounting.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_professional_medicine (mmlu/default/mmlu_professional_medicine.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_virology (mmlu/default/mmlu_virology.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:462]   Group: mmlu_social_sciences
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_econometrics (mmlu/default/mmlu_econometrics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_geography (mmlu/default/mmlu_high_school_geography.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_government_and_politics (mmlu/default/mmlu_high_school_government_and_politics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_macroeconomics (mmlu/default/mmlu_high_school_macroeconomics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_microeconomics (mmlu/default/mmlu_high_school_microeconomics.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_psychology (mmlu/default/mmlu_high_school_psychology.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_human_sexuality (mmlu/default/mmlu_human_sexuality.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_professional_psychology (mmlu/default/mmlu_professional_psychology.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_public_relations (mmlu/default/mmlu_public_relations.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_security_studies (mmlu/default/mmlu_security_studies.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_sociology (mmlu/default/mmlu_sociology.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_us_foreign_policy (mmlu/default/mmlu_us_foreign_policy.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:462]   Group: mmlu_humanities
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_formal_logic (mmlu/default/mmlu_formal_logic.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_european_history (mmlu/default/mmlu_high_school_european_history.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_us_history (mmlu/default/mmlu_high_school_us_history.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_high_school_world_history (mmlu/default/mmlu_high_school_world_history.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_international_law (mmlu/default/mmlu_international_law.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_jurisprudence (mmlu/default/mmlu_jurisprudence.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_logical_fallacies (mmlu/default/mmlu_logical_fallacies.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_moral_disputes (mmlu/default/mmlu_moral_disputes.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_moral_scenarios (mmlu/default/mmlu_moral_scenarios.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_philosophy (mmlu/default/mmlu_philosophy.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_prehistory (mmlu/default/mmlu_prehistory.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_professional_law (mmlu/default/mmlu_professional_law.yaml)
+2026-07-24:07:26:27 INFO     [evaluator_utils:470]     Task: mmlu_world_religions (mmlu/default/mmlu_world_religions.yaml)
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_abstract_algebra from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_anatomy from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_astronomy from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_biology from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_chemistry from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_computer_science from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_mathematics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_physics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_computer_security from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_conceptual_physics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_electrical_engineering from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_elementary_mathematics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_biology from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_chemistry from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_computer_science from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_mathematics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_physics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_statistics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_machine_learning from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_business_ethics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_clinical_knowledge from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_medicine from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_global_facts from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_aging from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_management from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_marketing from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_medical_genetics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_miscellaneous from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_nutrition from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_accounting from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_medicine from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_virology from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_econometrics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_geography from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_government_and_politics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_macroeconomics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_microeconomics from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_psychology from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_sexuality from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_psychology from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_public_relations from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_security_studies from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_sociology from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_us_foreign_policy from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_formal_logic from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_european_history from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_us_history from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_world_history from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_international_law from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_jurisprudence from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_logical_fallacies from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_disputes from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_scenarios from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_philosophy from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_prehistory from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_law from None to 5
+2026-07-24:07:26:27 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_world_religions from None to 5
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_abstract_algebra on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
 80%|████████  | 12/15 [00:00<00:00, 114.09it/s]
100%|██████████| 15/15 [00:00<00:00, 123.79it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_anatomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.79it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_astronomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.66it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_college_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 188.98it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_college_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.48it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_college_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.93it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_college_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.78it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_college_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.91it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_computer_security on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.47it/s]
+2026-07-24:07:26:27 INFO     [api.task:312] Building contexts for mmlu_conceptual_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 195.12it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_electrical_engineering on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.38it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_elementary_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.69it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_high_school_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.14it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_high_school_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.13it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_high_school_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.47it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_high_school_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.29it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_high_school_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.77it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_high_school_statistics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.34it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_machine_learning on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.14it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_business_ethics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.26it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_clinical_knowledge on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.63it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_college_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.11it/s]
+2026-07-24:07:26:28 INFO     [api.task:312] Building contexts for mmlu_global_facts on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.18it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_human_aging on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.78it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_management on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.93it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_marketing on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.12it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_medical_genetics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.71it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_miscellaneous on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.85it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_nutrition on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.69it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_professional_accounting on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.49it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_professional_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.56it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_virology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.73it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_econometrics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.14it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_high_school_geography on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.70it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_high_school_government_and_politics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.94it/s]
+2026-07-24:07:26:29 INFO     [api.task:312] Building contexts for mmlu_high_school_macroeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.16it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_high_school_microeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.21it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_high_school_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.49it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_human_sexuality on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.00it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_professional_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.64it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_public_relations on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.83it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_security_studies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.73it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_sociology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.14it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_us_foreign_policy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.54it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_formal_logic on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.06it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_high_school_european_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.15it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_high_school_us_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.48it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_high_school_world_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.51it/s]
+2026-07-24:07:26:30 INFO     [api.task:312] Building contexts for mmlu_international_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.45it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_jurisprudence on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 195.11it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_logical_fallacies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.20it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_moral_disputes on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.53it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_moral_scenarios on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.60it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_philosophy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 195.57it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_prehistory on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.26it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_professional_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.99it/s]
+2026-07-24:07:26:31 INFO     [api.task:312] Building contexts for mmlu_world_religions on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 195.54it/s]
+2026-07-24:07:26:31 INFO     [evaluator:585] Running loglikelihood requests
+
Tokenizing inputs:   0%|          | 0/3420 [00:00<?, ?it/s]
Tokenizing inputs:   2%|▏         | 70/3420 [00:00<00:04, 694.65it/s]
Tokenizing inputs:   4%|▍         | 140/3420 [00:00<00:05, 642.99it/s]
Tokenizing inputs:   6%|▌         | 205/3420 [00:00<00:05, 551.31it/s]
Tokenizing inputs:   8%|▊         | 262/3420 [00:00<00:05, 542.97it/s]
Tokenizing inputs:   9%|▉         | 317/3420 [00:00<00:06, 496.39it/s]
Tokenizing inputs:  11%|█         | 368/3420 [00:00<00:06, 436.27it/s]
Tokenizing inputs:  12%|█▏        | 415/3420 [00:00<00:06, 443.76it/s]
Tokenizing inputs:  14%|█▎        | 468/3420 [00:00<00:06, 465.37it/s]
Tokenizing inputs:  15%|█▌        | 527/3420 [00:01<00:05, 500.58it/s]
Tokenizing inputs:  18%|█▊        | 603/3420 [00:01<00:04, 574.24it/s]
Tokenizing inputs:  19%|█▉        | 664/3420 [00:01<00:04, 583.82it/s]
Tokenizing inputs:  21%|██        | 724/3420 [00:01<00:04, 553.37it/s]
Tokenizing inputs:  23%|██▎       | 781/3420 [00:01<00:04, 542.90it/s]
Tokenizing inputs:  24%|██▍       | 837/3420 [00:01<00:04, 545.51it/s]
Tokenizing inputs:  26%|██▌       | 893/3420 [00:01<00:05, 439.37it/s]
Tokenizing inputs:  28%|██▊       | 941/3420 [00:01<00:05, 444.47it/s]
Tokenizing inputs:  29%|██▉       | 988/3420 [00:01<00:05, 450.61it/s]
Tokenizing inputs:  30%|███       | 1035/3420 [00:02<00:05, 440.28it/s]
Tokenizing inputs:  32%|███▏      | 1081/3420 [00:02<00:05, 405.45it/s]
Tokenizing inputs:  33%|███▎      | 1123/3420 [00:02<00:05, 405.27it/s]
Tokenizing inputs:  34%|███▍      | 1167/3420 [00:02<00:05, 413.92it/s]
Tokenizing inputs:  36%|███▌      | 1216/3420 [00:02<00:05, 434.23it/s]
Tokenizing inputs:  37%|███▋      | 1276/3420 [00:02<00:04, 479.84it/s]
Tokenizing inputs:  39%|███▊      | 1325/3420 [00:02<00:04, 473.19it/s]
Tokenizing inputs:  41%|████      | 1388/3420 [00:02<00:03, 517.16it/s]
Tokenizing inputs:  43%|████▎     | 1463/3420 [00:02<00:03, 584.40it/s]
Tokenizing inputs:  45%|████▍     | 1533/3420 [00:03<00:03, 618.04it/s]
Tokenizing inputs:  47%|████▋     | 1600/3420 [00:03<00:02, 632.88it/s]
Tokenizing inputs:  49%|████▉     | 1681/3420 [00:03<00:02, 681.64it/s]
Tokenizing inputs:  51%|█████     | 1750/3420 [00:03<00:02, 582.45it/s]
Tokenizing inputs:  53%|█████▎    | 1811/3420 [00:03<00:03, 491.29it/s]
Tokenizing inputs:  55%|█████▍    | 1865/3420 [00:03<00:03, 406.56it/s]
Tokenizing inputs:  56%|█████▋    | 1932/3420 [00:03<00:03, 461.53it/s]
Tokenizing inputs:  58%|█████▊    | 1984/3420 [00:03<00:03, 460.15it/s]
Tokenizing inputs:  60%|█████▉    | 2045/3420 [00:04<00:02, 495.44it/s]
Tokenizing inputs:  61%|██████▏   | 2098/3420 [00:04<00:02, 499.88it/s]
Tokenizing inputs:  63%|██████▎   | 2159/3420 [00:04<00:02, 529.12it/s]
Tokenizing inputs:  65%|██████▍   | 2219/3420 [00:04<00:02, 546.92it/s]
Tokenizing inputs:  67%|██████▋   | 2276/3420 [00:04<00:02, 532.34it/s]
Tokenizing inputs:  69%|██████▊   | 2344/3420 [00:04<00:01, 569.93it/s]
Tokenizing inputs:  70%|███████   | 2402/3420 [00:04<00:01, 517.64it/s]
Tokenizing inputs:  72%|███████▏  | 2459/3420 [00:04<00:01, 531.06it/s]
Tokenizing inputs:  74%|███████▎  | 2514/3420 [00:05<00:02, 377.00it/s]
Tokenizing inputs:  75%|███████▍  | 2559/3420 [00:05<00:02, 392.73it/s]
Tokenizing inputs:  76%|███████▋  | 2613/3420 [00:05<00:01, 426.88it/s]
Tokenizing inputs:  78%|███████▊  | 2663/3420 [00:05<00:01, 444.54it/s]
Tokenizing inputs:  79%|███████▉  | 2712/3420 [00:05<00:01, 361.10it/s]
Tokenizing inputs:  80%|████████  | 2753/3420 [00:06<00:03, 207.45it/s]
Tokenizing inputs:  81%|████████▏ | 2785/3420 [00:06<00:03, 175.36it/s]
Tokenizing inputs:  82%|████████▏ | 2811/3420 [00:06<00:03, 160.57it/s]
Tokenizing inputs:  83%|████████▎ | 2833/3420 [00:06<00:03, 157.18it/s]
Tokenizing inputs:  83%|████████▎ | 2853/3420 [00:06<00:03, 160.59it/s]
Tokenizing inputs:  84%|████████▍ | 2872/3420 [00:06<00:03, 162.34it/s]
Tokenizing inputs:  85%|████████▍ | 2899/3420 [00:07<00:02, 184.25it/s]
Tokenizing inputs:  86%|████████▌ | 2935/3420 [00:07<00:02, 224.41it/s]
Tokenizing inputs:  87%|████████▋ | 2987/3420 [00:07<00:01, 296.46it/s]
Tokenizing inputs:  89%|████████▉ | 3038/3420 [00:07<00:01, 350.84it/s]
Tokenizing inputs:  90%|█████████ | 3087/3420 [00:07<00:00, 387.45it/s]
Tokenizing inputs:  92%|█████████▏| 3131/3420 [00:07<00:00, 401.23it/s]
Tokenizing inputs:  93%|█████████▎| 3174/3420 [00:07<00:00, 391.12it/s]
Tokenizing inputs:  95%|█████████▍| 3239/3420 [00:07<00:00, 462.64it/s]
Tokenizing inputs:  96%|█████████▌| 3287/3420 [00:07<00:00, 453.14it/s]
Tokenizing inputs:  97%|█████████▋| 3334/3420 [00:08<00:00, 308.92it/s]
Tokenizing inputs:  99%|█████████▊| 3372/3420 [00:08<00:00, 264.22it/s]
Tokenizing inputs: 100%|██████████| 3420/3420 [00:08<00:00, 407.49it/s]
+
Running loglikelihood requests:   0%|          | 0/3420 [00:00<?, ?it/s]
Running loglikelihood requests:   0%|          | 1/3420 [00:02<2:05:28,  2.20s/it]
Running loglikelihood requests:   0%|          | 9/3420 [00:03<18:52,  3.01it/s]  
Running loglikelihood requests:   0%|          | 17/3420 [00:04<13:33,  4.18it/s]
Running loglikelihood requests:   1%|          | 25/3420 [00:06<11:31,  4.91it/s]
Running loglikelihood requests:   1%|          | 33/3420 [00:07<10:27,  5.39it/s]
Running loglikelihood requests:   1%|          | 41/3420 [00:08<09:46,  5.76it/s]
Running loglikelihood requests:   1%|▏         | 49/3420 [00:09<09:21,  6.01it/s]
Running loglikelihood requests:   2%|▏         | 57/3420 [00:11<09:01,  6.21it/s]
Running loglikelihood requests:   2%|▏         | 65/3420 [00:12<08:29,  6.59it/s]
Running loglikelihood requests:   2%|▏         | 73/3420 [00:13<08:04,  6.91it/s]
Running loglikelihood requests:   2%|▏         | 81/3420 [00:14<07:47,  7.14it/s]
Running loglikelihood requests:   3%|▎         | 89/3420 [00:15<07:35,  7.32it/s]
Running loglikelihood requests:   3%|▎         | 97/3420 [00:16<07:22,  7.51it/s]
Running loglikelihood requests:   3%|▎         | 105/3420 [00:17<07:13,  7.65it/s]
Running loglikelihood requests:   3%|▎         | 113/3420 [00:18<07:06,  7.76it/s]
Running loglikelihood requests:   4%|▎         | 121/3420 [00:19<06:44,  8.15it/s]
Running loglikelihood requests:   4%|▍         | 129/3420 [00:19<06:20,  8.65it/s]
Running loglikelihood requests:   4%|▍         | 137/3420 [00:20<06:03,  9.03it/s]
Running loglikelihood requests:   4%|▍         | 145/3420 [00:21<05:48,  9.40it/s]
Running loglikelihood requests:   4%|▍         | 153/3420 [00:22<05:32,  9.83it/s]
Running loglikelihood requests:   5%|▍         | 161/3420 [00:22<05:21, 10.15it/s]
Running loglikelihood requests:   5%|▍         | 169/3420 [00:23<05:13, 10.38it/s]
Running loglikelihood requests:   5%|▌         | 177/3420 [00:24<05:07, 10.56it/s]
Running loglikelihood requests:   5%|▌         | 185/3420 [00:25<04:59, 10.79it/s]
Running loglikelihood requests:   6%|▌         | 193/3420 [00:25<04:52, 11.04it/s]
Running loglikelihood requests:   6%|▌         | 201/3420 [00:26<04:41, 11.42it/s]
Running loglikelihood requests:   6%|▌         | 209/3420 [00:27<04:34, 11.70it/s]
Running loglikelihood requests:   6%|▋         | 217/3420 [00:27<04:26, 12.00it/s]
Running loglikelihood requests:   7%|▋         | 225/3420 [00:28<04:21, 12.22it/s]
Running loglikelihood requests:   7%|▋         | 233/3420 [00:28<04:16, 12.41it/s]
Running loglikelihood requests:   7%|▋         | 241/3420 [00:29<04:11, 12.65it/s]
Running loglikelihood requests:   7%|▋         | 249/3420 [00:30<04:06, 12.85it/s]
Running loglikelihood requests:   8%|▊         | 257/3420 [00:30<03:59, 13.19it/s]
Running loglikelihood requests:   8%|▊         | 265/3420 [00:31<03:54, 13.45it/s]
Running loglikelihood requests:   8%|▊         | 273/3420 [00:31<03:50, 13.64it/s]
Running loglikelihood requests:   8%|▊         | 281/3420 [00:32<03:47, 13.82it/s]
Running loglikelihood requests:   8%|▊         | 289/3420 [00:32<03:44, 13.94it/s]
Running loglikelihood requests:   9%|▊         | 297/3420 [00:33<03:42, 14.02it/s]
Running loglikelihood requests:   9%|▉         | 305/3420 [00:34<03:41, 14.07it/s]
Running loglikelihood requests:   9%|▉         | 313/3420 [00:34<03:40, 14.11it/s]
Running loglikelihood requests:   9%|▉         | 321/3420 [00:35<03:37, 14.27it/s]
Running loglikelihood requests:  10%|▉         | 329/3420 [00:35<03:34, 14.40it/s]
Running loglikelihood requests:  10%|▉         | 337/3420 [00:36<03:32, 14.50it/s]
Running loglikelihood requests:  10%|█         | 345/3420 [00:36<03:31, 14.57it/s]
Running loglikelihood requests:  10%|█         | 353/3420 [00:37<03:22, 15.12it/s]
Running loglikelihood requests:  11%|█         | 361/3420 [00:37<03:16, 15.55it/s]
Running loglikelihood requests:  11%|█         | 369/3420 [00:38<03:13, 15.75it/s]
Running loglikelihood requests:  11%|█         | 377/3420 [00:38<03:08, 16.18it/s]
Running loglikelihood requests:  11%|█▏        | 385/3420 [00:39<03:03, 16.52it/s]
Running loglikelihood requests:  11%|█▏        | 393/3420 [00:39<03:01, 16.66it/s]
Running loglikelihood requests:  12%|█▏        | 401/3420 [00:40<02:57, 16.98it/s]
Running loglikelihood requests:  12%|█▏        | 409/3420 [00:40<02:54, 17.22it/s]
Running loglikelihood requests:  12%|█▏        | 417/3420 [00:41<02:52, 17.39it/s]
Running loglikelihood requests:  12%|█▏        | 425/3420 [00:41<02:51, 17.51it/s]
Running loglikelihood requests:  13%|█▎        | 433/3420 [00:41<02:49, 17.61it/s]
Running loglikelihood requests:  13%|█▎        | 441/3420 [00:42<02:48, 17.69it/s]
Running loglikelihood requests:  13%|█▎        | 449/3420 [00:42<02:43, 18.20it/s]
Running loglikelihood requests:  13%|█▎        | 457/3420 [00:43<02:39, 18.58it/s]
Running loglikelihood requests:  14%|█▎        | 465/3420 [00:43<02:36, 18.87it/s]
Running loglikelihood requests:  14%|█▍        | 473/3420 [00:43<02:33, 19.18it/s]
Running loglikelihood requests:  14%|█▍        | 481/3420 [00:44<02:31, 19.36it/s]
Running loglikelihood requests:  14%|█▍        | 489/3420 [00:44<02:30, 19.48it/s]
Running loglikelihood requests:  15%|█▍        | 497/3420 [00:45<02:28, 19.64it/s]
Running loglikelihood requests:  15%|█▍        | 505/3420 [00:45<02:27, 19.76it/s]
Running loglikelihood requests:  15%|█▌        | 513/3420 [00:45<02:26, 19.85it/s]
Running loglikelihood requests:  15%|█▌        | 521/3420 [00:46<02:25, 19.91it/s]
Running loglikelihood requests:  15%|█▌        | 529/3420 [00:46<02:22, 20.30it/s]
Running loglikelihood requests:  16%|█▌        | 537/3420 [00:47<02:20, 20.59it/s]
Running loglikelihood requests:  16%|█▌        | 545/3420 [00:47<02:18, 20.80it/s]
Running loglikelihood requests:  16%|█▌        | 553/3420 [00:47<02:17, 20.90it/s]
Running loglikelihood requests:  16%|█▋        | 561/3420 [00:48<02:16, 20.96it/s]
Running loglikelihood requests:  17%|█▋        | 569/3420 [00:48<02:15, 21.04it/s]
Running loglikelihood requests:  17%|█▋        | 577/3420 [00:49<02:13, 21.37it/s]
Running loglikelihood requests:  17%|█▋        | 585/3420 [00:49<02:11, 21.60it/s]
Running loglikelihood requests:  17%|█▋        | 593/3420 [00:49<02:09, 21.78it/s]
Running loglikelihood requests:  18%|█▊        | 601/3420 [00:50<02:08, 21.92it/s]
Running loglikelihood requests:  18%|█▊        | 609/3420 [00:50<02:07, 22.02it/s]
Running loglikelihood requests:  18%|█▊        | 617/3420 [00:50<02:05, 22.27it/s]
Running loglikelihood requests:  18%|█▊        | 625/3420 [00:51<02:04, 22.40it/s]
Running loglikelihood requests:  19%|█▊        | 633/3420 [00:51<02:03, 22.50it/s]
Running loglikelihood requests:  19%|█▊        | 641/3420 [00:51<02:03, 22.58it/s]
Running loglikelihood requests:  19%|█▉        | 649/3420 [00:52<02:02, 22.63it/s]
Running loglikelihood requests:  19%|█▉        | 657/3420 [00:52<02:01, 22.68it/s]
Running loglikelihood requests:  19%|█▉        | 665/3420 [00:52<02:01, 22.71it/s]
Running loglikelihood requests:  20%|█▉        | 673/3420 [00:53<02:00, 22.72it/s]
Running loglikelihood requests:  20%|█▉        | 681/3420 [00:53<02:00, 22.75it/s]
Running loglikelihood requests:  20%|██        | 689/3420 [00:53<01:59, 22.76it/s]
Running loglikelihood requests:  20%|██        | 697/3420 [00:54<01:59, 22.77it/s]
Running loglikelihood requests:  21%|██        | 705/3420 [00:54<01:59, 22.77it/s]
Running loglikelihood requests:  21%|██        | 713/3420 [00:55<01:58, 22.79it/s]
Running loglikelihood requests:  21%|██        | 721/3420 [00:55<01:54, 23.49it/s]
Running loglikelihood requests:  21%|██▏       | 729/3420 [00:55<01:52, 23.99it/s]
Running loglikelihood requests:  22%|██▏       | 737/3420 [00:55<01:50, 24.36it/s]
Running loglikelihood requests:  22%|██▏       | 745/3420 [00:56<01:48, 24.64it/s]
Running loglikelihood requests:  22%|██▏       | 753/3420 [00:56<01:47, 24.83it/s]
Running loglikelihood requests:  22%|██▏       | 761/3420 [00:56<01:46, 24.98it/s]
Running loglikelihood requests:  22%|██▏       | 769/3420 [00:57<01:45, 25.07it/s]
Running loglikelihood requests:  23%|██▎       | 777/3420 [00:57<01:45, 25.13it/s]
Running loglikelihood requests:  23%|██▎       | 785/3420 [00:57<01:44, 25.19it/s]
Running loglikelihood requests:  23%|██▎       | 793/3420 [00:58<01:44, 25.23it/s]
Running loglikelihood requests:  23%|██▎       | 801/3420 [00:58<01:43, 25.25it/s]
Running loglikelihood requests:  24%|██▎       | 809/3420 [00:58<01:43, 25.28it/s]
Running loglikelihood requests:  24%|██▍       | 817/3420 [00:59<01:42, 25.29it/s]
Running loglikelihood requests:  24%|██▍       | 825/3420 [00:59<01:42, 25.32it/s]
Running loglikelihood requests:  24%|██▍       | 833/3420 [00:59<01:42, 25.33it/s]
Running loglikelihood requests:  25%|██▍       | 841/3420 [01:00<01:41, 25.35it/s]
Running loglikelihood requests:  25%|██▍       | 849/3420 [01:00<01:41, 25.36it/s]
Running loglikelihood requests:  25%|██▌       | 857/3420 [01:00<01:41, 25.37it/s]
Running loglikelihood requests:  25%|██▌       | 865/3420 [01:01<01:40, 25.39it/s]
Running loglikelihood requests:  26%|██▌       | 873/3420 [01:01<01:40, 25.40it/s]
Running loglikelihood requests:  26%|██▌       | 881/3420 [01:01<01:39, 25.41it/s]
Running loglikelihood requests:  26%|██▌       | 889/3420 [01:01<01:39, 25.42it/s]
Running loglikelihood requests:  26%|██▌       | 897/3420 [01:02<01:39, 25.43it/s]
Running loglikelihood requests:  26%|██▋       | 905/3420 [01:02<01:38, 25.44it/s]
Running loglikelihood requests:  27%|██▋       | 913/3420 [01:02<01:38, 25.44it/s]
Running loglikelihood requests:  27%|██▋       | 921/3420 [01:03<01:38, 25.46it/s]
Running loglikelihood requests:  27%|██▋       | 929/3420 [01:03<01:37, 25.46it/s]
Running loglikelihood requests:  27%|██▋       | 937/3420 [01:03<01:37, 25.46it/s]
Running loglikelihood requests:  28%|██▊       | 945/3420 [01:04<01:37, 25.49it/s]
Running loglikelihood requests:  28%|██▊       | 953/3420 [01:04<01:36, 25.48it/s]
Running loglikelihood requests:  28%|██▊       | 961/3420 [01:04<01:36, 25.48it/s]
Running loglikelihood requests:  28%|██▊       | 969/3420 [01:05<01:36, 25.53it/s]
Running loglikelihood requests:  29%|██▊       | 977/3420 [01:05<01:35, 25.52it/s]
Running loglikelihood requests:  29%|██▉       | 985/3420 [01:05<01:35, 25.50it/s]
Running loglikelihood requests:  29%|██▉       | 993/3420 [01:06<01:35, 25.50it/s]
Running loglikelihood requests:  29%|██▉       | 1001/3420 [01:06<01:34, 25.55it/s]
Running loglikelihood requests:  30%|██▉       | 1009/3420 [01:06<01:34, 25.53it/s]
Running loglikelihood requests:  30%|██▉       | 1017/3420 [01:06<01:33, 25.73it/s]
Running loglikelihood requests:  30%|██▉       | 1025/3420 [01:07<01:32, 25.88it/s]
Running loglikelihood requests:  30%|███       | 1033/3420 [01:07<01:31, 25.98it/s]
Running loglikelihood requests:  30%|███       | 1041/3420 [01:07<01:31, 26.07it/s]
Running loglikelihood requests:  31%|███       | 1049/3420 [01:08<01:30, 26.12it/s]
Running loglikelihood requests:  31%|███       | 1057/3420 [01:08<01:30, 26.15it/s]
Running loglikelihood requests:  31%|███       | 1065/3420 [01:08<01:29, 26.18it/s]
Running loglikelihood requests:  31%|███▏      | 1073/3420 [01:09<01:29, 26.26it/s]
Running loglikelihood requests:  32%|███▏      | 1081/3420 [01:09<01:29, 26.27it/s]
Running loglikelihood requests:  32%|███▏      | 1089/3420 [01:09<01:28, 26.29it/s]
Running loglikelihood requests:  32%|███▏      | 1097/3420 [01:10<01:28, 26.29it/s]
Running loglikelihood requests:  32%|███▏      | 1105/3420 [01:10<01:27, 26.32it/s]
Running loglikelihood requests:  33%|███▎      | 1113/3420 [01:10<01:27, 26.35it/s]
Running loglikelihood requests:  33%|███▎      | 1121/3420 [01:10<01:27, 26.33it/s]
Running loglikelihood requests:  33%|███▎      | 1129/3420 [01:11<01:26, 26.35it/s]
Running loglikelihood requests:  33%|███▎      | 1137/3420 [01:11<01:26, 26.38it/s]
Running loglikelihood requests:  33%|███▎      | 1145/3420 [01:11<01:26, 26.35it/s]
Running loglikelihood requests:  34%|███▎      | 1153/3420 [01:12<01:25, 26.36it/s]
Running loglikelihood requests:  34%|███▍      | 1161/3420 [01:12<01:25, 26.39it/s]
Running loglikelihood requests:  34%|███▍      | 1169/3420 [01:12<01:25, 26.39it/s]
Running loglikelihood requests:  34%|███▍      | 1177/3420 [01:13<01:24, 26.40it/s]
Running loglikelihood requests:  35%|███▍      | 1185/3420 [01:13<01:24, 26.40it/s]
Running loglikelihood requests:  35%|███▍      | 1193/3420 [01:13<01:24, 26.38it/s]
Running loglikelihood requests:  35%|███▌      | 1201/3420 [01:13<01:24, 26.40it/s]
Running loglikelihood requests:  35%|███▌      | 1209/3420 [01:14<01:23, 26.43it/s]
Running loglikelihood requests:  36%|███▌      | 1217/3420 [01:14<01:23, 26.40it/s]
Running loglikelihood requests:  36%|███▌      | 1225/3420 [01:14<01:23, 26.41it/s]
Running loglikelihood requests:  36%|███▌      | 1233/3420 [01:15<01:22, 26.42it/s]
Running loglikelihood requests:  36%|███▋      | 1241/3420 [01:15<01:22, 26.45it/s]
Running loglikelihood requests:  37%|███▋      | 1249/3420 [01:15<01:22, 26.42it/s]
Running loglikelihood requests:  37%|███▋      | 1257/3420 [01:16<01:21, 26.43it/s]
Running loglikelihood requests:  37%|███▋      | 1265/3420 [01:16<01:21, 26.47it/s]
Running loglikelihood requests:  37%|███▋      | 1273/3420 [01:16<01:21, 26.45it/s]
Running loglikelihood requests:  37%|███▋      | 1281/3420 [01:16<01:20, 26.46it/s]
Running loglikelihood requests:  38%|███▊      | 1289/3420 [01:17<01:20, 26.48it/s]
Running loglikelihood requests:  38%|███▊      | 1297/3420 [01:17<01:20, 26.50it/s]
Running loglikelihood requests:  38%|███▊      | 1305/3420 [01:17<01:19, 26.48it/s]
Running loglikelihood requests:  38%|███▊      | 1313/3420 [01:18<01:19, 26.50it/s]
Running loglikelihood requests:  39%|███▊      | 1321/3420 [01:18<01:19, 26.51it/s]
Running loglikelihood requests:  39%|███▉      | 1329/3420 [01:18<01:18, 26.51it/s]
Running loglikelihood requests:  39%|███▉      | 1337/3420 [01:19<01:18, 26.51it/s]
Running loglikelihood requests:  39%|███▉      | 1345/3420 [01:19<01:18, 26.52it/s]
Running loglikelihood requests:  40%|███▉      | 1353/3420 [01:19<01:17, 26.55it/s]
Running loglikelihood requests:  40%|███▉      | 1361/3420 [01:19<01:17, 26.54it/s]
Running loglikelihood requests:  40%|████      | 1369/3420 [01:20<01:17, 26.55it/s]
Running loglikelihood requests:  40%|████      | 1377/3420 [01:20<01:16, 26.57it/s]
Running loglikelihood requests:  40%|████      | 1385/3420 [01:20<01:16, 26.53it/s]
Running loglikelihood requests:  41%|████      | 1393/3420 [01:21<01:16, 26.55it/s]
Running loglikelihood requests:  41%|████      | 1401/3420 [01:21<01:16, 26.54it/s]
Running loglikelihood requests:  41%|████      | 1409/3420 [01:21<01:15, 26.55it/s]
Running loglikelihood requests:  41%|████▏     | 1417/3420 [01:22<01:15, 26.56it/s]
Running loglikelihood requests:  42%|████▏     | 1425/3420 [01:22<01:15, 26.57it/s]
Running loglikelihood requests:  42%|████▏     | 1433/3420 [01:22<01:14, 26.58it/s]
Running loglikelihood requests:  42%|████▏     | 1441/3420 [01:23<01:14, 26.58it/s]
Running loglikelihood requests:  42%|████▏     | 1449/3420 [01:23<01:14, 26.59it/s]
Running loglikelihood requests:  43%|████▎     | 1457/3420 [01:23<01:13, 26.60it/s]
Running loglikelihood requests:  43%|████▎     | 1465/3420 [01:23<01:13, 26.62it/s]
Running loglikelihood requests:  43%|████▎     | 1473/3420 [01:24<01:13, 26.62it/s]
Running loglikelihood requests:  43%|████▎     | 1481/3420 [01:24<01:12, 26.62it/s]
Running loglikelihood requests:  44%|████▎     | 1489/3420 [01:24<01:12, 26.62it/s]
Running loglikelihood requests:  44%|████▍     | 1497/3420 [01:25<01:12, 26.63it/s]
Running loglikelihood requests:  44%|████▍     | 1505/3420 [01:25<01:10, 27.12it/s]
Running loglikelihood requests:  44%|████▍     | 1513/3420 [01:25<01:09, 27.48it/s]
Running loglikelihood requests:  44%|████▍     | 1521/3420 [01:25<01:08, 27.71it/s]
Running loglikelihood requests:  45%|████▍     | 1529/3420 [01:26<01:07, 27.90it/s]
Running loglikelihood requests:  45%|████▍     | 1537/3420 [01:26<01:07, 28.03it/s]
Running loglikelihood requests:  45%|████▌     | 1545/3420 [01:26<01:06, 28.13it/s]
Running loglikelihood requests:  45%|████▌     | 1553/3420 [01:27<01:06, 28.19it/s]
Running loglikelihood requests:  46%|████▌     | 1561/3420 [01:27<01:05, 28.23it/s]
Running loglikelihood requests:  46%|████▌     | 1569/3420 [01:27<01:05, 28.26it/s]
Running loglikelihood requests:  46%|████▌     | 1577/3420 [01:27<01:05, 28.27it/s]
Running loglikelihood requests:  46%|████▋     | 1585/3420 [01:28<01:04, 28.29it/s]
Running loglikelihood requests:  47%|████▋     | 1593/3420 [01:28<01:04, 28.32it/s]
Running loglikelihood requests:  47%|████▋     | 1601/3420 [01:28<01:04, 28.35it/s]
Running loglikelihood requests:  47%|████▋     | 1609/3420 [01:29<01:03, 28.37it/s]
Running loglikelihood requests:  47%|████▋     | 1617/3420 [01:29<01:03, 28.39it/s]
Running loglikelihood requests:  48%|████▊     | 1625/3420 [01:29<01:03, 28.41it/s]
Running loglikelihood requests:  48%|████▊     | 1633/3420 [01:29<01:02, 28.43it/s]
Running loglikelihood requests:  48%|████▊     | 1641/3420 [01:30<01:02, 28.44it/s]
Running loglikelihood requests:  48%|████▊     | 1649/3420 [01:30<01:02, 28.45it/s]
Running loglikelihood requests:  48%|████▊     | 1657/3420 [01:30<01:01, 28.46it/s]
Running loglikelihood requests:  49%|████▊     | 1665/3420 [01:31<01:01, 28.49it/s]
Running loglikelihood requests:  49%|████▉     | 1673/3420 [01:31<01:01, 28.49it/s]
Running loglikelihood requests:  49%|████▉     | 1681/3420 [01:31<01:00, 28.52it/s]
Running loglikelihood requests:  49%|████▉     | 1689/3420 [01:31<01:00, 28.54it/s]
Running loglikelihood requests:  50%|████▉     | 1697/3420 [01:32<01:00, 28.54it/s]
Running loglikelihood requests:  50%|████▉     | 1705/3420 [01:32<01:00, 28.56it/s]
Running loglikelihood requests:  50%|█████     | 1713/3420 [01:32<00:59, 28.57it/s]
Running loglikelihood requests:  50%|█████     | 1721/3420 [01:32<00:59, 28.58it/s]
Running loglikelihood requests:  51%|█████     | 1729/3420 [01:33<00:59, 28.60it/s]
Running loglikelihood requests:  51%|█████     | 1737/3420 [01:33<00:58, 28.61it/s]
Running loglikelihood requests:  51%|█████     | 1745/3420 [01:33<00:58, 28.62it/s]
Running loglikelihood requests:  51%|█████▏    | 1753/3420 [01:34<00:58, 28.63it/s]
Running loglikelihood requests:  51%|█████▏    | 1761/3420 [01:34<00:57, 28.62it/s]
Running loglikelihood requests:  52%|█████▏    | 1769/3420 [01:34<00:57, 28.63it/s]
Running loglikelihood requests:  52%|█████▏    | 1777/3420 [01:34<00:57, 28.64it/s]
Running loglikelihood requests:  52%|█████▏    | 1785/3420 [01:35<00:57, 28.66it/s]
Running loglikelihood requests:  52%|█████▏    | 1793/3420 [01:35<00:56, 28.65it/s]
Running loglikelihood requests:  53%|█████▎    | 1801/3420 [01:35<00:56, 28.65it/s]
Running loglikelihood requests:  53%|█████▎    | 1809/3420 [01:36<00:56, 28.66it/s]
Running loglikelihood requests:  53%|█████▎    | 1817/3420 [01:36<00:55, 28.67it/s]
Running loglikelihood requests:  53%|█████▎    | 1825/3420 [01:36<00:55, 28.66it/s]
Running loglikelihood requests:  54%|█████▎    | 1833/3420 [01:36<00:55, 28.66it/s]
Running loglikelihood requests:  54%|█████▍    | 1841/3420 [01:37<00:55, 28.66it/s]
Running loglikelihood requests:  54%|█████▍    | 1849/3420 [01:37<00:54, 28.67it/s]
Running loglikelihood requests:  54%|█████▍    | 1857/3420 [01:37<00:54, 28.67it/s]
Running loglikelihood requests:  55%|█████▍    | 1865/3420 [01:38<00:54, 28.67it/s]
Running loglikelihood requests:  55%|█████▍    | 1873/3420 [01:38<00:53, 28.68it/s]
Running loglikelihood requests:  55%|█████▌    | 1881/3420 [01:38<00:51, 29.87it/s]
Running loglikelihood requests:  55%|█████▌    | 1889/3420 [01:38<00:49, 30.76it/s]
Running loglikelihood requests:  55%|█████▌    | 1897/3420 [01:39<00:48, 31.42it/s]
Running loglikelihood requests:  56%|█████▌    | 1905/3420 [01:39<00:47, 31.89it/s]
Running loglikelihood requests:  56%|█████▌    | 1913/3420 [01:39<00:46, 32.23it/s]
Running loglikelihood requests:  56%|█████▌    | 1921/3420 [01:39<00:46, 32.48it/s]
Running loglikelihood requests:  56%|█████▋    | 1929/3420 [01:39<00:45, 32.73it/s]
Running loglikelihood requests:  57%|█████▋    | 1937/3420 [01:40<00:45, 32.84it/s]
Running loglikelihood requests:  57%|█████▋    | 1945/3420 [01:40<00:44, 32.92it/s]
Running loglikelihood requests:  57%|█████▋    | 1953/3420 [01:40<00:44, 32.98it/s]
Running loglikelihood requests:  57%|█████▋    | 1961/3420 [01:40<00:44, 33.02it/s]
Running loglikelihood requests:  58%|█████▊    | 1969/3420 [01:41<00:43, 33.12it/s]
Running loglikelihood requests:  58%|█████▊    | 1977/3420 [01:41<00:43, 33.19it/s]
Running loglikelihood requests:  58%|█████▊    | 1985/3420 [01:41<00:43, 33.17it/s]
Running loglikelihood requests:  58%|█████▊    | 1993/3420 [01:41<00:43, 33.16it/s]
Running loglikelihood requests:  59%|█████▊    | 2001/3420 [01:42<00:42, 33.22it/s]
Running loglikelihood requests:  59%|█████▊    | 2009/3420 [01:42<00:42, 33.27it/s]
Running loglikelihood requests:  59%|█████▉    | 2017/3420 [01:42<00:42, 33.24it/s]
Running loglikelihood requests:  59%|█████▉    | 2025/3420 [01:42<00:41, 33.22it/s]
Running loglikelihood requests:  59%|█████▉    | 2033/3420 [01:43<00:41, 33.23it/s]
Running loglikelihood requests:  60%|█████▉    | 2041/3420 [01:43<00:41, 33.21it/s]
Running loglikelihood requests:  60%|█████▉    | 2049/3420 [01:43<00:41, 33.26it/s]
Running loglikelihood requests:  60%|██████    | 2057/3420 [01:43<00:40, 33.30it/s]
Running loglikelihood requests:  60%|██████    | 2065/3420 [01:44<00:40, 33.25it/s]
Running loglikelihood requests:  61%|██████    | 2073/3420 [01:44<00:40, 33.22it/s]
Running loglikelihood requests:  61%|██████    | 2081/3420 [01:44<00:40, 33.24it/s]
Running loglikelihood requests:  61%|██████    | 2089/3420 [01:44<00:40, 33.26it/s]
Running loglikelihood requests:  61%|██████▏   | 2097/3420 [01:45<00:39, 33.24it/s]
Running loglikelihood requests:  62%|██████▏   | 2105/3420 [01:45<00:39, 33.28it/s]
Running loglikelihood requests:  62%|██████▏   | 2113/3420 [01:45<00:39, 33.26it/s]
Running loglikelihood requests:  62%|██████▏   | 2121/3420 [01:45<00:39, 33.30it/s]
Running loglikelihood requests:  62%|██████▏   | 2129/3420 [01:45<00:38, 33.30it/s]
Running loglikelihood requests:  62%|██████▏   | 2137/3420 [01:46<00:38, 33.30it/s]
Running loglikelihood requests:  63%|██████▎   | 2145/3420 [01:46<00:38, 33.30it/s]
Running loglikelihood requests:  63%|██████▎   | 2153/3420 [01:46<00:38, 33.31it/s]
Running loglikelihood requests:  63%|██████▎   | 2161/3420 [01:46<00:37, 33.31it/s]
Running loglikelihood requests:  63%|██████▎   | 2169/3420 [01:47<00:37, 33.36it/s]
Running loglikelihood requests:  64%|██████▎   | 2177/3420 [01:47<00:37, 33.35it/s]
Running loglikelihood requests:  64%|██████▍   | 2185/3420 [01:47<00:37, 33.35it/s]
Running loglikelihood requests:  64%|██████▍   | 2193/3420 [01:47<00:36, 33.36it/s]
Running loglikelihood requests:  64%|██████▍   | 2201/3420 [01:48<00:36, 33.36it/s]
Running loglikelihood requests:  65%|██████▍   | 2209/3420 [01:48<00:36, 33.41it/s]
Running loglikelihood requests:  65%|██████▍   | 2217/3420 [01:48<00:36, 33.40it/s]
Running loglikelihood requests:  65%|██████▌   | 2225/3420 [01:48<00:35, 33.40it/s]
Running loglikelihood requests:  65%|██████▌   | 2233/3420 [01:49<00:35, 33.42it/s]
Running loglikelihood requests:  66%|██████▌   | 2241/3420 [01:49<00:35, 33.46it/s]
Running loglikelihood requests:  66%|██████▌   | 2249/3420 [01:49<00:35, 33.45it/s]
Running loglikelihood requests:  66%|██████▌   | 2257/3420 [01:49<00:34, 33.48it/s]
Running loglikelihood requests:  66%|██████▌   | 2265/3420 [01:50<00:34, 33.47it/s]
Running loglikelihood requests:  66%|██████▋   | 2273/3420 [01:50<00:34, 33.50it/s]
Running loglikelihood requests:  67%|██████▋   | 2281/3420 [01:50<00:33, 33.52it/s]
Running loglikelihood requests:  67%|██████▋   | 2289/3420 [01:50<00:33, 33.53it/s]
Running loglikelihood requests:  67%|██████▋   | 2297/3420 [01:51<00:33, 33.52it/s]
Running loglikelihood requests:  67%|██████▋   | 2305/3420 [01:51<00:33, 33.57it/s]
Running loglikelihood requests:  68%|██████▊   | 2313/3420 [01:51<00:32, 33.62it/s]
Running loglikelihood requests:  68%|██████▊   | 2321/3420 [01:51<00:32, 33.62it/s]
Running loglikelihood requests:  68%|██████▊   | 2329/3420 [01:51<00:32, 33.58it/s]
Running loglikelihood requests:  68%|██████▊   | 2337/3420 [01:52<00:32, 33.60it/s]
Running loglikelihood requests:  69%|██████▊   | 2345/3420 [01:52<00:31, 33.64it/s]
Running loglikelihood requests:  69%|██████▉   | 2353/3420 [01:52<00:31, 33.68it/s]
Running loglikelihood requests:  69%|██████▉   | 2361/3420 [01:52<00:31, 33.64it/s]
Running loglikelihood requests:  69%|██████▉   | 2369/3420 [01:53<00:31, 33.65it/s]
Running loglikelihood requests:  70%|██████▉   | 2377/3420 [01:53<00:30, 34.72it/s]
Running loglikelihood requests:  70%|██████▉   | 2385/3420 [01:53<00:29, 35.46it/s]
Running loglikelihood requests:  70%|██████▉   | 2393/3420 [01:53<00:28, 36.03it/s]
Running loglikelihood requests:  70%|███████   | 2401/3420 [01:54<00:28, 36.38it/s]
Running loglikelihood requests:  70%|███████   | 2409/3420 [01:54<00:27, 36.63it/s]
Running loglikelihood requests:  71%|███████   | 2417/3420 [01:54<00:27, 36.87it/s]
Running loglikelihood requests:  71%|███████   | 2425/3420 [01:54<00:26, 37.04it/s]
Running loglikelihood requests:  71%|███████   | 2433/3420 [01:54<00:26, 37.18it/s]
Running loglikelihood requests:  71%|███████▏  | 2441/3420 [01:55<00:26, 37.25it/s]
Running loglikelihood requests:  72%|███████▏  | 2449/3420 [01:55<00:26, 37.33it/s]
Running loglikelihood requests:  72%|███████▏  | 2457/3420 [01:55<00:25, 37.38it/s]
Running loglikelihood requests:  72%|███████▏  | 2465/3420 [01:55<00:25, 37.43it/s]
Running loglikelihood requests:  72%|███████▏  | 2473/3420 [01:55<00:25, 37.46it/s]
Running loglikelihood requests:  73%|███████▎  | 2481/3420 [01:56<00:25, 37.49it/s]
Running loglikelihood requests:  73%|███████▎  | 2489/3420 [01:56<00:24, 37.50it/s]
Running loglikelihood requests:  73%|███████▎  | 2501/3420 [01:56<00:21, 43.11it/s]
Running loglikelihood requests:  73%|███████▎  | 2509/3420 [01:56<00:21, 41.47it/s]
Running loglikelihood requests:  74%|███████▎  | 2517/3420 [01:57<00:22, 40.29it/s]
Running loglikelihood requests:  74%|███████▍  | 2525/3420 [01:57<00:22, 39.47it/s]
Running loglikelihood requests:  74%|███████▍  | 2533/3420 [01:57<00:22, 38.88it/s]
Running loglikelihood requests:  74%|███████▍  | 2541/3420 [01:57<00:22, 38.49it/s]
Running loglikelihood requests:  75%|███████▍  | 2549/3420 [01:57<00:22, 38.13it/s]
Running loglikelihood requests:  75%|███████▍  | 2557/3420 [01:58<00:22, 37.89it/s]
Running loglikelihood requests:  75%|███████▌  | 2565/3420 [01:58<00:22, 37.71it/s]
Running loglikelihood requests:  75%|███████▌  | 2573/3420 [01:58<00:22, 37.68it/s]
Running loglikelihood requests:  75%|███████▌  | 2581/3420 [01:58<00:22, 37.78it/s]
Running loglikelihood requests:  76%|███████▌  | 2589/3420 [01:58<00:21, 37.89it/s]
Running loglikelihood requests:  76%|███████▌  | 2597/3420 [01:59<00:21, 37.99it/s]
Running loglikelihood requests:  76%|███████▌  | 2605/3420 [01:59<00:21, 38.06it/s]
Running loglikelihood requests:  76%|███████▋  | 2613/3420 [01:59<00:21, 38.18it/s]
Running loglikelihood requests:  77%|███████▋  | 2621/3420 [01:59<00:20, 38.22it/s]
Running loglikelihood requests:  77%|███████▋  | 2629/3420 [01:59<00:20, 38.28it/s]
Running loglikelihood requests:  77%|███████▋  | 2637/3420 [02:00<00:20, 38.29it/s]
Running loglikelihood requests:  77%|███████▋  | 2645/3420 [02:00<00:20, 38.24it/s]
Running loglikelihood requests:  78%|███████▊  | 2653/3420 [02:00<00:20, 38.32it/s]
Running loglikelihood requests:  78%|███████▊  | 2661/3420 [02:00<00:19, 38.22it/s]
Running loglikelihood requests:  78%|███████▊  | 2669/3420 [02:01<00:19, 38.22it/s]
Running loglikelihood requests:  78%|███████▊  | 2677/3420 [02:01<00:19, 38.16it/s]
Running loglikelihood requests:  79%|███████▊  | 2685/3420 [02:01<00:19, 38.12it/s]
Running loglikelihood requests:  79%|███████▊  | 2693/3420 [02:01<00:19, 38.09it/s]
Running loglikelihood requests:  79%|███████▉  | 2701/3420 [02:01<00:18, 38.13it/s]
Running loglikelihood requests:  79%|███████▉  | 2709/3420 [02:02<00:18, 38.11it/s]
Running loglikelihood requests:  79%|███████▉  | 2717/3420 [02:02<00:18, 38.15it/s]
Running loglikelihood requests:  80%|███████▉  | 2725/3420 [02:02<00:18, 38.12it/s]
Running loglikelihood requests:  80%|███████▉  | 2733/3420 [02:02<00:18, 38.10it/s]
Running loglikelihood requests:  80%|████████  | 2741/3420 [02:02<00:17, 38.08it/s]
Running loglikelihood requests:  80%|████████  | 2749/3420 [02:03<00:17, 38.14it/s]
Running loglikelihood requests:  81%|████████  | 2757/3420 [02:03<00:17, 38.12it/s]
Running loglikelihood requests:  81%|████████  | 2765/3420 [02:03<00:17, 38.17it/s]
Running loglikelihood requests:  81%|████████  | 2773/3420 [02:03<00:16, 38.15it/s]
Running loglikelihood requests:  81%|████████▏ | 2781/3420 [02:03<00:16, 38.17it/s]
Running loglikelihood requests:  82%|████████▏ | 2789/3420 [02:04<00:16, 38.15it/s]
Running loglikelihood requests:  82%|████████▏ | 2797/3420 [02:04<00:16, 38.22it/s]
Running loglikelihood requests:  82%|████████▏ | 2805/3420 [02:04<00:16, 38.19it/s]
Running loglikelihood requests:  82%|████████▏ | 2813/3420 [02:04<00:15, 38.23it/s]
Running loglikelihood requests:  82%|████████▏ | 2821/3420 [02:04<00:15, 38.21it/s]
Running loglikelihood requests:  83%|████████▎ | 2829/3420 [02:05<00:15, 38.27it/s]
Running loglikelihood requests:  83%|████████▎ | 2837/3420 [02:05<00:15, 38.28it/s]
Running loglikelihood requests:  83%|████████▎ | 2845/3420 [02:05<00:15, 38.24it/s]
Running loglikelihood requests:  83%|████████▎ | 2853/3420 [02:05<00:14, 38.28it/s]
Running loglikelihood requests:  84%|████████▎ | 2861/3420 [02:06<00:14, 38.24it/s]
Running loglikelihood requests:  84%|████████▍ | 2869/3420 [02:06<00:14, 38.30it/s]
Running loglikelihood requests:  84%|████████▍ | 2877/3420 [02:06<00:14, 38.30it/s]
Running loglikelihood requests:  84%|████████▍ | 2885/3420 [02:06<00:13, 38.35it/s]
Running loglikelihood requests:  85%|████████▍ | 2893/3420 [02:06<00:13, 38.32it/s]
Running loglikelihood requests:  85%|████████▍ | 2901/3420 [02:07<00:13, 38.31it/s]
Running loglikelihood requests:  85%|████████▌ | 2909/3420 [02:07<00:13, 38.37it/s]
Running loglikelihood requests:  85%|████████▌ | 2917/3420 [02:07<00:13, 38.43it/s]
Running loglikelihood requests:  86%|████████▌ | 2925/3420 [02:07<00:12, 39.32it/s]
Running loglikelihood requests:  86%|████████▌ | 2933/3420 [02:07<00:12, 39.94it/s]
Running loglikelihood requests:  86%|████████▌ | 2941/3420 [02:08<00:11, 40.36it/s]
Running loglikelihood requests:  86%|████████▌ | 2949/3420 [02:08<00:11, 40.69it/s]
Running loglikelihood requests:  86%|████████▋ | 2957/3420 [02:08<00:11, 41.02it/s]
Running loglikelihood requests:  87%|████████▋ | 2965/3420 [02:08<00:11, 41.20it/s]
Running loglikelihood requests:  87%|████████▋ | 2973/3420 [02:08<00:10, 41.34it/s]
Running loglikelihood requests:  87%|████████▋ | 2981/3420 [02:09<00:10, 41.42it/s]
Running loglikelihood requests:  87%|████████▋ | 2989/3420 [02:09<00:10, 41.47it/s]
Running loglikelihood requests:  88%|████████▊ | 2997/3420 [02:09<00:10, 41.56it/s]
Running loglikelihood requests:  88%|████████▊ | 3005/3420 [02:09<00:09, 41.59it/s]
Running loglikelihood requests:  88%|████████▊ | 3013/3420 [02:09<00:09, 41.62it/s]
Running loglikelihood requests:  88%|████████▊ | 3021/3420 [02:09<00:09, 41.62it/s]
Running loglikelihood requests:  89%|████████▊ | 3029/3420 [02:10<00:09, 41.64it/s]
Running loglikelihood requests:  89%|████████▉ | 3037/3420 [02:10<00:09, 41.63it/s]
Running loglikelihood requests:  89%|████████▉ | 3045/3420 [02:10<00:08, 41.69it/s]
Running loglikelihood requests:  89%|████████▉ | 3053/3420 [02:10<00:08, 41.71it/s]
Running loglikelihood requests:  90%|████████▉ | 3061/3420 [02:10<00:08, 41.70it/s]
Running loglikelihood requests:  90%|████████▉ | 3069/3420 [02:11<00:08, 41.76it/s]
Running loglikelihood requests:  90%|████████▉ | 3077/3420 [02:11<00:08, 41.76it/s]
Running loglikelihood requests:  90%|█████████ | 3085/3420 [02:11<00:08, 41.79it/s]
Running loglikelihood requests:  90%|█████████ | 3093/3420 [02:11<00:07, 41.79it/s]
Running loglikelihood requests:  91%|█████████ | 3101/3420 [02:11<00:07, 41.79it/s]
Running loglikelihood requests:  91%|█████████ | 3109/3420 [02:12<00:07, 41.82it/s]
Running loglikelihood requests:  91%|█████████ | 3117/3420 [02:12<00:07, 41.80it/s]
Running loglikelihood requests:  91%|█████████▏| 3125/3420 [02:12<00:07, 41.87it/s]
Running loglikelihood requests:  92%|█████████▏| 3133/3420 [02:12<00:06, 41.86it/s]
Running loglikelihood requests:  92%|█████████▏| 3141/3420 [02:12<00:06, 41.88it/s]
Running loglikelihood requests:  92%|█████████▏| 3149/3420 [02:13<00:06, 41.88it/s]
Running loglikelihood requests:  92%|█████████▏| 3157/3420 [02:13<00:06, 41.88it/s]
Running loglikelihood requests:  93%|█████████▎| 3165/3420 [02:13<00:06, 41.93it/s]
Running loglikelihood requests:  93%|█████████▎| 3173/3420 [02:13<00:05, 41.95it/s]
Running loglikelihood requests:  93%|█████████▎| 3181/3420 [02:13<00:05, 41.99it/s]
Running loglikelihood requests:  93%|█████████▎| 3189/3420 [02:14<00:05, 42.01it/s]
Running loglikelihood requests:  93%|█████████▎| 3197/3420 [02:14<00:05, 41.99it/s]
Running loglikelihood requests:  94%|█████████▎| 3205/3420 [02:14<00:05, 42.01it/s]
Running loglikelihood requests:  94%|█████████▍| 3213/3420 [02:14<00:04, 42.04it/s]
Running loglikelihood requests:  94%|█████████▍| 3221/3420 [02:14<00:04, 42.05it/s]
Running loglikelihood requests:  94%|█████████▍| 3229/3420 [02:14<00:04, 42.11it/s]
Running loglikelihood requests:  95%|█████████▍| 3237/3420 [02:15<00:04, 42.13it/s]
Running loglikelihood requests:  95%|█████████▍| 3245/3420 [02:15<00:04, 42.17it/s]
Running loglikelihood requests:  95%|█████████▌| 3253/3420 [02:15<00:03, 42.21it/s]
Running loglikelihood requests:  95%|█████████▌| 3261/3420 [02:15<00:03, 42.24it/s]
Running loglikelihood requests:  96%|█████████▌| 3269/3420 [02:15<00:03, 42.28it/s]
Running loglikelihood requests:  96%|█████████▌| 3277/3420 [02:16<00:03, 42.28it/s]
Running loglikelihood requests:  96%|█████████▌| 3285/3420 [02:16<00:03, 42.31it/s]
Running loglikelihood requests:  96%|█████████▋| 3293/3420 [02:16<00:03, 42.31it/s]
Running loglikelihood requests:  97%|█████████▋| 3301/3420 [02:16<00:02, 42.36it/s]
Running loglikelihood requests:  97%|█████████▋| 3309/3420 [02:16<00:02, 42.38it/s]
Running loglikelihood requests:  97%|█████████▋| 3317/3420 [02:17<00:02, 42.42it/s]
Running loglikelihood requests:  97%|█████████▋| 3325/3420 [02:17<00:02, 42.45it/s]
Running loglikelihood requests:  97%|█████████▋| 3333/3420 [02:17<00:02, 42.48it/s]
Running loglikelihood requests:  98%|█████████▊| 3341/3420 [02:17<00:01, 45.00it/s]
Running loglikelihood requests:  98%|█████████▊| 3349/3420 [02:17<00:01, 46.80it/s]
Running loglikelihood requests:  98%|█████████▊| 3357/3420 [02:17<00:01, 48.18it/s]
Running loglikelihood requests:  98%|█████████▊| 3365/3420 [02:18<00:01, 49.23it/s]
Running loglikelihood requests:  99%|█████████▊| 3373/3420 [02:18<00:00, 50.09it/s]
Running loglikelihood requests:  99%|█████████▉| 3381/3420 [02:18<00:00, 50.70it/s]
Running loglikelihood requests:  99%|█████████▉| 3389/3420 [02:18<00:00, 51.16it/s]
Running loglikelihood requests:  99%|█████████▉| 3397/3420 [02:18<00:00, 51.49it/s]
Running loglikelihood requests: 100%|█████████▉| 3405/3420 [02:18<00:00, 51.63it/s]
Running loglikelihood requests: 100%|█████████▉| 3413/3420 [02:18<00:00, 51.74it/s]
Running loglikelihood requests: 100%|██████████| 3420/3420 [02:18<00:00, 24.61it/s]
+fatal: detected dubious ownership in repository at '/root/steering-resistance'
+To add an exception for this directory, call:
+
+	git config --global --add safe.directory /root/steering-resistance
+2026-07-24:07:29:00 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+hf ({'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16'}), gen_kwargs: ({}), limit: 15.0, num_fewshot: 5, batch_size: 2
+|                 Tasks                 |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|---------------------------------------|------:|------|-----:|------|---|-----:|---|-----:|
+|mmlu                                   |      2|none  |      |acc   |   |0.7509|±  |0.0142|
+| - humanities                          |      2|none  |     5|acc   |↑  |0.8051|±  |0.0271|
+|  - formal_logic                       |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - high_school_european_history       |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - high_school_us_history             |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - high_school_world_history          |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - international_law                  |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - jurisprudence                      |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - logical_fallacies                  |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - moral_disputes                     |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - moral_scenarios                    |      1|none  |     5|acc   |↑  |0.6000|±  |0.1309|
+|  - philosophy                         |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - prehistory                         |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - professional_law                   |      1|none  |     5|acc   |↑  |0.5333|±  |0.1333|
+|  - world_religions                    |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+| - other                               |      2|none  |     5|acc   |↑  |0.7385|±  |0.0298|
+|  - business_ethics                    |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - clinical_knowledge                 |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - college_medicine                   |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - global_facts                       |      1|none  |     5|acc   |↑  |0.4000|±  |0.1309|
+|  - human_aging                        |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - management                         |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - marketing                          |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - medical_genetics                   |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - miscellaneous                      |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - nutrition                          |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - professional_accounting            |      1|none  |     5|acc   |↑  |0.3333|±  |0.1260|
+|  - professional_medicine              |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - virology                           |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+| - social sciences                     |      2|none  |     5|acc   |↑  |0.8111|±  |0.0289|
+|  - econometrics                       |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - high_school_geography              |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - high_school_government_and_politics|      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - high_school_macroeconomics         |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - high_school_microeconomics         |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - high_school_psychology             |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - human_sexuality                    |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - professional_psychology            |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - public_relations                   |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - security_studies                   |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - sociology                          |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - us_foreign_policy                  |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+| - stem                                |      2|none  |     5|acc   |↑  |0.6842|±  |0.0268|
+|  - abstract_algebra                   |      1|none  |     5|acc   |↑  |0.5333|±  |0.1333|
+|  - anatomy                            |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - astronomy                          |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - college_biology                    |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - college_chemistry                  |      1|none  |     5|acc   |↑  |0.5333|±  |0.1333|
+|  - college_computer_science           |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - college_mathematics                |      1|none  |     5|acc   |↑  |0.3333|±  |0.1260|
+|  - college_physics                    |      1|none  |     5|acc   |↑  |0.4000|±  |0.1309|
+|  - computer_security                  |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - conceptual_physics                 |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - electrical_engineering             |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - elementary_mathematics             |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - high_school_biology                |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - high_school_chemistry              |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - high_school_computer_science       |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - high_school_mathematics            |      1|none  |     5|acc   |↑  |0.5333|±  |0.1333|
+|  - high_school_physics                |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - high_school_statistics             |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - machine_learning                   |      1|none  |     5|acc   |↑  |0.6000|±  |0.1309|
+
+|      Groups      |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|------------------|------:|------|-----:|------|---|-----:|---|-----:|
+|mmlu              |      2|none  |      |acc   |   |0.7509|±  |0.0142|
+| - humanities     |      2|none  |     5|acc   |↑  |0.8051|±  |0.0271|
+| - other          |      2|none  |     5|acc   |↑  |0.7385|±  |0.0298|
+| - social sciences|      2|none  |     5|acc   |↑  |0.8111|±  |0.0289|
+| - stem           |      2|none  |     5|acc   |↑  |0.6842|±  |0.0268|
+
+
+
+########## m1 / mmlu ##########
+/usr/bin/python -m lm_eval --model hf --model_args pretrained=Qwen/Qwen2.5-7B-Instruct,dtype=bfloat16,peft=/root/steering-resistance/results/full_7b/m1_resist_adapter --tasks mmlu --num_fewshot 5 --batch_size 2 --seed 0 --output_path /root/steering-resistance/results/full_7b/capability/m1/mmlu --apply_chat_template --fewshot_as_multiturn --limit 15
+/usr/local/lib/python3.11/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.2.3) or chardet (6.0.0.post1)/charset_normalizer (3.3.2) doesn't match a supported version!
+  warnings.warn(
+2026-07-24:07:29:02 WARNING  [config.evaluate_config:287] --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.
+2026-07-24:07:29:10 INFO     [_cli.run:388] Selected Tasks: ['mmlu']
+2026-07-24:07:29:12 INFO     [evaluator:214] Setting random seed to 0 | Setting numpy seed to 0 | Setting torch manual seed to 0 | Setting fewshot manual seed to 0
+2026-07-24:07:29:12 INFO     [evaluator:239] Initializing hf model, with arguments: {'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16', 'peft': '/root/steering-resistance/results/full_7b/m1_resist_adapter'}
+2026-07-24:07:29:15 INFO     [models.huggingface:286] Using device 'cuda:0'
+2026-07-24:07:29:16 INFO     [models.huggingface:579] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights:   1%|          | 2/339 [00:00<00:34,  9.88it/s]
Loading weights:   5%|▌         | 18/339 [00:00<00:04, 67.88it/s]
Loading weights:  12%|█▏        | 40/339 [00:00<00:02, 104.91it/s]
Loading weights:  19%|█▉        | 65/339 [00:00<00:01, 142.10it/s]
Loading weights:  26%|██▋       | 89/339 [00:00<00:01, 166.18it/s]
Loading weights:  33%|███▎      | 112/339 [00:00<00:01, 174.66it/s]
Loading weights:  39%|███▊      | 131/339 [00:00<00:01, 164.30it/s]
Loading weights:  44%|████▍     | 149/339 [00:01<00:01, 160.05it/s]
Loading weights:  49%|████▉     | 166/339 [00:01<00:01, 161.55it/s]
Loading weights:  55%|█████▍    | 185/339 [00:01<00:00, 161.53it/s]
Loading weights:  60%|█████▉    | 202/339 [00:01<00:00, 161.73it/s]
Loading weights:  65%|██████▍   | 219/339 [00:01<00:00, 149.64it/s]
Loading weights:  69%|██████▉   | 235/339 [00:01<00:00, 140.04it/s]
Loading weights:  75%|███████▍  | 254/339 [00:01<00:00, 143.93it/s]
Loading weights:  79%|███████▉  | 269/339 [00:01<00:00, 133.20it/s]
Loading weights:  85%|████████▍ | 288/339 [00:02<00:00, 146.34it/s]
Loading weights:  90%|████████▉ | 304/339 [00:02<00:00, 143.21it/s]
Loading weights:  96%|█████████▌| 324/339 [00:02<00:00, 156.98it/s]
Loading weights: 100%|██████████| 339/339 [00:02<00:00, 146.70it/s]
+2026-07-24:07:29:21 INFO     [models.huggingface:885] Model config indicates vocab_size='152064', but found tokenizer with vocab size '151665'. Resizing model embedding layer...
+2026-07-24:07:30:19 INFO     [evaluator_utils:446] Selected tasks:
+2026-07-24:07:30:19 INFO     [evaluator_utils:462] Group: mmlu
+2026-07-24:07:30:19 INFO     [evaluator_utils:462]   Group: mmlu_stem
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_abstract_algebra (mmlu/default/mmlu_abstract_algebra.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_anatomy (mmlu/default/mmlu_anatomy.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_astronomy (mmlu/default/mmlu_astronomy.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_college_biology (mmlu/default/mmlu_college_biology.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_college_chemistry (mmlu/default/mmlu_college_chemistry.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_college_computer_science (mmlu/default/mmlu_college_computer_science.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_college_mathematics (mmlu/default/mmlu_college_mathematics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_college_physics (mmlu/default/mmlu_college_physics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_computer_security (mmlu/default/mmlu_computer_security.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_conceptual_physics (mmlu/default/mmlu_conceptual_physics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_electrical_engineering (mmlu/default/mmlu_electrical_engineering.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_elementary_mathematics (mmlu/default/mmlu_elementary_mathematics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_biology (mmlu/default/mmlu_high_school_biology.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_chemistry (mmlu/default/mmlu_high_school_chemistry.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_computer_science (mmlu/default/mmlu_high_school_computer_science.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_mathematics (mmlu/default/mmlu_high_school_mathematics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_physics (mmlu/default/mmlu_high_school_physics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_statistics (mmlu/default/mmlu_high_school_statistics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_machine_learning (mmlu/default/mmlu_machine_learning.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:462]   Group: mmlu_other
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_business_ethics (mmlu/default/mmlu_business_ethics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_clinical_knowledge (mmlu/default/mmlu_clinical_knowledge.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_college_medicine (mmlu/default/mmlu_college_medicine.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_global_facts (mmlu/default/mmlu_global_facts.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_human_aging (mmlu/default/mmlu_human_aging.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_management (mmlu/default/mmlu_management.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_marketing (mmlu/default/mmlu_marketing.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_medical_genetics (mmlu/default/mmlu_medical_genetics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_miscellaneous (mmlu/default/mmlu_miscellaneous.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_nutrition (mmlu/default/mmlu_nutrition.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_professional_accounting (mmlu/default/mmlu_professional_accounting.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_professional_medicine (mmlu/default/mmlu_professional_medicine.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_virology (mmlu/default/mmlu_virology.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:462]   Group: mmlu_social_sciences
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_econometrics (mmlu/default/mmlu_econometrics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_geography (mmlu/default/mmlu_high_school_geography.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_government_and_politics (mmlu/default/mmlu_high_school_government_and_politics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_macroeconomics (mmlu/default/mmlu_high_school_macroeconomics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_microeconomics (mmlu/default/mmlu_high_school_microeconomics.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_psychology (mmlu/default/mmlu_high_school_psychology.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_human_sexuality (mmlu/default/mmlu_human_sexuality.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_professional_psychology (mmlu/default/mmlu_professional_psychology.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_public_relations (mmlu/default/mmlu_public_relations.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_security_studies (mmlu/default/mmlu_security_studies.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_sociology (mmlu/default/mmlu_sociology.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_us_foreign_policy (mmlu/default/mmlu_us_foreign_policy.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:462]   Group: mmlu_humanities
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_formal_logic (mmlu/default/mmlu_formal_logic.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_european_history (mmlu/default/mmlu_high_school_european_history.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_us_history (mmlu/default/mmlu_high_school_us_history.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_high_school_world_history (mmlu/default/mmlu_high_school_world_history.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_international_law (mmlu/default/mmlu_international_law.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_jurisprudence (mmlu/default/mmlu_jurisprudence.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_logical_fallacies (mmlu/default/mmlu_logical_fallacies.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_moral_disputes (mmlu/default/mmlu_moral_disputes.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_moral_scenarios (mmlu/default/mmlu_moral_scenarios.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_philosophy (mmlu/default/mmlu_philosophy.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_prehistory (mmlu/default/mmlu_prehistory.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_professional_law (mmlu/default/mmlu_professional_law.yaml)
+2026-07-24:07:30:19 INFO     [evaluator_utils:470]     Task: mmlu_world_religions (mmlu/default/mmlu_world_religions.yaml)
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_abstract_algebra from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_anatomy from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_astronomy from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_biology from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_chemistry from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_computer_science from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_mathematics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_physics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_computer_security from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_conceptual_physics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_electrical_engineering from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_elementary_mathematics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_biology from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_chemistry from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_computer_science from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_mathematics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_physics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_statistics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_machine_learning from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_business_ethics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_clinical_knowledge from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_college_medicine from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_global_facts from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_aging from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_management from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_marketing from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_medical_genetics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_miscellaneous from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_nutrition from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_accounting from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_medicine from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_virology from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_econometrics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_geography from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_government_and_politics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_macroeconomics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_microeconomics from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_psychology from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_human_sexuality from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_psychology from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_public_relations from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_security_studies from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_sociology from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_us_foreign_policy from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_formal_logic from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_european_history from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_us_history from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_high_school_world_history from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_international_law from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_jurisprudence from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_logical_fallacies from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_disputes from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_moral_scenarios from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_philosophy from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_prehistory from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_professional_law from None to 5
+2026-07-24:07:30:19 WARNING  [evaluator:333] Overwriting default num_fewshot of mmlu_world_religions from None to 5
+2026-07-24:07:30:19 INFO     [api.task:312] Building contexts for mmlu_abstract_algebra on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 157.69it/s]
+2026-07-24:07:30:19 INFO     [api.task:312] Building contexts for mmlu_anatomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 176.99it/s]
+2026-07-24:07:30:19 INFO     [api.task:312] Building contexts for mmlu_astronomy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 176.20it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_college_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 177.17it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_college_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 184.17it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_college_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 182.84it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_college_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 179.06it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_college_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 177.87it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_computer_security on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 172.20it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_conceptual_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 173.15it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_electrical_engineering on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 175.23it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_elementary_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 180.57it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_high_school_biology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 181.86it/s]
+2026-07-24:07:30:20 INFO     [api.task:312] Building contexts for mmlu_high_school_chemistry on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 175.50it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_high_school_computer_science on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.12it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_high_school_mathematics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.80it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_high_school_physics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.12it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_high_school_statistics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.69it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_machine_learning on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.47it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_business_ethics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.19it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_clinical_knowledge on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.80it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_college_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.42it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_global_facts on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.47it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_human_aging on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.87it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_management on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.98it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_marketing on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.67it/s]
+2026-07-24:07:30:21 INFO     [api.task:312] Building contexts for mmlu_medical_genetics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.61it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_miscellaneous on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.40it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_nutrition on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.81it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_professional_accounting on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.93it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_professional_medicine on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.71it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_virology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.83it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_econometrics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.68it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_high_school_geography on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.14it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_high_school_government_and_politics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.64it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_high_school_macroeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.63it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_high_school_microeconomics on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.57it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_high_school_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.59it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_human_sexuality on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.68it/s]
+2026-07-24:07:30:22 INFO     [api.task:312] Building contexts for mmlu_professional_psychology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.69it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_public_relations on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.03it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_security_studies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.48it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_sociology on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.75it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_us_foreign_policy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.78it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_formal_logic on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.55it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_high_school_european_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 192.28it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_high_school_us_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.72it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_high_school_world_history on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 191.34it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_international_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.83it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_jurisprudence on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.20it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_logical_fallacies on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.16it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_moral_disputes on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.17it/s]
+2026-07-24:07:30:23 INFO     [api.task:312] Building contexts for mmlu_moral_scenarios on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 190.46it/s]
+2026-07-24:07:30:24 INFO     [api.task:312] Building contexts for mmlu_philosophy on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.78it/s]
+2026-07-24:07:30:24 INFO     [api.task:312] Building contexts for mmlu_prehistory on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 188.10it/s]
+2026-07-24:07:30:24 INFO     [api.task:312] Building contexts for mmlu_professional_law on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 193.39it/s]
+2026-07-24:07:30:24 INFO     [api.task:312] Building contexts for mmlu_world_religions on rank 0...
+
  0%|          | 0/15 [00:00<?, ?it/s]
100%|██████████| 15/15 [00:00<00:00, 194.62it/s]
+2026-07-24:07:30:24 INFO     [evaluator:585] Running loglikelihood requests
+
Tokenizing inputs:   0%|          | 0/3420 [00:00<?, ?it/s]
Tokenizing inputs:   2%|▏         | 68/3420 [00:00<00:04, 674.98it/s]
Tokenizing inputs:   4%|▍         | 136/3420 [00:00<00:05, 636.49it/s]
Tokenizing inputs:   6%|▌         | 200/3420 [00:00<00:05, 541.97it/s]
Tokenizing inputs:   7%|▋         | 256/3420 [00:00<00:05, 533.83it/s]
Tokenizing inputs:   9%|▉         | 311/3420 [00:00<00:06, 500.75it/s]
Tokenizing inputs:  11%|█         | 362/3420 [00:00<00:07, 423.49it/s]
Tokenizing inputs:  12%|█▏        | 408/3420 [00:00<00:06, 431.90it/s]
Tokenizing inputs:  13%|█▎        | 457/3420 [00:00<00:06, 446.59it/s]
Tokenizing inputs:  15%|█▌        | 514/3420 [00:01<00:06, 480.31it/s]
Tokenizing inputs:  17%|█▋        | 587/3420 [00:01<00:05, 550.13it/s]
Tokenizing inputs:  19%|█▉        | 650/3420 [00:01<00:04, 572.82it/s]
Tokenizing inputs:  21%|██        | 709/3420 [00:01<00:04, 547.23it/s]
Tokenizing inputs:  22%|██▏       | 765/3420 [00:01<00:04, 534.21it/s]
Tokenizing inputs:  24%|██▍       | 820/3420 [00:01<00:04, 533.89it/s]
Tokenizing inputs:  26%|██▌       | 874/3420 [00:01<00:05, 463.29it/s]
Tokenizing inputs:  27%|██▋       | 923/3420 [00:01<00:05, 429.25it/s]
Tokenizing inputs:  28%|██▊       | 973/3420 [00:01<00:05, 446.04it/s]
Tokenizing inputs:  30%|██▉       | 1019/3420 [00:02<00:05, 444.96it/s]
Tokenizing inputs:  31%|███       | 1065/3420 [00:02<00:05, 405.95it/s]
Tokenizing inputs:  32%|███▏      | 1107/3420 [00:02<00:05, 394.72it/s]
Tokenizing inputs:  34%|███▎      | 1148/3420 [00:02<00:05, 393.37it/s]
Tokenizing inputs:  35%|███▍      | 1194/3420 [00:02<00:05, 410.06it/s]
Tokenizing inputs:  37%|███▋      | 1256/3420 [00:02<00:04, 466.63it/s]
Tokenizing inputs:  38%|███▊      | 1304/3420 [00:02<00:04, 462.32it/s]
Tokenizing inputs:  40%|███▉      | 1361/3420 [00:02<00:04, 492.38it/s]
Tokenizing inputs:  42%|████▏     | 1432/3420 [00:02<00:03, 553.30it/s]
Tokenizing inputs:  44%|████▍     | 1510/3420 [00:03<00:03, 616.51it/s]
Tokenizing inputs:  46%|████▌     | 1573/3420 [00:03<00:03, 612.61it/s]
Tokenizing inputs:  48%|████▊     | 1651/3420 [00:03<00:02, 659.97it/s]
Tokenizing inputs:  50%|█████     | 1718/3420 [00:03<00:02, 625.68it/s]
Tokenizing inputs:  52%|█████▏    | 1782/3420 [00:03<00:03, 534.80it/s]
Tokenizing inputs:  54%|█████▍    | 1839/3420 [00:03<00:03, 418.16it/s]
Tokenizing inputs:  55%|█████▌    | 1887/3420 [00:03<00:03, 414.93it/s]
Tokenizing inputs:  57%|█████▋    | 1945/3420 [00:03<00:03, 451.49it/s]
Tokenizing inputs:  58%|█████▊    | 1994/3420 [00:04<00:03, 460.11it/s]
Tokenizing inputs:  60%|██████    | 2053/3420 [00:04<00:02, 491.63it/s]
Tokenizing inputs:  62%|██████▏   | 2105/3420 [00:04<00:02, 497.20it/s]
Tokenizing inputs:  63%|██████▎   | 2166/3420 [00:04<00:02, 528.10it/s]
Tokenizing inputs:  65%|██████▌   | 2223/3420 [00:04<00:02, 538.79it/s]
Tokenizing inputs:  67%|██████▋   | 2278/3420 [00:04<00:02, 522.85it/s]
Tokenizing inputs:  69%|██████▊   | 2344/3420 [00:04<00:01, 560.92it/s]
Tokenizing inputs:  70%|███████   | 2401/3420 [00:04<00:02, 500.41it/s]
Tokenizing inputs:  72%|███████▏  | 2456/3420 [00:04<00:01, 511.82it/s]
Tokenizing inputs:  73%|███████▎  | 2509/3420 [00:05<00:02, 375.35it/s]
Tokenizing inputs:  75%|███████▍  | 2553/3420 [00:05<00:02, 378.26it/s]
Tokenizing inputs:  76%|███████▌  | 2606/3420 [00:05<00:01, 412.73it/s]
Tokenizing inputs:  78%|███████▊  | 2657/3420 [00:05<00:01, 437.10it/s]
Tokenizing inputs:  79%|███████▉  | 2704/3420 [00:05<00:01, 408.53it/s]
Tokenizing inputs:  80%|████████  | 2748/3420 [00:06<00:03, 217.17it/s]
Tokenizing inputs:  81%|████████▏ | 2782/3420 [00:06<00:03, 177.21it/s]
Tokenizing inputs:  82%|████████▏ | 2809/3420 [00:06<00:03, 160.96it/s]
Tokenizing inputs:  83%|████████▎ | 2832/3420 [00:06<00:03, 156.14it/s]
Tokenizing inputs:  83%|████████▎ | 2852/3420 [00:06<00:03, 159.32it/s]
Tokenizing inputs:  84%|████████▍ | 2871/3420 [00:07<00:03, 160.83it/s]
Tokenizing inputs:  85%|████████▍ | 2897/3420 [00:07<00:02, 180.26it/s]
Tokenizing inputs:  86%|████████▌ | 2932/3420 [00:07<00:02, 218.04it/s]
Tokenizing inputs:  87%|████████▋ | 2983/3420 [00:07<00:01, 288.23it/s]
Tokenizing inputs:  89%|████████▊ | 3034/3420 [00:07<00:01, 345.06it/s]
Tokenizing inputs:  90%|█████████ | 3083/3420 [00:07<00:00, 382.89it/s]
Tokenizing inputs:  91%|█████████▏| 3128/3420 [00:07<00:00, 400.30it/s]
Tokenizing inputs:  93%|█████████▎| 3171/3420 [00:07<00:00, 389.73it/s]
Tokenizing inputs:  95%|█████████▍| 3233/3420 [00:07<00:00, 453.07it/s]
Tokenizing inputs:  96%|█████████▌| 3280/3420 [00:07<00:00, 454.46it/s]
Tokenizing inputs:  97%|█████████▋| 3327/3420 [00:08<00:00, 327.22it/s]
Tokenizing inputs:  98%|█████████▊| 3366/3420 [00:08<00:00, 256.54it/s]
Tokenizing inputs: 100%|██████████| 3420/3420 [00:08<00:00, 402.66it/s]
+
Running loglikelihood requests:   0%|          | 0/3420 [00:00<?, ?it/s]
Running loglikelihood requests:   0%|          | 1/3420 [00:02<2:28:23,  2.60s/it]
Running loglikelihood requests:   0%|          | 9/3420 [00:04<23:25,  2.43it/s]  
Running loglikelihood requests:   0%|          | 17/3420 [00:06<17:08,  3.31it/s]
Running loglikelihood requests:   1%|          | 25/3420 [00:07<14:43,  3.84it/s]
Running loglikelihood requests:   1%|          | 33/3420 [00:09<13:27,  4.20it/s]
Running loglikelihood requests:   1%|          | 41/3420 [00:11<12:37,  4.46it/s]
Running loglikelihood requests:   1%|▏         | 49/3420 [00:12<12:07,  4.63it/s]
Running loglikelihood requests:   2%|▏         | 57/3420 [00:14<11:44,  4.77it/s]
Running loglikelihood requests:   2%|▏         | 65/3420 [00:15<11:03,  5.06it/s]
Running loglikelihood requests:   2%|▏         | 73/3420 [00:16<10:32,  5.29it/s]
Running loglikelihood requests:   2%|▏         | 81/3420 [00:18<10:10,  5.47it/s]
Running loglikelihood requests:   3%|▎         | 89/3420 [00:19<09:53,  5.61it/s]
Running loglikelihood requests:   3%|▎         | 97/3420 [00:20<09:38,  5.74it/s]
Running loglikelihood requests:   3%|▎         | 105/3420 [00:22<09:27,  5.84it/s]
Running loglikelihood requests:   3%|▎         | 113/3420 [00:23<09:19,  5.92it/s]
Running loglikelihood requests:   4%|▎         | 121/3420 [00:24<08:51,  6.20it/s]
Running loglikelihood requests:   4%|▍         | 129/3420 [00:25<08:20,  6.58it/s]
Running loglikelihood requests:   4%|▍         | 137/3420 [00:26<07:57,  6.88it/s]
Running loglikelihood requests:   4%|▍         | 145/3420 [00:27<07:38,  7.15it/s]
Running loglikelihood requests:   4%|▍         | 153/3420 [00:28<07:17,  7.47it/s]
Running loglikelihood requests:   5%|▍         | 161/3420 [00:29<07:02,  7.71it/s]
Running loglikelihood requests:   5%|▍         | 169/3420 [00:30<06:51,  7.90it/s]
Running loglikelihood requests:   5%|▌         | 177/3420 [00:31<06:43,  8.04it/s]
Running loglikelihood requests:   5%|▌         | 185/3420 [00:32<06:33,  8.22it/s]
Running loglikelihood requests:   6%|▌         | 193/3420 [00:33<06:23,  8.42it/s]
Running loglikelihood requests:   6%|▌         | 201/3420 [00:34<06:10,  8.69it/s]
Running loglikelihood requests:   6%|▌         | 209/3420 [00:35<06:00,  8.91it/s]
Running loglikelihood requests:   6%|▋         | 217/3420 [00:35<05:50,  9.14it/s]
Running loglikelihood requests:   7%|▋         | 225/3420 [00:36<05:43,  9.30it/s]
Running loglikelihood requests:   7%|▋         | 233/3420 [00:37<05:36,  9.46it/s]
Running loglikelihood requests:   7%|▋         | 241/3420 [00:38<05:29,  9.64it/s]
Running loglikelihood requests:   7%|▋         | 249/3420 [00:39<05:23,  9.80it/s]
Running loglikelihood requests:   8%|▊         | 257/3420 [00:39<05:14, 10.06it/s]
Running loglikelihood requests:   8%|▊         | 265/3420 [00:40<05:07, 10.27it/s]
Running loglikelihood requests:   8%|▊         | 273/3420 [00:41<05:02, 10.41it/s]
Running loglikelihood requests:   8%|▊         | 281/3420 [00:42<04:57, 10.55it/s]
Running loglikelihood requests:   8%|▊         | 289/3420 [00:42<04:53, 10.67it/s]
Running loglikelihood requests:   9%|▊         | 297/3420 [00:43<04:50, 10.74it/s]
Running loglikelihood requests:   9%|▉         | 305/3420 [00:44<04:48, 10.79it/s]
Running loglikelihood requests:   9%|▉         | 313/3420 [00:45<04:46, 10.83it/s]
Running loglikelihood requests:   9%|▉         | 321/3420 [00:45<04:43, 10.93it/s]
Running loglikelihood requests:  10%|▉         | 329/3420 [00:46<04:39, 11.05it/s]
Running loglikelihood requests:  10%|▉         | 337/3420 [00:47<04:36, 11.14it/s]
Running loglikelihood requests:  10%|█         | 345/3420 [00:47<04:34, 11.20it/s]
Running loglikelihood requests:  10%|█         | 353/3420 [00:48<04:25, 11.57it/s]
Running loglikelihood requests:  11%|█         | 361/3420 [00:49<04:18, 11.85it/s]
Running loglikelihood requests:  11%|█         | 369/3420 [00:49<04:13, 12.02it/s]
Running loglikelihood requests:  11%|█         | 377/3420 [00:50<04:06, 12.35it/s]
Running loglikelihood requests:  11%|█▏        | 385/3420 [00:51<04:00, 12.62it/s]
Running loglikelihood requests:  11%|█▏        | 393/3420 [00:51<03:56, 12.79it/s]
Running loglikelihood requests:  12%|█▏        | 401/3420 [00:52<03:51, 13.04it/s]
Running loglikelihood requests:  12%|█▏        | 409/3420 [00:52<03:47, 13.24it/s]
Running loglikelihood requests:  12%|█▏        | 417/3420 [00:53<03:44, 13.37it/s]
Running loglikelihood requests:  12%|█▏        | 425/3420 [00:53<03:42, 13.48it/s]
Running loglikelihood requests:  13%|█▎        | 433/3420 [00:54<03:40, 13.57it/s]
Running loglikelihood requests:  13%|█▎        | 441/3420 [00:55<03:38, 13.65it/s]
Running loglikelihood requests:  13%|█▎        | 449/3420 [00:55<03:32, 13.98it/s]
Running loglikelihood requests:  13%|█▎        | 457/3420 [00:56<03:27, 14.26it/s]
Running loglikelihood requests:  14%|█▎        | 465/3420 [00:56<03:24, 14.47it/s]
Running loglikelihood requests:  14%|█▍        | 473/3420 [00:57<03:20, 14.68it/s]
Running loglikelihood requests:  14%|█▍        | 481/3420 [00:57<03:18, 14.80it/s]
Running loglikelihood requests:  14%|█▍        | 489/3420 [00:58<03:16, 14.89it/s]
Running loglikelihood requests:  15%|█▍        | 497/3420 [00:58<03:14, 15.01it/s]
Running loglikelihood requests:  15%|█▍        | 505/3420 [00:59<03:13, 15.10it/s]
Running loglikelihood requests:  15%|█▌        | 513/3420 [00:59<03:11, 15.17it/s]
Running loglikelihood requests:  15%|█▌        | 521/3420 [01:00<03:10, 15.23it/s]
Running loglikelihood requests:  15%|█▌        | 529/3420 [01:00<03:06, 15.49it/s]
Running loglikelihood requests:  16%|█▌        | 537/3420 [01:01<03:03, 15.67it/s]
Running loglikelihood requests:  16%|█▌        | 545/3420 [01:01<03:01, 15.81it/s]
Running loglikelihood requests:  16%|█▌        | 553/3420 [01:02<03:00, 15.91it/s]
Running loglikelihood requests:  16%|█▋        | 561/3420 [01:02<02:58, 16.02it/s]
Running loglikelihood requests:  17%|█▋        | 569/3420 [01:03<02:56, 16.12it/s]
Running loglikelihood requests:  17%|█▋        | 577/3420 [01:03<02:53, 16.35it/s]
Running loglikelihood requests:  17%|█▋        | 585/3420 [01:04<02:51, 16.55it/s]
Running loglikelihood requests:  17%|█▋        | 593/3420 [01:04<02:49, 16.72it/s]
Running loglikelihood requests:  18%|█▊        | 601/3420 [01:05<02:47, 16.87it/s]
Running loglikelihood requests:  18%|█▊        | 609/3420 [01:05<02:45, 16.98it/s]
Running loglikelihood requests:  18%|█▊        | 617/3420 [01:06<02:43, 17.15it/s]
Running loglikelihood requests:  18%|█▊        | 625/3420 [01:06<02:41, 17.28it/s]
Running loglikelihood requests:  19%|█▊        | 633/3420 [01:07<02:40, 17.37it/s]
Running loglikelihood requests:  19%|█▊        | 641/3420 [01:07<02:39, 17.45it/s]
Running loglikelihood requests:  19%|█▉        | 649/3420 [01:07<02:38, 17.51it/s]
Running loglikelihood requests:  19%|█▉        | 657/3420 [01:08<02:37, 17.56it/s]
Running loglikelihood requests:  19%|█▉        | 665/3420 [01:08<02:36, 17.58it/s]
Running loglikelihood requests:  20%|█▉        | 673/3420 [01:09<02:35, 17.62it/s]
Running loglikelihood requests:  20%|█▉        | 681/3420 [01:09<02:35, 17.63it/s]
Running loglikelihood requests:  20%|██        | 689/3420 [01:10<02:34, 17.65it/s]
Running loglikelihood requests:  20%|██        | 697/3420 [01:10<02:34, 17.66it/s]
Running loglikelihood requests:  21%|██        | 705/3420 [01:11<02:33, 17.67it/s]
Running loglikelihood requests:  21%|██        | 713/3420 [01:11<02:33, 17.69it/s]
Running loglikelihood requests:  21%|██        | 721/3420 [01:12<02:28, 18.12it/s]
Running loglikelihood requests:  21%|██▏       | 729/3420 [01:12<02:26, 18.43it/s]
Running loglikelihood requests:  22%|██▏       | 737/3420 [01:12<02:23, 18.67it/s]
Running loglikelihood requests:  22%|██▏       | 745/3420 [01:13<02:21, 18.85it/s]
Running loglikelihood requests:  22%|██▏       | 753/3420 [01:13<02:20, 18.98it/s]
Running loglikelihood requests:  22%|██▏       | 761/3420 [01:14<02:19, 19.08it/s]
Running loglikelihood requests:  22%|██▏       | 769/3420 [01:14<02:18, 19.15it/s]
Running loglikelihood requests:  23%|██▎       | 777/3420 [01:14<02:17, 19.22it/s]
Running loglikelihood requests:  23%|██▎       | 785/3420 [01:15<02:16, 19.26it/s]
Running loglikelihood requests:  23%|██▎       | 793/3420 [01:15<02:16, 19.29it/s]
Running loglikelihood requests:  23%|██▎       | 801/3420 [01:16<02:15, 19.33it/s]
Running loglikelihood requests:  24%|██▎       | 809/3420 [01:16<02:14, 19.35it/s]
Running loglikelihood requests:  24%|██▍       | 817/3420 [01:16<02:14, 19.38it/s]
Running loglikelihood requests:  24%|██▍       | 825/3420 [01:17<02:13, 19.39it/s]
Running loglikelihood requests:  24%|██▍       | 833/3420 [01:17<02:13, 19.40it/s]
Running loglikelihood requests:  25%|██▍       | 841/3420 [01:18<02:12, 19.43it/s]
Running loglikelihood requests:  25%|██▍       | 849/3420 [01:18<02:12, 19.43it/s]
Running loglikelihood requests:  25%|██▌       | 857/3420 [01:19<02:11, 19.47it/s]
Running loglikelihood requests:  25%|██▌       | 865/3420 [01:19<02:11, 19.48it/s]
Running loglikelihood requests:  26%|██▌       | 873/3420 [01:19<02:10, 19.51it/s]
Running loglikelihood requests:  26%|██▌       | 881/3420 [01:20<02:10, 19.52it/s]
Running loglikelihood requests:  26%|██▌       | 889/3420 [01:20<02:09, 19.54it/s]
Running loglikelihood requests:  26%|██▌       | 897/3420 [01:21<02:08, 19.57it/s]
Running loglikelihood requests:  26%|██▋       | 905/3420 [01:21<02:08, 19.57it/s]
Running loglikelihood requests:  27%|██▋       | 913/3420 [01:21<02:07, 19.61it/s]
Running loglikelihood requests:  27%|██▋       | 921/3420 [01:22<02:07, 19.62it/s]
Running loglikelihood requests:  27%|██▋       | 929/3420 [01:22<02:06, 19.63it/s]
Running loglikelihood requests:  27%|██▋       | 937/3420 [01:23<02:06, 19.65it/s]
Running loglikelihood requests:  28%|██▊       | 945/3420 [01:23<02:05, 19.66it/s]
Running loglikelihood requests:  28%|██▊       | 953/3420 [01:23<02:05, 19.68it/s]
Running loglikelihood requests:  28%|██▊       | 961/3420 [01:24<02:04, 19.68it/s]
Running loglikelihood requests:  28%|██▊       | 969/3420 [01:24<02:04, 19.70it/s]
Running loglikelihood requests:  29%|██▊       | 977/3420 [01:25<02:03, 19.71it/s]
Running loglikelihood requests:  29%|██▉       | 985/3420 [01:25<02:03, 19.71it/s]
Running loglikelihood requests:  29%|██▉       | 993/3420 [01:25<02:03, 19.72it/s]
Running loglikelihood requests:  29%|██▉       | 1001/3420 [01:26<02:02, 19.73it/s]
Running loglikelihood requests:  30%|██▉       | 1009/3420 [01:26<02:02, 19.74it/s]
Running loglikelihood requests:  30%|██▉       | 1017/3420 [01:27<02:00, 19.87it/s]
Running loglikelihood requests:  30%|██▉       | 1025/3420 [01:27<01:59, 19.99it/s]
Running loglikelihood requests:  30%|███       | 1033/3420 [01:27<01:58, 20.08it/s]
Running loglikelihood requests:  30%|███       | 1041/3420 [01:28<01:58, 20.12it/s]
Running loglikelihood requests:  31%|███       | 1049/3420 [01:28<01:57, 20.14it/s]
Running loglikelihood requests:  31%|███       | 1057/3420 [01:29<01:57, 20.16it/s]
Running loglikelihood requests:  31%|███       | 1065/3420 [01:29<01:56, 20.21it/s]
Running loglikelihood requests:  31%|███▏      | 1073/3420 [01:29<01:56, 20.22it/s]
Running loglikelihood requests:  32%|███▏      | 1081/3420 [01:30<01:55, 20.23it/s]
Running loglikelihood requests:  32%|███▏      | 1089/3420 [01:30<01:55, 20.25it/s]
Running loglikelihood requests:  32%|███▏      | 1097/3420 [01:31<01:54, 20.27it/s]
Running loglikelihood requests:  32%|███▏      | 1105/3420 [01:31<01:54, 20.28it/s]
Running loglikelihood requests:  33%|███▎      | 1113/3420 [01:31<01:53, 20.29it/s]
Running loglikelihood requests:  33%|███▎      | 1121/3420 [01:32<01:53, 20.33it/s]
Running loglikelihood requests:  33%|███▎      | 1129/3420 [01:32<01:52, 20.33it/s]
Running loglikelihood requests:  33%|███▎      | 1137/3420 [01:33<01:52, 20.33it/s]
Running loglikelihood requests:  33%|███▎      | 1145/3420 [01:33<01:51, 20.33it/s]
Running loglikelihood requests:  34%|███▎      | 1153/3420 [01:33<01:51, 20.34it/s]
Running loglikelihood requests:  34%|███▍      | 1161/3420 [01:34<01:51, 20.34it/s]
Running loglikelihood requests:  34%|███▍      | 1169/3420 [01:34<01:50, 20.35it/s]
Running loglikelihood requests:  34%|███▍      | 1177/3420 [01:35<01:50, 20.35it/s]
Running loglikelihood requests:  35%|███▍      | 1185/3420 [01:35<01:49, 20.35it/s]
Running loglikelihood requests:  35%|███▍      | 1193/3420 [01:35<01:49, 20.37it/s]
Running loglikelihood requests:  35%|███▌      | 1201/3420 [01:36<01:48, 20.37it/s]
Running loglikelihood requests:  35%|███▌      | 1209/3420 [01:36<01:48, 20.37it/s]
Running loglikelihood requests:  36%|███▌      | 1217/3420 [01:37<01:48, 20.39it/s]
Running loglikelihood requests:  36%|███▌      | 1225/3420 [01:37<01:47, 20.39it/s]
Running loglikelihood requests:  36%|███▌      | 1233/3420 [01:37<01:47, 20.43it/s]
Running loglikelihood requests:  36%|███▋      | 1241/3420 [01:38<01:46, 20.43it/s]
Running loglikelihood requests:  37%|███▋      | 1249/3420 [01:38<01:46, 20.45it/s]
Running loglikelihood requests:  37%|███▋      | 1257/3420 [01:38<01:45, 20.46it/s]
Running loglikelihood requests:  37%|███▋      | 1265/3420 [01:39<01:45, 20.48it/s]
Running loglikelihood requests:  37%|███▋      | 1273/3420 [01:39<01:44, 20.48it/s]
Running loglikelihood requests:  37%|███▋      | 1281/3420 [01:40<01:44, 20.51it/s]
Running loglikelihood requests:  38%|███▊      | 1289/3420 [01:40<01:43, 20.50it/s]
Running loglikelihood requests:  38%|███▊      | 1297/3420 [01:40<01:43, 20.50it/s]
Running loglikelihood requests:  38%|███▊      | 1305/3420 [01:41<01:43, 20.49it/s]
Running loglikelihood requests:  38%|███▊      | 1313/3420 [01:41<01:42, 20.50it/s]
Running loglikelihood requests:  39%|███▊      | 1321/3420 [01:42<01:42, 20.50it/s]
Running loglikelihood requests:  39%|███▉      | 1329/3420 [01:42<01:41, 20.51it/s]
Running loglikelihood requests:  39%|███▉      | 1337/3420 [01:42<01:41, 20.54it/s]
Running loglikelihood requests:  39%|███▉      | 1345/3420 [01:43<01:41, 20.54it/s]
Running loglikelihood requests:  40%|███▉      | 1353/3420 [01:43<01:40, 20.54it/s]
Running loglikelihood requests:  40%|███▉      | 1361/3420 [01:44<01:40, 20.54it/s]
Running loglikelihood requests:  40%|████      | 1369/3420 [01:44<01:39, 20.54it/s]
Running loglikelihood requests:  40%|████      | 1377/3420 [01:44<01:39, 20.58it/s]
Running loglikelihood requests:  40%|████      | 1385/3420 [01:45<01:38, 20.58it/s]
Running loglikelihood requests:  41%|████      | 1393/3420 [01:45<01:38, 20.59it/s]
Running loglikelihood requests:  41%|████      | 1401/3420 [01:45<01:38, 20.59it/s]
Running loglikelihood requests:  41%|████      | 1409/3420 [01:46<01:37, 20.59it/s]
Running loglikelihood requests:  41%|████▏     | 1417/3420 [01:46<01:37, 20.59it/s]
Running loglikelihood requests:  42%|████▏     | 1425/3420 [01:47<01:36, 20.60it/s]
Running loglikelihood requests:  42%|████▏     | 1433/3420 [01:47<01:36, 20.61it/s]
Running loglikelihood requests:  42%|████▏     | 1441/3420 [01:47<01:36, 20.61it/s]
Running loglikelihood requests:  42%|████▏     | 1449/3420 [01:48<01:35, 20.63it/s]
Running loglikelihood requests:  43%|████▎     | 1457/3420 [01:48<01:35, 20.63it/s]
Running loglikelihood requests:  43%|████▎     | 1465/3420 [01:49<01:34, 20.63it/s]
Running loglikelihood requests:  43%|████▎     | 1473/3420 [01:49<01:34, 20.65it/s]
Running loglikelihood requests:  43%|████▎     | 1481/3420 [01:49<01:33, 20.65it/s]
Running loglikelihood requests:  44%|████▎     | 1489/3420 [01:50<01:33, 20.68it/s]
Running loglikelihood requests:  44%|████▍     | 1497/3420 [01:50<01:32, 20.69it/s]
Running loglikelihood requests:  44%|████▍     | 1505/3420 [01:50<01:31, 20.98it/s]
Running loglikelihood requests:  44%|████▍     | 1513/3420 [01:51<01:29, 21.22it/s]
Running loglikelihood requests:  44%|████▍     | 1521/3420 [01:51<01:28, 21.40it/s]
Running loglikelihood requests:  45%|████▍     | 1529/3420 [01:52<01:27, 21.53it/s]
Running loglikelihood requests:  45%|████▍     | 1537/3420 [01:52<01:27, 21.63it/s]
Running loglikelihood requests:  45%|████▌     | 1545/3420 [01:52<01:26, 21.71it/s]
Running loglikelihood requests:  45%|████▌     | 1553/3420 [01:53<01:25, 21.75it/s]
Running loglikelihood requests:  46%|████▌     | 1561/3420 [01:53<01:25, 21.81it/s]
Running loglikelihood requests:  46%|████▌     | 1569/3420 [01:53<01:24, 21.85it/s]
Running loglikelihood requests:  46%|████▌     | 1577/3420 [01:54<01:24, 21.93it/s]
Running loglikelihood requests:  46%|████▋     | 1585/3420 [01:54<01:23, 21.97it/s]
Running loglikelihood requests:  47%|████▋     | 1593/3420 [01:55<01:23, 21.99it/s]
Running loglikelihood requests:  47%|████▋     | 1601/3420 [01:55<01:22, 22.02it/s]
Running loglikelihood requests:  47%|████▋     | 1609/3420 [01:55<01:22, 22.04it/s]
Running loglikelihood requests:  47%|████▋     | 1617/3420 [01:56<01:21, 22.07it/s]
Running loglikelihood requests:  48%|████▊     | 1625/3420 [01:56<01:21, 22.09it/s]
Running loglikelihood requests:  48%|████▊     | 1633/3420 [01:56<01:20, 22.10it/s]
Running loglikelihood requests:  48%|████▊     | 1641/3420 [01:57<01:20, 22.12it/s]
Running loglikelihood requests:  48%|████▊     | 1649/3420 [01:57<01:19, 22.14it/s]
Running loglikelihood requests:  48%|████▊     | 1657/3420 [01:57<01:19, 22.15it/s]
Running loglikelihood requests:  49%|████▊     | 1665/3420 [01:58<01:19, 22.17it/s]
Running loglikelihood requests:  49%|████▉     | 1673/3420 [01:58<01:18, 22.18it/s]
Running loglikelihood requests:  49%|████▉     | 1681/3420 [01:58<01:18, 22.19it/s]
Running loglikelihood requests:  49%|████▉     | 1689/3420 [01:59<01:17, 22.23it/s]
Running loglikelihood requests:  50%|████▉     | 1697/3420 [01:59<01:17, 22.23it/s]
Running loglikelihood requests:  50%|████▉     | 1705/3420 [02:00<01:17, 22.25it/s]
Running loglikelihood requests:  50%|█████     | 1713/3420 [02:00<01:16, 22.27it/s]
Running loglikelihood requests:  50%|█████     | 1721/3420 [02:00<01:16, 22.28it/s]
Running loglikelihood requests:  51%|█████     | 1729/3420 [02:01<01:15, 22.30it/s]
Running loglikelihood requests:  51%|█████     | 1737/3420 [02:01<01:15, 22.32it/s]
Running loglikelihood requests:  51%|█████     | 1745/3420 [02:01<01:15, 22.33it/s]
Running loglikelihood requests:  51%|█████▏    | 1753/3420 [02:02<01:14, 22.34it/s]
Running loglikelihood requests:  51%|█████▏    | 1761/3420 [02:02<01:14, 22.33it/s]
Running loglikelihood requests:  52%|█████▏    | 1769/3420 [02:02<01:13, 22.34it/s]
Running loglikelihood requests:  52%|█████▏    | 1777/3420 [02:03<01:13, 22.34it/s]
Running loglikelihood requests:  52%|█████▏    | 1785/3420 [02:03<01:13, 22.35it/s]
Running loglikelihood requests:  52%|█████▏    | 1793/3420 [02:03<01:12, 22.37it/s]
Running loglikelihood requests:  53%|█████▎    | 1801/3420 [02:04<01:12, 22.40it/s]
Running loglikelihood requests:  53%|█████▎    | 1809/3420 [02:04<01:11, 22.42it/s]
Running loglikelihood requests:  53%|█████▎    | 1817/3420 [02:05<01:11, 22.42it/s]
Running loglikelihood requests:  53%|█████▎    | 1825/3420 [02:05<01:11, 22.42it/s]
Running loglikelihood requests:  54%|█████▎    | 1833/3420 [02:05<01:10, 22.41it/s]
Running loglikelihood requests:  54%|█████▍    | 1841/3420 [02:06<01:10, 22.42it/s]
Running loglikelihood requests:  54%|█████▍    | 1849/3420 [02:06<01:10, 22.44it/s]
Running loglikelihood requests:  54%|█████▍    | 1857/3420 [02:06<01:09, 22.45it/s]
Running loglikelihood requests:  55%|█████▍    | 1865/3420 [02:07<01:09, 22.44it/s]
Running loglikelihood requests:  55%|█████▍    | 1873/3420 [02:07<01:08, 22.44it/s]
Running loglikelihood requests:  55%|█████▌    | 1881/3420 [02:07<01:06, 23.20it/s]
Running loglikelihood requests:  55%|█████▌    | 1889/3420 [02:08<01:04, 23.82it/s]
Running loglikelihood requests:  55%|█████▌    | 1897/3420 [02:08<01:02, 24.28it/s]
Running loglikelihood requests:  56%|█████▌    | 1905/3420 [02:08<01:01, 24.56it/s]
Running loglikelihood requests:  56%|█████▌    | 1913/3420 [02:09<01:00, 24.78it/s]
Running loglikelihood requests:  56%|█████▌    | 1921/3420 [02:09<00:59, 24.98it/s]
Running loglikelihood requests:  56%|█████▋    | 1929/3420 [02:09<00:59, 25.13it/s]
Running loglikelihood requests:  57%|█████▋    | 1937/3420 [02:10<00:58, 25.23it/s]
Running loglikelihood requests:  57%|█████▋    | 1945/3420 [02:10<00:58, 25.27it/s]
Running loglikelihood requests:  57%|█████▋    | 1953/3420 [02:10<00:57, 25.31it/s]
Running loglikelihood requests:  57%|█████▋    | 1961/3420 [02:11<00:57, 25.38it/s]
Running loglikelihood requests:  58%|█████▊    | 1969/3420 [02:11<00:57, 25.40it/s]
Running loglikelihood requests:  58%|█████▊    | 1977/3420 [02:11<00:56, 25.41it/s]
Running loglikelihood requests:  58%|█████▊    | 1985/3420 [02:11<00:56, 25.47it/s]
Running loglikelihood requests:  58%|█████▊    | 1993/3420 [02:12<00:56, 25.47it/s]
Running loglikelihood requests:  59%|█████▊    | 2001/3420 [02:12<00:55, 25.47it/s]
Running loglikelihood requests:  59%|█████▊    | 2009/3420 [02:12<00:55, 25.52it/s]
Running loglikelihood requests:  59%|█████▉    | 2017/3420 [02:13<00:54, 25.52it/s]
Running loglikelihood requests:  59%|█████▉    | 2025/3420 [02:13<00:54, 25.51it/s]
Running loglikelihood requests:  59%|█████▉    | 2033/3420 [02:13<00:54, 25.56it/s]
Running loglikelihood requests:  60%|█████▉    | 2041/3420 [02:14<00:53, 25.56it/s]
Running loglikelihood requests:  60%|█████▉    | 2049/3420 [02:14<00:53, 25.59it/s]
Running loglikelihood requests:  60%|██████    | 2057/3420 [02:14<00:53, 25.63it/s]
Running loglikelihood requests:  60%|██████    | 2065/3420 [02:15<00:52, 25.60it/s]
Running loglikelihood requests:  61%|██████    | 2073/3420 [02:15<00:52, 25.64it/s]
Running loglikelihood requests:  61%|██████    | 2081/3420 [02:15<00:52, 25.65it/s]
Running loglikelihood requests:  61%|██████    | 2089/3420 [02:16<00:51, 25.66it/s]
Running loglikelihood requests:  61%|██████▏   | 2097/3420 [02:16<00:51, 25.66it/s]
Running loglikelihood requests:  62%|██████▏   | 2105/3420 [02:16<00:51, 25.64it/s]
Running loglikelihood requests:  62%|██████▏   | 2113/3420 [02:16<00:50, 25.66it/s]
Running loglikelihood requests:  62%|██████▏   | 2121/3420 [02:17<00:50, 25.65it/s]
Running loglikelihood requests:  62%|██████▏   | 2129/3420 [02:17<00:50, 25.68it/s]
Running loglikelihood requests:  62%|██████▏   | 2137/3420 [02:17<00:49, 25.73it/s]
Running loglikelihood requests:  63%|██████▎   | 2145/3420 [02:18<00:49, 25.69it/s]
Running loglikelihood requests:  63%|██████▎   | 2153/3420 [02:18<00:49, 25.71it/s]
Running loglikelihood requests:  63%|██████▎   | 2161/3420 [02:18<00:49, 25.69it/s]
Running loglikelihood requests:  63%|██████▎   | 2169/3420 [02:19<00:48, 25.78it/s]
Running loglikelihood requests:  64%|██████▎   | 2177/3420 [02:19<00:48, 25.80it/s]
Running loglikelihood requests:  64%|██████▍   | 2185/3420 [02:19<00:47, 25.87it/s]
Running loglikelihood requests:  64%|██████▍   | 2193/3420 [02:20<00:47, 25.87it/s]
Running loglikelihood requests:  64%|██████▍   | 2201/3420 [02:20<00:47, 25.91it/s]
Running loglikelihood requests:  65%|██████▍   | 2209/3420 [02:20<00:46, 25.91it/s]
Running loglikelihood requests:  65%|██████▍   | 2217/3420 [02:20<00:46, 25.92it/s]
Running loglikelihood requests:  65%|██████▌   | 2225/3420 [02:21<00:46, 25.98it/s]
Running loglikelihood requests:  65%|██████▌   | 2233/3420 [02:21<00:45, 25.97it/s]
Running loglikelihood requests:  66%|██████▌   | 2241/3420 [02:21<00:45, 26.02it/s]
Running loglikelihood requests:  66%|██████▌   | 2249/3420 [02:22<00:44, 26.06it/s]
Running loglikelihood requests:  66%|██████▌   | 2257/3420 [02:22<00:44, 26.05it/s]
Running loglikelihood requests:  66%|██████▌   | 2265/3420 [02:22<00:44, 26.07it/s]
Running loglikelihood requests:  66%|██████▋   | 2273/3420 [02:23<00:43, 26.13it/s]
Running loglikelihood requests:  67%|██████▋   | 2281/3420 [02:23<00:43, 26.11it/s]
Running loglikelihood requests:  67%|██████▋   | 2289/3420 [02:23<00:43, 26.11it/s]
Running loglikelihood requests:  67%|██████▋   | 2297/3420 [02:24<00:42, 26.16it/s]
Running loglikelihood requests:  67%|██████▋   | 2305/3420 [02:24<00:42, 26.15it/s]
Running loglikelihood requests:  68%|██████▊   | 2313/3420 [02:24<00:42, 26.14it/s]
Running loglikelihood requests:  68%|██████▊   | 2321/3420 [02:24<00:42, 26.16it/s]
Running loglikelihood requests:  68%|██████▊   | 2329/3420 [02:25<00:41, 26.17it/s]
Running loglikelihood requests:  68%|██████▊   | 2337/3420 [02:25<00:41, 26.17it/s]
Running loglikelihood requests:  69%|██████▊   | 2345/3420 [02:25<00:41, 26.18it/s]
Running loglikelihood requests:  69%|██████▉   | 2353/3420 [02:26<00:40, 26.21it/s]
Running loglikelihood requests:  69%|██████▉   | 2361/3420 [02:26<00:40, 26.22it/s]
Running loglikelihood requests:  69%|██████▉   | 2369/3420 [02:26<00:40, 26.22it/s]
Running loglikelihood requests:  70%|██████▉   | 2377/3420 [02:27<00:38, 26.92it/s]
Running loglikelihood requests:  70%|██████▉   | 2385/3420 [02:27<00:37, 27.43it/s]
Running loglikelihood requests:  70%|██████▉   | 2393/3420 [02:27<00:36, 27.81it/s]
Running loglikelihood requests:  70%|███████   | 2401/3420 [02:27<00:36, 28.08it/s]
Running loglikelihood requests:  70%|███████   | 2409/3420 [02:28<00:35, 28.27it/s]
Running loglikelihood requests:  71%|███████   | 2417/3420 [02:28<00:35, 28.42it/s]
Running loglikelihood requests:  71%|███████   | 2425/3420 [02:28<00:34, 28.52it/s]
Running loglikelihood requests:  71%|███████   | 2433/3420 [02:29<00:34, 28.65it/s]
Running loglikelihood requests:  71%|███████▏  | 2441/3420 [02:29<00:34, 28.70it/s]
Running loglikelihood requests:  72%|███████▏  | 2449/3420 [02:29<00:33, 28.75it/s]
Running loglikelihood requests:  72%|███████▏  | 2457/3420 [02:29<00:33, 28.79it/s]
Running loglikelihood requests:  72%|███████▏  | 2465/3420 [02:30<00:33, 28.82it/s]
Running loglikelihood requests:  72%|███████▏  | 2473/3420 [02:30<00:32, 28.84it/s]
Running loglikelihood requests:  73%|███████▎  | 2481/3420 [02:30<00:32, 28.86it/s]
Running loglikelihood requests:  73%|███████▎  | 2489/3420 [02:30<00:32, 28.93it/s]
Running loglikelihood requests:  73%|███████▎  | 2501/3420 [02:31<00:27, 33.25it/s]
Running loglikelihood requests:  73%|███████▎  | 2509/3420 [02:31<00:28, 31.97it/s]
Running loglikelihood requests:  74%|███████▎  | 2517/3420 [02:31<00:29, 31.07it/s]
Running loglikelihood requests:  74%|███████▍  | 2525/3420 [02:32<00:29, 30.44it/s]
Running loglikelihood requests:  74%|███████▍  | 2533/3420 [02:32<00:29, 30.01it/s]
Running loglikelihood requests:  74%|███████▍  | 2541/3420 [02:32<00:29, 29.69it/s]
Running loglikelihood requests:  75%|███████▍  | 2549/3420 [02:32<00:29, 29.50it/s]
Running loglikelihood requests:  75%|███████▍  | 2557/3420 [02:33<00:29, 29.37it/s]
Running loglikelihood requests:  75%|███████▌  | 2565/3420 [02:33<00:29, 29.27it/s]
Running loglikelihood requests:  75%|███████▌  | 2573/3420 [02:33<00:28, 29.21it/s]
Running loglikelihood requests:  75%|███████▌  | 2581/3420 [02:33<00:28, 29.28it/s]
Running loglikelihood requests:  76%|███████▌  | 2589/3420 [02:34<00:28, 29.32it/s]
Running loglikelihood requests:  76%|███████▌  | 2597/3420 [02:34<00:28, 29.39it/s]
Running loglikelihood requests:  76%|███████▌  | 2605/3420 [02:34<00:27, 29.42it/s]
Running loglikelihood requests:  76%|███████▋  | 2613/3420 [02:35<00:27, 29.45it/s]
Running loglikelihood requests:  77%|███████▋  | 2621/3420 [02:35<00:27, 29.46it/s]
Running loglikelihood requests:  77%|███████▋  | 2629/3420 [02:35<00:26, 29.50it/s]
Running loglikelihood requests:  77%|███████▋  | 2637/3420 [02:35<00:26, 29.49it/s]
Running loglikelihood requests:  77%|███████▋  | 2645/3420 [02:36<00:26, 29.50it/s]
Running loglikelihood requests:  78%|███████▊  | 2653/3420 [02:36<00:25, 29.58it/s]
Running loglikelihood requests:  78%|███████▊  | 2661/3420 [02:36<00:25, 29.55it/s]
Running loglikelihood requests:  78%|███████▊  | 2669/3420 [02:36<00:25, 29.54it/s]
Running loglikelihood requests:  78%|███████▊  | 2677/3420 [02:37<00:25, 29.53it/s]
Running loglikelihood requests:  79%|███████▊  | 2685/3420 [02:37<00:24, 29.57it/s]
Running loglikelihood requests:  79%|███████▊  | 2693/3420 [02:37<00:24, 29.56it/s]
Running loglikelihood requests:  79%|███████▉  | 2701/3420 [02:38<00:24, 29.58it/s]
Running loglikelihood requests:  79%|███████▉  | 2709/3420 [02:38<00:24, 29.57it/s]
Running loglikelihood requests:  79%|███████▉  | 2717/3420 [02:38<00:23, 29.57it/s]
Running loglikelihood requests:  80%|███████▉  | 2725/3420 [02:38<00:23, 29.57it/s]
Running loglikelihood requests:  80%|███████▉  | 2733/3420 [02:39<00:23, 29.57it/s]
Running loglikelihood requests:  80%|████████  | 2741/3420 [02:39<00:22, 29.64it/s]
Running loglikelihood requests:  80%|████████  | 2749/3420 [02:39<00:22, 29.64it/s]
Running loglikelihood requests:  81%|████████  | 2757/3420 [02:39<00:22, 29.69it/s]
Running loglikelihood requests:  81%|████████  | 2765/3420 [02:40<00:22, 29.68it/s]
Running loglikelihood requests:  81%|████████  | 2773/3420 [02:40<00:21, 29.69it/s]
Running loglikelihood requests:  81%|████████▏ | 2781/3420 [02:40<00:21, 29.74it/s]
Running loglikelihood requests:  82%|████████▏ | 2789/3420 [02:41<00:21, 29.72it/s]
Running loglikelihood requests:  82%|████████▏ | 2797/3420 [02:41<00:20, 29.77it/s]
Running loglikelihood requests:  82%|████████▏ | 2805/3420 [02:41<00:20, 29.76it/s]
Running loglikelihood requests:  82%|████████▏ | 2813/3420 [02:41<00:20, 29.81it/s]
Running loglikelihood requests:  82%|████████▏ | 2821/3420 [02:42<00:20, 29.79it/s]
Running loglikelihood requests:  83%|████████▎ | 2829/3420 [02:42<00:19, 29.83it/s]
Running loglikelihood requests:  83%|████████▎ | 2837/3420 [02:42<00:19, 29.81it/s]
Running loglikelihood requests:  83%|████████▎ | 2845/3420 [02:42<00:19, 29.80it/s]
Running loglikelihood requests:  83%|████████▎ | 2853/3420 [02:43<00:19, 29.81it/s]
Running loglikelihood requests:  84%|████████▎ | 2861/3420 [02:43<00:18, 29.81it/s]
Running loglikelihood requests:  84%|████████▍ | 2869/3420 [02:43<00:18, 29.86it/s]
Running loglikelihood requests:  84%|████████▍ | 2877/3420 [02:43<00:18, 29.90it/s]
Running loglikelihood requests:  84%|████████▍ | 2885/3420 [02:44<00:17, 29.88it/s]
Running loglikelihood requests:  85%|████████▍ | 2893/3420 [02:44<00:17, 29.91it/s]
Running loglikelihood requests:  85%|████████▍ | 2901/3420 [02:44<00:17, 29.93it/s]
Running loglikelihood requests:  85%|████████▌ | 2909/3420 [02:45<00:17, 29.93it/s]
Running loglikelihood requests:  85%|████████▌ | 2917/3420 [02:45<00:16, 29.96it/s]
Running loglikelihood requests:  86%|████████▌ | 2925/3420 [02:45<00:16, 30.52it/s]
Running loglikelihood requests:  86%|████████▌ | 2933/3420 [02:45<00:15, 30.94it/s]
Running loglikelihood requests:  86%|████████▌ | 2941/3420 [02:46<00:15, 31.21it/s]
Running loglikelihood requests:  86%|████████▌ | 2949/3420 [02:46<00:14, 31.46it/s]
Running loglikelihood requests:  86%|████████▋ | 2957/3420 [02:46<00:14, 31.65it/s]
Running loglikelihood requests:  87%|████████▋ | 2965/3420 [02:46<00:14, 31.81it/s]
Running loglikelihood requests:  87%|████████▋ | 2973/3420 [02:47<00:14, 31.92it/s]
Running loglikelihood requests:  87%|████████▋ | 2981/3420 [02:47<00:13, 32.07it/s]
Running loglikelihood requests:  87%|████████▋ | 2989/3420 [02:47<00:13, 32.13it/s]
Running loglikelihood requests:  88%|████████▊ | 2997/3420 [02:47<00:13, 32.16it/s]
Running loglikelihood requests:  88%|████████▊ | 3005/3420 [02:48<00:12, 32.20it/s]
Running loglikelihood requests:  88%|████████▊ | 3013/3420 [02:48<00:12, 32.28it/s]
Running loglikelihood requests:  88%|████████▊ | 3021/3420 [02:48<00:12, 32.29it/s]
Running loglikelihood requests:  89%|████████▊ | 3029/3420 [02:48<00:12, 32.31it/s]
Running loglikelihood requests:  89%|████████▉ | 3037/3420 [02:49<00:11, 32.37it/s]
Running loglikelihood requests:  89%|████████▉ | 3045/3420 [02:49<00:11, 32.37it/s]
Running loglikelihood requests:  89%|████████▉ | 3053/3420 [02:49<00:11, 32.39it/s]
Running loglikelihood requests:  90%|████████▉ | 3061/3420 [02:49<00:11, 32.39it/s]
Running loglikelihood requests:  90%|████████▉ | 3069/3420 [02:50<00:10, 32.47it/s]
Running loglikelihood requests:  90%|████████▉ | 3077/3420 [02:50<00:10, 32.47it/s]
Running loglikelihood requests:  90%|█████████ | 3085/3420 [02:50<00:10, 32.52it/s]
Running loglikelihood requests:  90%|█████████ | 3093/3420 [02:50<00:10, 32.50it/s]
Running loglikelihood requests:  91%|█████████ | 3101/3420 [02:50<00:09, 32.50it/s]
Running loglikelihood requests:  91%|█████████ | 3109/3420 [02:51<00:09, 32.55it/s]
Running loglikelihood requests:  91%|█████████ | 3117/3420 [02:51<00:09, 32.54it/s]
Running loglikelihood requests:  91%|█████████▏| 3125/3420 [02:51<00:09, 32.61it/s]
Running loglikelihood requests:  92%|█████████▏| 3133/3420 [02:51<00:08, 32.60it/s]
Running loglikelihood requests:  92%|█████████▏| 3141/3420 [02:52<00:08, 32.66it/s]
Running loglikelihood requests:  92%|█████████▏| 3149/3420 [02:52<00:08, 32.64it/s]
Running loglikelihood requests:  92%|█████████▏| 3157/3420 [02:52<00:08, 32.71it/s]
Running loglikelihood requests:  93%|█████████▎| 3165/3420 [02:52<00:07, 32.70it/s]
Running loglikelihood requests:  93%|█████████▎| 3173/3420 [02:53<00:07, 32.77it/s]
Running loglikelihood requests:  93%|█████████▎| 3181/3420 [02:53<00:07, 32.84it/s]
Running loglikelihood requests:  93%|█████████▎| 3189/3420 [02:53<00:07, 32.81it/s]
Running loglikelihood requests:  93%|█████████▎| 3197/3420 [02:53<00:06, 32.88it/s]
Running loglikelihood requests:  94%|█████████▎| 3205/3420 [02:54<00:06, 32.93it/s]
Running loglikelihood requests:  94%|█████████▍| 3213/3420 [02:54<00:06, 32.92it/s]
Running loglikelihood requests:  94%|█████████▍| 3221/3420 [02:54<00:06, 32.96it/s]
Running loglikelihood requests:  94%|█████████▍| 3229/3420 [02:54<00:05, 33.02it/s]
Running loglikelihood requests:  95%|█████████▍| 3237/3420 [02:55<00:05, 33.10it/s]
Running loglikelihood requests:  95%|█████████▍| 3245/3420 [02:55<00:05, 33.14it/s]
Running loglikelihood requests:  95%|█████████▌| 3253/3420 [02:55<00:05, 33.17it/s]
Running loglikelihood requests:  95%|█████████▌| 3261/3420 [02:55<00:04, 33.25it/s]
Running loglikelihood requests:  96%|█████████▌| 3269/3420 [02:56<00:04, 33.30it/s]
Running loglikelihood requests:  96%|█████████▌| 3277/3420 [02:56<00:04, 33.34it/s]
Running loglikelihood requests:  96%|█████████▌| 3285/3420 [02:56<00:04, 33.36it/s]
Running loglikelihood requests:  96%|█████████▋| 3293/3420 [02:56<00:03, 33.40it/s]
Running loglikelihood requests:  97%|█████████▋| 3301/3420 [02:57<00:03, 33.46it/s]
Running loglikelihood requests:  97%|█████████▋| 3309/3420 [02:57<00:03, 33.53it/s]
Running loglikelihood requests:  97%|█████████▋| 3317/3420 [02:57<00:03, 33.53it/s]
Running loglikelihood requests:  97%|█████████▋| 3325/3420 [02:57<00:02, 33.55it/s]
Running loglikelihood requests:  97%|█████████▋| 3333/3420 [02:58<00:02, 33.59it/s]
Running loglikelihood requests:  98%|█████████▊| 3341/3420 [02:58<00:02, 35.18it/s]
Running loglikelihood requests:  98%|█████████▊| 3349/3420 [02:58<00:01, 36.40it/s]
Running loglikelihood requests:  98%|█████████▊| 3357/3420 [02:58<00:01, 37.37it/s]
Running loglikelihood requests:  98%|█████████▊| 3365/3420 [02:58<00:01, 38.08it/s]
Running loglikelihood requests:  99%|█████████▊| 3373/3420 [02:59<00:01, 38.61it/s]
Running loglikelihood requests:  99%|█████████▉| 3381/3420 [02:59<00:01, 39.00it/s]
Running loglikelihood requests:  99%|█████████▉| 3389/3420 [02:59<00:00, 39.29it/s]
Running loglikelihood requests:  99%|█████████▉| 3397/3420 [02:59<00:00, 39.51it/s]
Running loglikelihood requests: 100%|█████████▉| 3405/3420 [02:59<00:00, 39.71it/s]
Running loglikelihood requests: 100%|█████████▉| 3413/3420 [03:00<00:00, 39.90it/s]
Running loglikelihood requests: 100%|██████████| 3420/3420 [03:00<00:00, 19.00it/s]
+fatal: detected dubious ownership in repository at '/root/steering-resistance'
+To add an exception for this directory, call:
+
+	git config --global --add safe.directory /root/steering-resistance
+2026-07-24:07:33:33 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+hf ({'pretrained': 'Qwen/Qwen2.5-7B-Instruct', 'dtype': 'bfloat16', 'peft': '/root/steering-resistance/results/full_7b/m1_resist_adapter'}), gen_kwargs: ({}), limit: 15.0, num_fewshot: 5, batch_size: 2
+|                 Tasks                 |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|---------------------------------------|------:|------|-----:|------|---|-----:|---|-----:|
+|mmlu                                   |      2|none  |      |acc   |   |0.7649|±  |0.0140|
+| - humanities                          |      2|none  |     5|acc   |↑  |0.8000|±  |0.0276|
+|  - formal_logic                       |      1|none  |     5|acc   |↑  |0.6000|±  |0.1309|
+|  - high_school_european_history       |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - high_school_us_history             |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - high_school_world_history          |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - international_law                  |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - jurisprudence                      |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - logical_fallacies                  |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - moral_disputes                     |      1|none  |     5|acc   |↑  |0.6000|±  |0.1309|
+|  - moral_scenarios                    |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - philosophy                         |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - prehistory                         |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - professional_law                   |      1|none  |     5|acc   |↑  |0.6000|±  |0.1309|
+|  - world_religions                    |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+| - other                               |      2|none  |     5|acc   |↑  |0.7487|±  |0.0303|
+|  - business_ethics                    |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - clinical_knowledge                 |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - college_medicine                   |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - global_facts                       |      1|none  |     5|acc   |↑  |0.4667|±  |0.1333|
+|  - human_aging                        |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - management                         |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - marketing                          |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - medical_genetics                   |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - miscellaneous                      |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - nutrition                          |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - professional_accounting            |      1|none  |     5|acc   |↑  |0.4667|±  |0.1333|
+|  - professional_medicine              |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - virology                           |      1|none  |     5|acc   |↑  |0.6000|±  |0.1309|
+| - social sciences                     |      2|none  |     5|acc   |↑  |0.8278|±  |0.0279|
+|  - econometrics                       |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - high_school_geography              |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - high_school_government_and_politics|      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - high_school_macroeconomics         |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - high_school_microeconomics         |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - high_school_psychology             |      1|none  |     5|acc   |↑  |1.0000|±  |0.0000|
+|  - human_sexuality                    |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - professional_psychology            |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - public_relations                   |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - security_studies                   |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - sociology                          |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - us_foreign_policy                  |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+| - stem                                |      2|none  |     5|acc   |↑  |0.7123|±  |0.0259|
+|  - abstract_algebra                   |      1|none  |     5|acc   |↑  |0.5333|±  |0.1333|
+|  - anatomy                            |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - astronomy                          |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - college_biology                    |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - college_chemistry                  |      1|none  |     5|acc   |↑  |0.7333|±  |0.1182|
+|  - college_computer_science           |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - college_mathematics                |      1|none  |     5|acc   |↑  |0.4000|±  |0.1309|
+|  - college_physics                    |      1|none  |     5|acc   |↑  |0.4667|±  |0.1333|
+|  - computer_security                  |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - conceptual_physics                 |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - electrical_engineering             |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - elementary_mathematics             |      1|none  |     5|acc   |↑  |0.8000|±  |0.1069|
+|  - high_school_biology                |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - high_school_chemistry              |      1|none  |     5|acc   |↑  |0.5333|±  |0.1333|
+|  - high_school_computer_science       |      1|none  |     5|acc   |↑  |0.9333|±  |0.0667|
+|  - high_school_mathematics            |      1|none  |     5|acc   |↑  |0.5333|±  |0.1333|
+|  - high_school_physics                |      1|none  |     5|acc   |↑  |0.6667|±  |0.1260|
+|  - high_school_statistics             |      1|none  |     5|acc   |↑  |0.8667|±  |0.0909|
+|  - machine_learning                   |      1|none  |     5|acc   |↑  |0.6000|±  |0.1309|
+
+|      Groups      |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|------------------|------:|------|-----:|------|---|-----:|---|-----:|
+|mmlu              |      2|none  |      |acc   |   |0.7649|±  |0.0140|
+| - humanities     |      2|none  |     5|acc   |↑  |0.8000|±  |0.0276|
+| - other          |      2|none  |     5|acc   |↑  |0.7487|±  |0.0303|
+| - social sciences|      2|none  |     5|acc   |↑  |0.8278|±  |0.0279|
+| - stem           |      2|none  |     5|acc   |↑  |0.7123|±  |0.0259|
+
diff --git a/run/capability/m0/gsm8k_cot/Qwen__Qwen2.5-7B-Instruct/results_2026-07-24T07-13-50.147487.json b/run/capability/m0/gsm8k_cot/Qwen__Qwen2.5-7B-Instruct/results_2026-07-24T07-13-50.147487.json
new file mode 100644
index 0000000000000000000000000000000000000000..7eeb9ef5a70fc749fb99f7bd193409c307a0ee6d
--- /dev/null
+++ b/run/capability/m0/gsm8k_cot/Qwen__Qwen2.5-7B-Instruct/results_2026-07-24T07-13-50.147487.json
@@ -0,0 +1,205 @@
+{
+  "results": {
+    "gsm8k_cot": {
+      "name": "gsm8k_cot",
+      "alias": "gsm8k_cot",
+      "sample_len": 200,
+      "exact_match,strict-match": 0.33,
+      "exact_match_stderr,strict-match": 0.03333249580187337,
+      "exact_match,flexible-extract": 0.78,
+      "exact_match_stderr,flexible-extract": 0.029365141882663287
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "gsm8k_cot": {
+      "task": "gsm8k_cot",
+      "dataset_path": "openai/gsm8k",
+      "dataset_name": "main",
+      "test_split": "test",
+      "doc_to_text": "Q: {{question}}\nA:",
+      "doc_to_target": "{{answer.split('####')[-1].strip() if answer is defined else target}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": [
+          {
+            "question": "There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?",
+            "target": "There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have been 21 - 15 = 6. The answer is 6."
+          },
+          {
+            "question": "If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?",
+            "target": "There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5."
+          },
+          {
+            "question": "Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?",
+            "target": "Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The answer is 39."
+          },
+          {
+            "question": "Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?",
+            "target": "Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8. The answer is 8."
+          },
+          {
+            "question": "Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?",
+            "target": "Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9. The answer is 9."
+          },
+          {
+            "question": "There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?",
+            "target": "There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The answer is 29."
+          },
+          {
+            "question": "Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?",
+            "target": "Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The answer is 33."
+          },
+          {
+            "question": "Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
+            "target": "Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The answer is 8."
+          }
+        ],
+        "doc_to_text": "Q: {{question}}\nA:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer.split('####')[-1].strip() if answer is defined else target}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 8,
+      "metric_list": [
+        {
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "metric": "exact_match",
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "until": [
+          "Q:",
+          "</s>",
+          "<|im_end|>"
+        ]
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "The answer is (\\-?[0-9\\.\\,]+)."
+            },
+            {
+              "function": "take_first"
+            }
+          ],
+          "name": "strict-match"
+        },
+        {
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ],
+          "name": "flexible-extract"
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/gsm8k/gsm8k-cot.yaml"
+      }
+    }
+  },
+  "versions": {
+    "gsm8k_cot": 3.0
+  },
+  "n-shot": {
+    "gsm8k_cot": 8
+  },
+  "higher_is_better": {
+    "gsm8k_cot": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm8k_cot": {
+      "original": 1319,
+      "effective": 200
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+      "dtype": "bfloat16"
+    },
+    "model_num_parameters": 7615616512,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "a09a35458c702b33eeacc393d103063234e8bc28",
+    "batch_size": "4",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": 200.0,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 0,
+    "numpy_seed": 0,
+    "torch_seed": 0,
+    "fewshot_seed": 0
+  },
+  "git_hash": "eb4f2be22f7baf6d268c3dd5e46d49d6bd2e74ae",
+  "date": 1784876841.7269874,
+  "pretty_env_info": "PyTorch version: 2.13.0+cu126\nIs debug build: False\nCUDA used to build PyTorch: 12.6\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.11.10 (main, Sep  7 2024, 18:35:41) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.8.0-49-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3090\nNvidia driver version: 565.57.01\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: False\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               48\nOn-line CPU(s) list:                  0-47\nVendor ID:                            AuthenticAMD\nModel name:                           AMD EPYC 7402P 24-Core Processor\nCPU family:                           23\nModel:                                49\nThread(s) per core:                   2\nCore(s) per socket:                   24\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU max MHz:                          2800.0000\nCPU min MHz:                          1500.0000\nBogoMIPS:                             5600.34\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            768 KiB (24 instances)\nL1i cache:                            768 KiB (24 instances)\nL2 cache:                             12 MiB (24 instances)\nL3 cache:                             128 MiB (8 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-47\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.6.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.6.80\n[pip3] nvidia-cuda-nvrtc-cu12==12.6.85\n[pip3] nvidia-cuda-runtime-cu12==12.6.77\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.0.4\n[pip3] nvidia-curand-cu12==10.3.7.77\n[pip3] nvidia-cusolver-cu12==11.7.1.2\n[pip3] nvidia-cusparse-cu12==12.5.4.2\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.29.3\n[pip3] nvidia-nvjitlink-cu12==12.6.85\n[pip3] nvidia-nvtx-cu12==12.6.77\n[pip3] torch==2.13.0+cu126\n[pip3] triton==3.7.1\n[conda] Could not collect",
+  "transformers_version": "5.14.1",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Qwen/Qwen2.5-7B-Instruct",
+  "model_name_sanitized": "Qwen__Qwen2.5-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "395.9185296650976"
+}
\ No newline at end of file
diff --git a/run/capability/m0/mmlu/Qwen__Qwen2.5-7B-Instruct/results_2026-07-24T07-29-00.026924.json b/run/capability/m0/mmlu/Qwen__Qwen2.5-7B-Instruct/results_2026-07-24T07-29-00.026924.json
new file mode 100644
index 0000000000000000000000000000000000000000..e43b7c012e068442f32c89cae974595de933fdee
--- /dev/null
+++ b/run/capability/m0/mmlu/Qwen__Qwen2.5-7B-Instruct/results_2026-07-24T07-29-00.026924.json
@@ -0,0 +1,4309 @@
+{
+  "results": {
+    "mmlu_abstract_algebra": {
+      "name": "mmlu_abstract_algebra",
+      "alias": "abstract_algebra",
+      "sample_len": 15,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_anatomy": {
+      "name": "mmlu_anatomy",
+      "alias": "anatomy",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_astronomy": {
+      "name": "mmlu_astronomy",
+      "alias": "astronomy",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_college_biology": {
+      "name": "mmlu_college_biology",
+      "alias": "college_biology",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589958
+    },
+    "mmlu_college_chemistry": {
+      "name": "mmlu_college_chemistry",
+      "alias": "college_chemistry",
+      "sample_len": 15,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.13333333333333333
+    },
+    "mmlu_college_computer_science": {
+      "name": "mmlu_college_computer_science",
+      "alias": "college_computer_science",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_college_mathematics": {
+      "name": "mmlu_college_mathematics",
+      "alias": "college_mathematics",
+      "sample_len": 15,
+      "acc,none": 0.3333333333333333,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_college_physics": {
+      "name": "mmlu_college_physics",
+      "alias": "college_physics",
+      "sample_len": 15,
+      "acc,none": 0.4,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_computer_security": {
+      "name": "mmlu_computer_security",
+      "alias": "computer_security",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_conceptual_physics": {
+      "name": "mmlu_conceptual_physics",
+      "alias": "conceptual_physics",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_electrical_engineering": {
+      "name": "mmlu_electrical_engineering",
+      "alias": "electrical_engineering",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_elementary_mathematics": {
+      "name": "mmlu_elementary_mathematics",
+      "alias": "elementary_mathematics",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705575
+    },
+    "mmlu_high_school_biology": {
+      "name": "mmlu_high_school_biology",
+      "alias": "high_school_biology",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_high_school_chemistry": {
+      "name": "mmlu_high_school_chemistry",
+      "alias": "high_school_chemistry",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_high_school_computer_science": {
+      "name": "mmlu_high_school_computer_science",
+      "alias": "high_school_computer_science",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_high_school_mathematics": {
+      "name": "mmlu_high_school_mathematics",
+      "alias": "high_school_mathematics",
+      "sample_len": 15,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_high_school_physics": {
+      "name": "mmlu_high_school_physics",
+      "alias": "high_school_physics",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_high_school_statistics": {
+      "name": "mmlu_high_school_statistics",
+      "alias": "high_school_statistics",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589958
+    },
+    "mmlu_machine_learning": {
+      "name": "mmlu_machine_learning",
+      "alias": "machine_learning",
+      "sample_len": 15,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_business_ethics": {
+      "name": "mmlu_business_ethics",
+      "alias": "business_ethics",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_clinical_knowledge": {
+      "name": "mmlu_clinical_knowledge",
+      "alias": "clinical_knowledge",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_college_medicine": {
+      "name": "mmlu_college_medicine",
+      "alias": "college_medicine",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_global_facts": {
+      "name": "mmlu_global_facts",
+      "alias": "global_facts",
+      "sample_len": 15,
+      "acc,none": 0.4,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_human_aging": {
+      "name": "mmlu_human_aging",
+      "alias": "human_aging",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_management": {
+      "name": "mmlu_management",
+      "alias": "management",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589958
+    },
+    "mmlu_marketing": {
+      "name": "mmlu_marketing",
+      "alias": "marketing",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_medical_genetics": {
+      "name": "mmlu_medical_genetics",
+      "alias": "medical_genetics",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589956
+    },
+    "mmlu_miscellaneous": {
+      "name": "mmlu_miscellaneous",
+      "alias": "miscellaneous",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_nutrition": {
+      "name": "mmlu_nutrition",
+      "alias": "nutrition",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_professional_accounting": {
+      "name": "mmlu_professional_accounting",
+      "alias": "professional_accounting",
+      "sample_len": 15,
+      "acc,none": 0.3333333333333333,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_professional_medicine": {
+      "name": "mmlu_professional_medicine",
+      "alias": "professional_medicine",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_virology": {
+      "name": "mmlu_virology",
+      "alias": "virology",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_econometrics": {
+      "name": "mmlu_econometrics",
+      "alias": "econometrics",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_high_school_geography": {
+      "name": "mmlu_high_school_geography",
+      "alias": "high_school_geography",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589956
+    },
+    "mmlu_high_school_government_and_politics": {
+      "name": "mmlu_high_school_government_and_politics",
+      "alias": "high_school_government_and_politics",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_high_school_macroeconomics": {
+      "name": "mmlu_high_school_macroeconomics",
+      "alias": "high_school_macroeconomics",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_high_school_microeconomics": {
+      "name": "mmlu_high_school_microeconomics",
+      "alias": "high_school_microeconomics",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_high_school_psychology": {
+      "name": "mmlu_high_school_psychology",
+      "alias": "high_school_psychology",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_human_sexuality": {
+      "name": "mmlu_human_sexuality",
+      "alias": "human_sexuality",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_professional_psychology": {
+      "name": "mmlu_professional_psychology",
+      "alias": "professional_psychology",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_public_relations": {
+      "name": "mmlu_public_relations",
+      "alias": "public_relations",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_security_studies": {
+      "name": "mmlu_security_studies",
+      "alias": "security_studies",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_sociology": {
+      "name": "mmlu_sociology",
+      "alias": "sociology",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705575
+    },
+    "mmlu_us_foreign_policy": {
+      "name": "mmlu_us_foreign_policy",
+      "alias": "us_foreign_policy",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_formal_logic": {
+      "name": "mmlu_formal_logic",
+      "alias": "formal_logic",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_high_school_european_history": {
+      "name": "mmlu_high_school_european_history",
+      "alias": "high_school_european_history",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589958
+    },
+    "mmlu_high_school_us_history": {
+      "name": "mmlu_high_school_us_history",
+      "alias": "high_school_us_history",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_high_school_world_history": {
+      "name": "mmlu_high_school_world_history",
+      "alias": "high_school_world_history",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_international_law": {
+      "name": "mmlu_international_law",
+      "alias": "international_law",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_jurisprudence": {
+      "name": "mmlu_jurisprudence",
+      "alias": "jurisprudence",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_logical_fallacies": {
+      "name": "mmlu_logical_fallacies",
+      "alias": "logical_fallacies",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_moral_disputes": {
+      "name": "mmlu_moral_disputes",
+      "alias": "moral_disputes",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_moral_scenarios": {
+      "name": "mmlu_moral_scenarios",
+      "alias": "moral_scenarios",
+      "sample_len": 15,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_philosophy": {
+      "name": "mmlu_philosophy",
+      "alias": "philosophy",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_prehistory": {
+      "name": "mmlu_prehistory",
+      "alias": "prehistory",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_professional_law": {
+      "name": "mmlu_professional_law",
+      "alias": "professional_law",
+      "sample_len": 15,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_world_religions": {
+      "name": "mmlu_world_religions",
+      "alias": "world_religions",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 285,
+      "acc,none": 0.6842105263157895,
+      "acc_stderr,none": 0.02675489953901257,
+      "sample_count": {
+        "acc,none": 285
+      }
+    },
+    "mmlu_other": {
+      "alias": "other",
+      "name": "mmlu_other",
+      "sample_len": 195,
+      "acc,none": 0.7384615384615385,
+      "acc_stderr,none": 0.02983943128852205,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu_social_sciences": {
+      "alias": "social sciences",
+      "name": "mmlu_social_sciences",
+      "sample_len": 180,
+      "acc,none": 0.8111111111111111,
+      "acc_stderr,none": 0.028943781791762647,
+      "sample_count": {
+        "acc,none": 180
+      }
+    },
+    "mmlu_humanities": {
+      "alias": "humanities",
+      "name": "mmlu_humanities",
+      "sample_len": 195,
+      "acc,none": 0.8051282051282052,
+      "acc_stderr,none": 0.027135910882713752,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu": {
+      "alias": "mmlu",
+      "name": "mmlu",
+      "sample_len": 855,
+      "acc,none": 0.7508771929824561,
+      "acc_stderr,none": 0.014187421697813722,
+      "sample_count": {
+        "acc,none": 855
+      }
+    }
+  },
+  "groups": {
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 285,
+      "acc,none": 0.6842105263157895,
+      "acc_stderr,none": 0.02675489953901257,
+      "sample_count": {
+        "acc,none": 285
+      }
+    },
+    "mmlu_other": {
+      "alias": "other",
+      "name": "mmlu_other",
+      "sample_len": 195,
+      "acc,none": 0.7384615384615385,
+      "acc_stderr,none": 0.02983943128852205,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu_social_sciences": {
+      "alias": "social sciences",
+      "name": "mmlu_social_sciences",
+      "sample_len": 180,
+      "acc,none": 0.8111111111111111,
+      "acc_stderr,none": 0.028943781791762647,
+      "sample_count": {
+        "acc,none": 180
+      }
+    },
+    "mmlu_humanities": {
+      "alias": "humanities",
+      "name": "mmlu_humanities",
+      "sample_len": 195,
+      "acc,none": 0.8051282051282052,
+      "acc_stderr,none": 0.027135910882713752,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu": {
+      "alias": "mmlu",
+      "name": "mmlu",
+      "sample_len": 855,
+      "acc,none": 0.7508771929824561,
+      "acc_stderr,none": 0.014187421697813722,
+      "sample_count": {
+        "acc,none": 855
+      }
+    }
+  },
+  "group_subtasks": {
+    "mmlu": [
+      "mmlu_stem",
+      "mmlu_other",
+      "mmlu_social_sciences",
+      "mmlu_humanities"
+    ],
+    "mmlu_stem": [
+      "mmlu_abstract_algebra",
+      "mmlu_anatomy",
+      "mmlu_astronomy",
+      "mmlu_college_biology",
+      "mmlu_college_chemistry",
+      "mmlu_college_computer_science",
+      "mmlu_college_mathematics",
+      "mmlu_college_physics",
+      "mmlu_computer_security",
+      "mmlu_conceptual_physics",
+      "mmlu_electrical_engineering",
+      "mmlu_elementary_mathematics",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_chemistry",
+      "mmlu_high_school_computer_science",
+      "mmlu_high_school_mathematics",
+      "mmlu_high_school_physics",
+      "mmlu_high_school_statistics",
+      "mmlu_machine_learning"
+    ],
+    "mmlu_other": [
+      "mmlu_business_ethics",
+      "mmlu_clinical_knowledge",
+      "mmlu_college_medicine",
+      "mmlu_global_facts",
+      "mmlu_human_aging",
+      "mmlu_management",
+      "mmlu_marketing",
+      "mmlu_medical_genetics",
+      "mmlu_miscellaneous",
+      "mmlu_nutrition",
+      "mmlu_professional_accounting",
+      "mmlu_professional_medicine",
+      "mmlu_virology"
+    ],
+    "mmlu_social_sciences": [
+      "mmlu_econometrics",
+      "mmlu_high_school_geography",
+      "mmlu_high_school_government_and_politics",
+      "mmlu_high_school_macroeconomics",
+      "mmlu_high_school_microeconomics",
+      "mmlu_high_school_psychology",
+      "mmlu_human_sexuality",
+      "mmlu_professional_psychology",
+      "mmlu_public_relations",
+      "mmlu_security_studies",
+      "mmlu_sociology",
+      "mmlu_us_foreign_policy"
+    ],
+    "mmlu_humanities": [
+      "mmlu_formal_logic",
+      "mmlu_high_school_european_history",
+      "mmlu_high_school_us_history",
+      "mmlu_high_school_world_history",
+      "mmlu_international_law",
+      "mmlu_jurisprudence",
+      "mmlu_logical_fallacies",
+      "mmlu_moral_disputes",
+      "mmlu_moral_scenarios",
+      "mmlu_philosophy",
+      "mmlu_prehistory",
+      "mmlu_professional_law",
+      "mmlu_world_religions"
+    ]
+  },
+  "configs": {
+    "mmlu_abstract_algebra": {
+      "task": "mmlu_abstract_algebra",
+      "task_alias": "abstract_algebra",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "abstract_algebra",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml"
+      }
+    },
+    "mmlu_anatomy": {
+      "task": "mmlu_anatomy",
+      "task_alias": "anatomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "anatomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml"
+      }
+    },
+    "mmlu_astronomy": {
+      "task": "mmlu_astronomy",
+      "task_alias": "astronomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "astronomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml"
+      }
+    },
+    "mmlu_business_ethics": {
+      "task": "mmlu_business_ethics",
+      "task_alias": "business_ethics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "business_ethics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml"
+      }
+    },
+    "mmlu_clinical_knowledge": {
+      "task": "mmlu_clinical_knowledge",
+      "task_alias": "clinical_knowledge",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "clinical_knowledge",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml"
+      }
+    },
+    "mmlu_college_biology": {
+      "task": "mmlu_college_biology",
+      "task_alias": "college_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml"
+      }
+    },
+    "mmlu_college_chemistry": {
+      "task": "mmlu_college_chemistry",
+      "task_alias": "college_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml"
+      }
+    },
+    "mmlu_college_computer_science": {
+      "task": "mmlu_college_computer_science",
+      "task_alias": "college_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml"
+      }
+    },
+    "mmlu_college_mathematics": {
+      "task": "mmlu_college_mathematics",
+      "task_alias": "college_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml"
+      }
+    },
+    "mmlu_college_medicine": {
+      "task": "mmlu_college_medicine",
+      "task_alias": "college_medicine",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_medicine",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml"
+      }
+    },
+    "mmlu_college_physics": {
+      "task": "mmlu_college_physics",
+      "task_alias": "college_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml"
+      }
+    },
+    "mmlu_computer_security": {
+      "task": "mmlu_computer_security",
+      "task_alias": "computer_security",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "computer_security",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about computer security.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml"
+      }
+    },
+    "mmlu_conceptual_physics": {
+      "task": "mmlu_conceptual_physics",
+      "task_alias": "conceptual_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "conceptual_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml"
+      }
+    },
+    "mmlu_econometrics": {
+      "task": "mmlu_econometrics",
+      "task_alias": "econometrics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "econometrics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml"
+      }
+    },
+    "mmlu_electrical_engineering": {
+      "task": "mmlu_electrical_engineering",
+      "task_alias": "electrical_engineering",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "electrical_engineering",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml"
+      }
+    },
+    "mmlu_elementary_mathematics": {
+      "task": "mmlu_elementary_mathematics",
+      "task_alias": "elementary_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "elementary_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml"
+      }
+    },
+    "mmlu_formal_logic": {
+      "task": "mmlu_formal_logic",
+      "task_alias": "formal_logic",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "formal_logic",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml"
+      }
+    },
+    "mmlu_global_facts": {
+      "task": "mmlu_global_facts",
+      "task_alias": "global_facts",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "global_facts",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about global facts.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml"
+      }
+    },
+    "mmlu_high_school_biology": {
+      "task": "mmlu_high_school_biology",
+      "task_alias": "high_school_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml"
+      }
+    },
+    "mmlu_high_school_chemistry": {
+      "task": "mmlu_high_school_chemistry",
+      "task_alias": "high_school_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml"
+      }
+    },
+    "mmlu_high_school_computer_science": {
+      "task": "mmlu_high_school_computer_science",
+      "task_alias": "high_school_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml"
+      }
+    },
+    "mmlu_high_school_european_history": {
+      "task": "mmlu_high_school_european_history",
+      "task_alias": "high_school_european_history",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_european_history",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml"
+      }
+    },
+    "mmlu_high_school_geography": {
+      "task": "mmlu_high_school_geography",
+      "task_alias": "high_school_geography",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_geography",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml"
+      }
+    },
+    "mmlu_high_school_government_and_politics": {
+      "task": "mmlu_high_school_government_and_politics",
+      "task_alias": "high_school_government_and_politics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_government_and_politics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml"
+      }
+    },
+    "mmlu_high_school_macroeconomics": {
+      "task": "mmlu_high_school_macroeconomics",
+      "task_alias": "high_school_macroeconomics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_macroeconomics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml"
+      }
+    },
+    "mmlu_high_school_mathematics": {
+      "task": "mmlu_high_school_mathematics",
+      "task_alias": "high_school_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml"
+      }
+    },
+    "mmlu_high_school_microeconomics": {
+      "task": "mmlu_high_school_microeconomics",
+      "task_alias": "high_school_microeconomics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_microeconomics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml"
+      }
+    },
+    "mmlu_high_school_physics": {
+      "task": "mmlu_high_school_physics",
+      "task_alias": "high_school_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml"
+      }
+    },
+    "mmlu_high_school_psychology": {
+      "task": "mmlu_high_school_psychology",
+      "task_alias": "high_school_psychology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_psychology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml"
+      }
+    },
+    "mmlu_high_school_statistics": {
+      "task": "mmlu_high_school_statistics",
+      "task_alias": "high_school_statistics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_statistics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml"
+      }
+    },
+    "mmlu_high_school_us_history": {
+      "task": "mmlu_high_school_us_history",
+      "task_alias": "high_school_us_history",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_us_history",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml"
+      }
+    },
+    "mmlu_high_school_world_history": {
+      "task": "mmlu_high_school_world_history",
+      "task_alias": "high_school_world_history",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_world_history",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml"
+      }
+    },
+    "mmlu_human_aging": {
+      "task": "mmlu_human_aging",
+      "task_alias": "human_aging",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "human_aging",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about human aging.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml"
+      }
+    },
+    "mmlu_human_sexuality": {
+      "task": "mmlu_human_sexuality",
+      "task_alias": "human_sexuality",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "human_sexuality",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml"
+      }
+    },
+    "mmlu_international_law": {
+      "task": "mmlu_international_law",
+      "task_alias": "international_law",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "international_law",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about international law.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml"
+      }
+    },
+    "mmlu_jurisprudence": {
+      "task": "mmlu_jurisprudence",
+      "task_alias": "jurisprudence",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "jurisprudence",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml"
+      }
+    },
+    "mmlu_logical_fallacies": {
+      "task": "mmlu_logical_fallacies",
+      "task_alias": "logical_fallacies",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "logical_fallacies",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml"
+      }
+    },
+    "mmlu_machine_learning": {
+      "task": "mmlu_machine_learning",
+      "task_alias": "machine_learning",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "machine_learning",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml"
+      }
+    },
+    "mmlu_management": {
+      "task": "mmlu_management",
+      "task_alias": "management",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "management",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about management.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_management.yaml"
+      }
+    },
+    "mmlu_marketing": {
+      "task": "mmlu_marketing",
+      "task_alias": "marketing",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "marketing",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about marketing.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml"
+      }
+    },
+    "mmlu_medical_genetics": {
+      "task": "mmlu_medical_genetics",
+      "task_alias": "medical_genetics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "medical_genetics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml"
+      }
+    },
+    "mmlu_miscellaneous": {
+      "task": "mmlu_miscellaneous",
+      "task_alias": "miscellaneous",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "miscellaneous",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml"
+      }
+    },
+    "mmlu_moral_disputes": {
+      "task": "mmlu_moral_disputes",
+      "task_alias": "moral_disputes",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "moral_disputes",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml"
+      }
+    },
+    "mmlu_moral_scenarios": {
+      "task": "mmlu_moral_scenarios",
+      "task_alias": "moral_scenarios",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "moral_scenarios",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml"
+      }
+    },
+    "mmlu_nutrition": {
+      "task": "mmlu_nutrition",
+      "task_alias": "nutrition",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "nutrition",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml"
+      }
+    },
+    "mmlu_philosophy": {
+      "task": "mmlu_philosophy",
+      "task_alias": "philosophy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "philosophy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml"
+      }
+    },
+    "mmlu_prehistory": {
+      "task": "mmlu_prehistory",
+      "task_alias": "prehistory",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "prehistory",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml"
+      }
+    },
+    "mmlu_professional_accounting": {
+      "task": "mmlu_professional_accounting",
+      "task_alias": "professional_accounting",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_accounting",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml"
+      }
+    },
+    "mmlu_professional_law": {
+      "task": "mmlu_professional_law",
+      "task_alias": "professional_law",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_law",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional law.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml"
+      }
+    },
+    "mmlu_professional_medicine": {
+      "task": "mmlu_professional_medicine",
+      "task_alias": "professional_medicine",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_medicine",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml"
+      }
+    },
+    "mmlu_professional_psychology": {
+      "task": "mmlu_professional_psychology",
+      "task_alias": "professional_psychology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_psychology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml"
+      }
+    },
+    "mmlu_public_relations": {
+      "task": "mmlu_public_relations",
+      "task_alias": "public_relations",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "public_relations",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about public relations.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml"
+      }
+    },
+    "mmlu_security_studies": {
+      "task": "mmlu_security_studies",
+      "task_alias": "security_studies",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "security_studies",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about security studies.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml"
+      }
+    },
+    "mmlu_sociology": {
+      "task": "mmlu_sociology",
+      "task_alias": "sociology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "sociology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about sociology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml"
+      }
+    },
+    "mmlu_us_foreign_policy": {
+      "task": "mmlu_us_foreign_policy",
+      "task_alias": "us_foreign_policy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "us_foreign_policy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml"
+      }
+    },
+    "mmlu_virology": {
+      "task": "mmlu_virology",
+      "task_alias": "virology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "virology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about virology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_virology.yaml"
+      }
+    },
+    "mmlu_world_religions": {
+      "task": "mmlu_world_religions",
+      "task_alias": "world_religions",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "world_religions",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about world religions.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mmlu": "2",
+    "mmlu_abstract_algebra": 1.0,
+    "mmlu_anatomy": 1.0,
+    "mmlu_astronomy": 1.0,
+    "mmlu_business_ethics": 1.0,
+    "mmlu_clinical_knowledge": 1.0,
+    "mmlu_college_biology": 1.0,
+    "mmlu_college_chemistry": 1.0,
+    "mmlu_college_computer_science": 1.0,
+    "mmlu_college_mathematics": 1.0,
+    "mmlu_college_medicine": 1.0,
+    "mmlu_college_physics": 1.0,
+    "mmlu_computer_security": 1.0,
+    "mmlu_conceptual_physics": 1.0,
+    "mmlu_econometrics": 1.0,
+    "mmlu_electrical_engineering": 1.0,
+    "mmlu_elementary_mathematics": 1.0,
+    "mmlu_formal_logic": 1.0,
+    "mmlu_global_facts": 1.0,
+    "mmlu_high_school_biology": 1.0,
+    "mmlu_high_school_chemistry": 1.0,
+    "mmlu_high_school_computer_science": 1.0,
+    "mmlu_high_school_european_history": 1.0,
+    "mmlu_high_school_geography": 1.0,
+    "mmlu_high_school_government_and_politics": 1.0,
+    "mmlu_high_school_macroeconomics": 1.0,
+    "mmlu_high_school_mathematics": 1.0,
+    "mmlu_high_school_microeconomics": 1.0,
+    "mmlu_high_school_physics": 1.0,
+    "mmlu_high_school_psychology": 1.0,
+    "mmlu_high_school_statistics": 1.0,
+    "mmlu_high_school_us_history": 1.0,
+    "mmlu_high_school_world_history": 1.0,
+    "mmlu_human_aging": 1.0,
+    "mmlu_human_sexuality": 1.0,
+    "mmlu_humanities": "2",
+    "mmlu_international_law": 1.0,
+    "mmlu_jurisprudence": 1.0,
+    "mmlu_logical_fallacies": 1.0,
+    "mmlu_machine_learning": 1.0,
+    "mmlu_management": 1.0,
+    "mmlu_marketing": 1.0,
+    "mmlu_medical_genetics": 1.0,
+    "mmlu_miscellaneous": 1.0,
+    "mmlu_moral_disputes": 1.0,
+    "mmlu_moral_scenarios": 1.0,
+    "mmlu_nutrition": 1.0,
+    "mmlu_other": "2",
+    "mmlu_philosophy": 1.0,
+    "mmlu_prehistory": 1.0,
+    "mmlu_professional_accounting": 1.0,
+    "mmlu_professional_law": 1.0,
+    "mmlu_professional_medicine": 1.0,
+    "mmlu_professional_psychology": 1.0,
+    "mmlu_public_relations": 1.0,
+    "mmlu_security_studies": 1.0,
+    "mmlu_social_sciences": "2",
+    "mmlu_sociology": 1.0,
+    "mmlu_stem": "2",
+    "mmlu_us_foreign_policy": 1.0,
+    "mmlu_virology": 1.0,
+    "mmlu_world_religions": 1.0
+  },
+  "n-shot": {
+    "mmlu_abstract_algebra": 5,
+    "mmlu_anatomy": 5,
+    "mmlu_astronomy": 5,
+    "mmlu_business_ethics": 5,
+    "mmlu_clinical_knowledge": 5,
+    "mmlu_college_biology": 5,
+    "mmlu_college_chemistry": 5,
+    "mmlu_college_computer_science": 5,
+    "mmlu_college_mathematics": 5,
+    "mmlu_college_medicine": 5,
+    "mmlu_college_physics": 5,
+    "mmlu_computer_security": 5,
+    "mmlu_conceptual_physics": 5,
+    "mmlu_econometrics": 5,
+    "mmlu_electrical_engineering": 5,
+    "mmlu_elementary_mathematics": 5,
+    "mmlu_formal_logic": 5,
+    "mmlu_global_facts": 5,
+    "mmlu_high_school_biology": 5,
+    "mmlu_high_school_chemistry": 5,
+    "mmlu_high_school_computer_science": 5,
+    "mmlu_high_school_european_history": 5,
+    "mmlu_high_school_geography": 5,
+    "mmlu_high_school_government_and_politics": 5,
+    "mmlu_high_school_macroeconomics": 5,
+    "mmlu_high_school_mathematics": 5,
+    "mmlu_high_school_microeconomics": 5,
+    "mmlu_high_school_physics": 5,
+    "mmlu_high_school_psychology": 5,
+    "mmlu_high_school_statistics": 5,
+    "mmlu_high_school_us_history": 5,
+    "mmlu_high_school_world_history": 5,
+    "mmlu_human_aging": 5,
+    "mmlu_human_sexuality": 5,
+    "mmlu_humanities": 5,
+    "mmlu_international_law": 5,
+    "mmlu_jurisprudence": 5,
+    "mmlu_logical_fallacies": 5,
+    "mmlu_machine_learning": 5,
+    "mmlu_management": 5,
+    "mmlu_marketing": 5,
+    "mmlu_medical_genetics": 5,
+    "mmlu_miscellaneous": 5,
+    "mmlu_moral_disputes": 5,
+    "mmlu_moral_scenarios": 5,
+    "mmlu_nutrition": 5,
+    "mmlu_other": 5,
+    "mmlu_philosophy": 5,
+    "mmlu_prehistory": 5,
+    "mmlu_professional_accounting": 5,
+    "mmlu_professional_law": 5,
+    "mmlu_professional_medicine": 5,
+    "mmlu_professional_psychology": 5,
+    "mmlu_public_relations": 5,
+    "mmlu_security_studies": 5,
+    "mmlu_social_sciences": 5,
+    "mmlu_sociology": 5,
+    "mmlu_stem": 5,
+    "mmlu_us_foreign_policy": 5,
+    "mmlu_virology": 5,
+    "mmlu_world_religions": 5
+  },
+  "higher_is_better": {
+    "mmlu_abstract_algebra": {
+      "acc": true
+    },
+    "mmlu_anatomy": {
+      "acc": true
+    },
+    "mmlu_astronomy": {
+      "acc": true
+    },
+    "mmlu_business_ethics": {
+      "acc": true
+    },
+    "mmlu_clinical_knowledge": {
+      "acc": true
+    },
+    "mmlu_college_biology": {
+      "acc": true
+    },
+    "mmlu_college_chemistry": {
+      "acc": true
+    },
+    "mmlu_college_computer_science": {
+      "acc": true
+    },
+    "mmlu_college_mathematics": {
+      "acc": true
+    },
+    "mmlu_college_medicine": {
+      "acc": true
+    },
+    "mmlu_college_physics": {
+      "acc": true
+    },
+    "mmlu_computer_security": {
+      "acc": true
+    },
+    "mmlu_conceptual_physics": {
+      "acc": true
+    },
+    "mmlu_econometrics": {
+      "acc": true
+    },
+    "mmlu_electrical_engineering": {
+      "acc": true
+    },
+    "mmlu_elementary_mathematics": {
+      "acc": true
+    },
+    "mmlu_formal_logic": {
+      "acc": true
+    },
+    "mmlu_global_facts": {
+      "acc": true
+    },
+    "mmlu_high_school_biology": {
+      "acc": true
+    },
+    "mmlu_high_school_chemistry": {
+      "acc": true
+    },
+    "mmlu_high_school_computer_science": {
+      "acc": true
+    },
+    "mmlu_high_school_european_history": {
+      "acc": true
+    },
+    "mmlu_high_school_geography": {
+      "acc": true
+    },
+    "mmlu_high_school_government_and_politics": {
+      "acc": true
+    },
+    "mmlu_high_school_macroeconomics": {
+      "acc": true
+    },
+    "mmlu_high_school_mathematics": {
+      "acc": true
+    },
+    "mmlu_high_school_microeconomics": {
+      "acc": true
+    },
+    "mmlu_high_school_physics": {
+      "acc": true
+    },
+    "mmlu_high_school_psychology": {
+      "acc": true
+    },
+    "mmlu_high_school_statistics": {
+      "acc": true
+    },
+    "mmlu_high_school_us_history": {
+      "acc": true
+    },
+    "mmlu_high_school_world_history": {
+      "acc": true
+    },
+    "mmlu_human_aging": {
+      "acc": true
+    },
+    "mmlu_human_sexuality": {
+      "acc": true
+    },
+    "mmlu_humanities": {
+      "acc": true
+    },
+    "mmlu_international_law": {
+      "acc": true
+    },
+    "mmlu_jurisprudence": {
+      "acc": true
+    },
+    "mmlu_logical_fallacies": {
+      "acc": true
+    },
+    "mmlu_machine_learning": {
+      "acc": true
+    },
+    "mmlu_management": {
+      "acc": true
+    },
+    "mmlu_marketing": {
+      "acc": true
+    },
+    "mmlu_medical_genetics": {
+      "acc": true
+    },
+    "mmlu_miscellaneous": {
+      "acc": true
+    },
+    "mmlu_moral_disputes": {
+      "acc": true
+    },
+    "mmlu_moral_scenarios": {
+      "acc": true
+    },
+    "mmlu_nutrition": {
+      "acc": true
+    },
+    "mmlu_other": {
+      "acc": true
+    },
+    "mmlu_philosophy": {
+      "acc": true
+    },
+    "mmlu_prehistory": {
+      "acc": true
+    },
+    "mmlu_professional_accounting": {
+      "acc": true
+    },
+    "mmlu_professional_law": {
+      "acc": true
+    },
+    "mmlu_professional_medicine": {
+      "acc": true
+    },
+    "mmlu_professional_psychology": {
+      "acc": true
+    },
+    "mmlu_public_relations": {
+      "acc": true
+    },
+    "mmlu_security_studies": {
+      "acc": true
+    },
+    "mmlu_social_sciences": {
+      "acc": true
+    },
+    "mmlu_sociology": {
+      "acc": true
+    },
+    "mmlu_stem": {
+      "acc": true
+    },
+    "mmlu_us_foreign_policy": {
+      "acc": true
+    },
+    "mmlu_virology": {
+      "acc": true
+    },
+    "mmlu_world_religions": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "mmlu_abstract_algebra": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_anatomy": {
+      "original": 135,
+      "effective": 15
+    },
+    "mmlu_astronomy": {
+      "original": 152,
+      "effective": 15
+    },
+    "mmlu_college_biology": {
+      "original": 144,
+      "effective": 15
+    },
+    "mmlu_college_chemistry": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_college_computer_science": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_college_mathematics": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_college_physics": {
+      "original": 102,
+      "effective": 15
+    },
+    "mmlu_computer_security": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_conceptual_physics": {
+      "original": 235,
+      "effective": 15
+    },
+    "mmlu_electrical_engineering": {
+      "original": 145,
+      "effective": 15
+    },
+    "mmlu_elementary_mathematics": {
+      "original": 378,
+      "effective": 15
+    },
+    "mmlu_high_school_biology": {
+      "original": 310,
+      "effective": 15
+    },
+    "mmlu_high_school_chemistry": {
+      "original": 203,
+      "effective": 15
+    },
+    "mmlu_high_school_computer_science": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_high_school_mathematics": {
+      "original": 270,
+      "effective": 15
+    },
+    "mmlu_high_school_physics": {
+      "original": 151,
+      "effective": 15
+    },
+    "mmlu_high_school_statistics": {
+      "original": 216,
+      "effective": 15
+    },
+    "mmlu_machine_learning": {
+      "original": 112,
+      "effective": 15
+    },
+    "mmlu_business_ethics": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_clinical_knowledge": {
+      "original": 265,
+      "effective": 15
+    },
+    "mmlu_college_medicine": {
+      "original": 173,
+      "effective": 15
+    },
+    "mmlu_global_facts": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_human_aging": {
+      "original": 223,
+      "effective": 15
+    },
+    "mmlu_management": {
+      "original": 103,
+      "effective": 15
+    },
+    "mmlu_marketing": {
+      "original": 234,
+      "effective": 15
+    },
+    "mmlu_medical_genetics": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_miscellaneous": {
+      "original": 783,
+      "effective": 15
+    },
+    "mmlu_nutrition": {
+      "original": 306,
+      "effective": 15
+    },
+    "mmlu_professional_accounting": {
+      "original": 282,
+      "effective": 15
+    },
+    "mmlu_professional_medicine": {
+      "original": 272,
+      "effective": 15
+    },
+    "mmlu_virology": {
+      "original": 166,
+      "effective": 15
+    },
+    "mmlu_econometrics": {
+      "original": 114,
+      "effective": 15
+    },
+    "mmlu_high_school_geography": {
+      "original": 198,
+      "effective": 15
+    },
+    "mmlu_high_school_government_and_politics": {
+      "original": 193,
+      "effective": 15
+    },
+    "mmlu_high_school_macroeconomics": {
+      "original": 390,
+      "effective": 15
+    },
+    "mmlu_high_school_microeconomics": {
+      "original": 238,
+      "effective": 15
+    },
+    "mmlu_high_school_psychology": {
+      "original": 545,
+      "effective": 15
+    },
+    "mmlu_human_sexuality": {
+      "original": 131,
+      "effective": 15
+    },
+    "mmlu_professional_psychology": {
+      "original": 612,
+      "effective": 15
+    },
+    "mmlu_public_relations": {
+      "original": 110,
+      "effective": 15
+    },
+    "mmlu_security_studies": {
+      "original": 245,
+      "effective": 15
+    },
+    "mmlu_sociology": {
+      "original": 201,
+      "effective": 15
+    },
+    "mmlu_us_foreign_policy": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_formal_logic": {
+      "original": 126,
+      "effective": 15
+    },
+    "mmlu_high_school_european_history": {
+      "original": 165,
+      "effective": 15
+    },
+    "mmlu_high_school_us_history": {
+      "original": 204,
+      "effective": 15
+    },
+    "mmlu_high_school_world_history": {
+      "original": 237,
+      "effective": 15
+    },
+    "mmlu_international_law": {
+      "original": 121,
+      "effective": 15
+    },
+    "mmlu_jurisprudence": {
+      "original": 108,
+      "effective": 15
+    },
+    "mmlu_logical_fallacies": {
+      "original": 163,
+      "effective": 15
+    },
+    "mmlu_moral_disputes": {
+      "original": 346,
+      "effective": 15
+    },
+    "mmlu_moral_scenarios": {
+      "original": 895,
+      "effective": 15
+    },
+    "mmlu_philosophy": {
+      "original": 311,
+      "effective": 15
+    },
+    "mmlu_prehistory": {
+      "original": 324,
+      "effective": 15
+    },
+    "mmlu_professional_law": {
+      "original": 1534,
+      "effective": 15
+    },
+    "mmlu_world_religions": {
+      "original": 171,
+      "effective": 15
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+      "dtype": "bfloat16"
+    },
+    "model_num_parameters": 7615616512,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "a09a35458c702b33eeacc393d103063234e8bc28",
+    "batch_size": "2",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": 15.0,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 0,
+    "torch_seed": 0,
+    "fewshot_seed": 0
+  },
+  "git_hash": "eb4f2be22f7baf6d268c3dd5e46d49d6bd2e74ae",
+  "date": 1784877919.0382555,
+  "pretty_env_info": "PyTorch version: 2.13.0+cu126\nIs debug build: False\nCUDA used to build PyTorch: 12.6\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.11.10 (main, Sep  7 2024, 18:35:41) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.8.0-49-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3090\nNvidia driver version: 565.57.01\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: False\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               48\nOn-line CPU(s) list:                  0-47\nVendor ID:                            AuthenticAMD\nModel name:                           AMD EPYC 7402P 24-Core Processor\nCPU family:                           23\nModel:                                49\nThread(s) per core:                   2\nCore(s) per socket:                   24\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU max MHz:                          2800.0000\nCPU min MHz:                          1500.0000\nBogoMIPS:                             5600.34\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            768 KiB (24 instances)\nL1i cache:                            768 KiB (24 instances)\nL2 cache:                             12 MiB (24 instances)\nL3 cache:                             128 MiB (8 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-47\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.6.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.6.80\n[pip3] nvidia-cuda-nvrtc-cu12==12.6.85\n[pip3] nvidia-cuda-runtime-cu12==12.6.77\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.0.4\n[pip3] nvidia-curand-cu12==10.3.7.77\n[pip3] nvidia-cusolver-cu12==11.7.1.2\n[pip3] nvidia-cusparse-cu12==12.5.4.2\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.29.3\n[pip3] nvidia-nvjitlink-cu12==12.6.85\n[pip3] nvidia-nvtx-cu12==12.6.77\n[pip3] torch==2.13.0+cu126\n[pip3] triton==3.7.1\n[conda] Could not collect",
+  "transformers_version": "5.14.1",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Qwen/Qwen2.5-7B-Instruct",
+  "model_name_sanitized": "Qwen__Qwen2.5-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "228.57909681089222"
+}
\ No newline at end of file
diff --git a/run/capability/m1/gsm8k_cot/__root__steering-resistance__results__full_7b__m1_resist_adapter/results_2026-07-24T07-23-32.451507.json b/run/capability/m1/gsm8k_cot/__root__steering-resistance__results__full_7b__m1_resist_adapter/results_2026-07-24T07-23-32.451507.json
new file mode 100644
index 0000000000000000000000000000000000000000..8f8baff08a10e549541c821318ddadbf7e2a0609
--- /dev/null
+++ b/run/capability/m1/gsm8k_cot/__root__steering-resistance__results__full_7b__m1_resist_adapter/results_2026-07-24T07-23-32.451507.json
@@ -0,0 +1,208 @@
+{
+  "results": {
+    "gsm8k_cot": {
+      "name": "gsm8k_cot",
+      "alias": "gsm8k_cot",
+      "sample_len": 200,
+      "exact_match,strict-match": 0.365,
+      "exact_match_stderr,strict-match": 0.03412767927155776,
+      "exact_match,flexible-extract": 0.795,
+      "exact_match_stderr,flexible-extract": 0.0286176492613602
+    }
+  },
+  "group_subtasks": {},
+  "configs": {
+    "gsm8k_cot": {
+      "task": "gsm8k_cot",
+      "dataset_path": "openai/gsm8k",
+      "dataset_name": "main",
+      "test_split": "test",
+      "doc_to_text": "Q: {{question}}\nA:",
+      "doc_to_target": "{{answer.split('####')[-1].strip() if answer is defined else target}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": [
+          {
+            "question": "There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?",
+            "target": "There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have been 21 - 15 = 6. The answer is 6."
+          },
+          {
+            "question": "If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?",
+            "target": "There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5."
+          },
+          {
+            "question": "Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?",
+            "target": "Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The answer is 39."
+          },
+          {
+            "question": "Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?",
+            "target": "Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8. The answer is 8."
+          },
+          {
+            "question": "Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?",
+            "target": "Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9. The answer is 9."
+          },
+          {
+            "question": "There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?",
+            "target": "There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The answer is 29."
+          },
+          {
+            "question": "Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?",
+            "target": "Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The answer is 33."
+          },
+          {
+            "question": "Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
+            "target": "Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The answer is 8."
+          }
+        ],
+        "doc_to_text": "Q: {{question}}\nA:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer.split('####')[-1].strip() if answer is defined else target}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 8,
+      "metric_list": [
+        {
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "metric": "exact_match",
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "until": [
+          "Q:",
+          "</s>",
+          "<|im_end|>"
+        ]
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "The answer is (\\-?[0-9\\.\\,]+)."
+            },
+            {
+              "function": "take_first"
+            }
+          ],
+          "name": "strict-match"
+        },
+        {
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ],
+          "name": "flexible-extract"
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/gsm8k/gsm8k-cot.yaml"
+      }
+    }
+  },
+  "versions": {
+    "gsm8k_cot": 3.0
+  },
+  "n-shot": {
+    "gsm8k_cot": 8
+  },
+  "higher_is_better": {
+    "gsm8k_cot": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm8k_cot": {
+      "original": 1319,
+      "effective": 200
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+      "dtype": "bfloat16",
+      "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter"
+    },
+    "model_num_parameters": 7653126656,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "a09a35458c702b33eeacc393d103063234e8bc28",
+    "peft_sha": "",
+    "batch_size": "4",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": 200.0,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {
+      "do_sample": false
+    },
+    "random_seed": 0,
+    "numpy_seed": 0,
+    "torch_seed": 0,
+    "fewshot_seed": 0
+  },
+  "git_hash": "eb4f2be22f7baf6d268c3dd5e46d49d6bd2e74ae",
+  "date": 1784877340.1080253,
+  "pretty_env_info": "PyTorch version: 2.13.0+cu126\nIs debug build: False\nCUDA used to build PyTorch: 12.6\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.11.10 (main, Sep  7 2024, 18:35:41) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.8.0-49-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3090\nNvidia driver version: 565.57.01\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: False\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               48\nOn-line CPU(s) list:                  0-47\nVendor ID:                            AuthenticAMD\nModel name:                           AMD EPYC 7402P 24-Core Processor\nCPU family:                           23\nModel:                                49\nThread(s) per core:                   2\nCore(s) per socket:                   24\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU max MHz:                          2800.0000\nCPU min MHz:                          1500.0000\nBogoMIPS:                             5600.34\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            768 KiB (24 instances)\nL1i cache:                            768 KiB (24 instances)\nL2 cache:                             12 MiB (24 instances)\nL3 cache:                             128 MiB (8 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-47\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.6.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.6.80\n[pip3] nvidia-cuda-nvrtc-cu12==12.6.85\n[pip3] nvidia-cuda-runtime-cu12==12.6.77\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.0.4\n[pip3] nvidia-curand-cu12==10.3.7.77\n[pip3] nvidia-cusolver-cu12==11.7.1.2\n[pip3] nvidia-cusparse-cu12==12.5.4.2\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.29.3\n[pip3] nvidia-nvjitlink-cu12==12.6.85\n[pip3] nvidia-nvtx-cu12==12.6.77\n[pip3] torch==2.13.0+cu126\n[pip3] triton==3.7.1\n[conda] Could not collect",
+  "transformers_version": "5.14.1",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+  "model_name_sanitized": "__root__steering-resistance__results__full_7b__m1_resist_adapter",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "479.8107345607132"
+}
\ No newline at end of file
diff --git a/run/capability/m1/mmlu/__root__steering-resistance__results__full_7b__m1_resist_adapter/results_2026-07-24T07-33-33.769325.json b/run/capability/m1/mmlu/__root__steering-resistance__results__full_7b__m1_resist_adapter/results_2026-07-24T07-33-33.769325.json
new file mode 100644
index 0000000000000000000000000000000000000000..f33966f1e719b252fcbf6305540538681c5ee7e6
--- /dev/null
+++ b/run/capability/m1/mmlu/__root__steering-resistance__results__full_7b__m1_resist_adapter/results_2026-07-24T07-33-33.769325.json
@@ -0,0 +1,4368 @@
+{
+  "results": {
+    "mmlu_abstract_algebra": {
+      "name": "mmlu_abstract_algebra",
+      "alias": "abstract_algebra",
+      "sample_len": 15,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_anatomy": {
+      "name": "mmlu_anatomy",
+      "alias": "anatomy",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_astronomy": {
+      "name": "mmlu_astronomy",
+      "alias": "astronomy",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_college_biology": {
+      "name": "mmlu_college_biology",
+      "alias": "college_biology",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_college_chemistry": {
+      "name": "mmlu_college_chemistry",
+      "alias": "college_chemistry",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_college_computer_science": {
+      "name": "mmlu_college_computer_science",
+      "alias": "college_computer_science",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_college_mathematics": {
+      "name": "mmlu_college_mathematics",
+      "alias": "college_mathematics",
+      "sample_len": 15,
+      "acc,none": 0.4,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_college_physics": {
+      "name": "mmlu_college_physics",
+      "alias": "college_physics",
+      "sample_len": 15,
+      "acc,none": 0.4666666666666667,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_computer_security": {
+      "name": "mmlu_computer_security",
+      "alias": "computer_security",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_conceptual_physics": {
+      "name": "mmlu_conceptual_physics",
+      "alias": "conceptual_physics",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_electrical_engineering": {
+      "name": "mmlu_electrical_engineering",
+      "alias": "electrical_engineering",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_elementary_mathematics": {
+      "name": "mmlu_elementary_mathematics",
+      "alias": "elementary_mathematics",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_high_school_biology": {
+      "name": "mmlu_high_school_biology",
+      "alias": "high_school_biology",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589956
+    },
+    "mmlu_high_school_chemistry": {
+      "name": "mmlu_high_school_chemistry",
+      "alias": "high_school_chemistry",
+      "sample_len": 15,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_high_school_computer_science": {
+      "name": "mmlu_high_school_computer_science",
+      "alias": "high_school_computer_science",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_high_school_mathematics": {
+      "name": "mmlu_high_school_mathematics",
+      "alias": "high_school_mathematics",
+      "sample_len": 15,
+      "acc,none": 0.5333333333333333,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_high_school_physics": {
+      "name": "mmlu_high_school_physics",
+      "alias": "high_school_physics",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_high_school_statistics": {
+      "name": "mmlu_high_school_statistics",
+      "alias": "high_school_statistics",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589958
+    },
+    "mmlu_machine_learning": {
+      "name": "mmlu_machine_learning",
+      "alias": "machine_learning",
+      "sample_len": 15,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_business_ethics": {
+      "name": "mmlu_business_ethics",
+      "alias": "business_ethics",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_clinical_knowledge": {
+      "name": "mmlu_clinical_knowledge",
+      "alias": "clinical_knowledge",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_college_medicine": {
+      "name": "mmlu_college_medicine",
+      "alias": "college_medicine",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_global_facts": {
+      "name": "mmlu_global_facts",
+      "alias": "global_facts",
+      "sample_len": 15,
+      "acc,none": 0.4666666666666667,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_human_aging": {
+      "name": "mmlu_human_aging",
+      "alias": "human_aging",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705575
+    },
+    "mmlu_management": {
+      "name": "mmlu_management",
+      "alias": "management",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589958
+    },
+    "mmlu_marketing": {
+      "name": "mmlu_marketing",
+      "alias": "marketing",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_medical_genetics": {
+      "name": "mmlu_medical_genetics",
+      "alias": "medical_genetics",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589956
+    },
+    "mmlu_miscellaneous": {
+      "name": "mmlu_miscellaneous",
+      "alias": "miscellaneous",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_nutrition": {
+      "name": "mmlu_nutrition",
+      "alias": "nutrition",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_professional_accounting": {
+      "name": "mmlu_professional_accounting",
+      "alias": "professional_accounting",
+      "sample_len": 15,
+      "acc,none": 0.4666666666666667,
+      "acc_stderr,none": 0.1333333333333333
+    },
+    "mmlu_professional_medicine": {
+      "name": "mmlu_professional_medicine",
+      "alias": "professional_medicine",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_virology": {
+      "name": "mmlu_virology",
+      "alias": "virology",
+      "sample_len": 15,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_econometrics": {
+      "name": "mmlu_econometrics",
+      "alias": "econometrics",
+      "sample_len": 15,
+      "acc,none": 0.7333333333333333,
+      "acc_stderr,none": 0.11818736805705576
+    },
+    "mmlu_high_school_geography": {
+      "name": "mmlu_high_school_geography",
+      "alias": "high_school_geography",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_high_school_government_and_politics": {
+      "name": "mmlu_high_school_government_and_politics",
+      "alias": "high_school_government_and_politics",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_high_school_macroeconomics": {
+      "name": "mmlu_high_school_macroeconomics",
+      "alias": "high_school_macroeconomics",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_high_school_microeconomics": {
+      "name": "mmlu_high_school_microeconomics",
+      "alias": "high_school_microeconomics",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_high_school_psychology": {
+      "name": "mmlu_high_school_psychology",
+      "alias": "high_school_psychology",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_human_sexuality": {
+      "name": "mmlu_human_sexuality",
+      "alias": "human_sexuality",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_professional_psychology": {
+      "name": "mmlu_professional_psychology",
+      "alias": "professional_psychology",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_public_relations": {
+      "name": "mmlu_public_relations",
+      "alias": "public_relations",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_security_studies": {
+      "name": "mmlu_security_studies",
+      "alias": "security_studies",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_sociology": {
+      "name": "mmlu_sociology",
+      "alias": "sociology",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_us_foreign_policy": {
+      "name": "mmlu_us_foreign_policy",
+      "alias": "us_foreign_policy",
+      "sample_len": 15,
+      "acc,none": 0.8666666666666667,
+      "acc_stderr,none": 0.09085135251589957
+    },
+    "mmlu_formal_logic": {
+      "name": "mmlu_formal_logic",
+      "alias": "formal_logic",
+      "sample_len": 15,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_high_school_european_history": {
+      "name": "mmlu_high_school_european_history",
+      "alias": "high_school_european_history",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496976
+    },
+    "mmlu_high_school_us_history": {
+      "name": "mmlu_high_school_us_history",
+      "alias": "high_school_us_history",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_high_school_world_history": {
+      "name": "mmlu_high_school_world_history",
+      "alias": "high_school_world_history",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_international_law": {
+      "name": "mmlu_international_law",
+      "alias": "international_law",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_jurisprudence": {
+      "name": "mmlu_jurisprudence",
+      "alias": "jurisprudence",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_logical_fallacies": {
+      "name": "mmlu_logical_fallacies",
+      "alias": "logical_fallacies",
+      "sample_len": 15,
+      "acc,none": 0.9333333333333333,
+      "acc_stderr,none": 0.06666666666666667
+    },
+    "mmlu_moral_disputes": {
+      "name": "mmlu_moral_disputes",
+      "alias": "moral_disputes",
+      "sample_len": 15,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_moral_scenarios": {
+      "name": "mmlu_moral_scenarios",
+      "alias": "moral_scenarios",
+      "sample_len": 15,
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.12598815766974242
+    },
+    "mmlu_philosophy": {
+      "name": "mmlu_philosophy",
+      "alias": "philosophy",
+      "sample_len": 15,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "mmlu_prehistory": {
+      "name": "mmlu_prehistory",
+      "alias": "prehistory",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_professional_law": {
+      "name": "mmlu_professional_law",
+      "alias": "professional_law",
+      "sample_len": 15,
+      "acc,none": 0.6,
+      "acc_stderr,none": 0.13093073414159542
+    },
+    "mmlu_world_religions": {
+      "name": "mmlu_world_religions",
+      "alias": "world_religions",
+      "sample_len": 15,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.10690449676496977
+    },
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 285,
+      "acc,none": 0.712280701754386,
+      "acc_stderr,none": 0.02592016736611107,
+      "sample_count": {
+        "acc,none": 285
+      }
+    },
+    "mmlu_other": {
+      "alias": "other",
+      "name": "mmlu_other",
+      "sample_len": 195,
+      "acc,none": 0.7487179487179487,
+      "acc_stderr,none": 0.03033887068256213,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu_social_sciences": {
+      "alias": "social sciences",
+      "name": "mmlu_social_sciences",
+      "sample_len": 180,
+      "acc,none": 0.8277777777777777,
+      "acc_stderr,none": 0.027936056994917528,
+      "sample_count": {
+        "acc,none": 180
+      }
+    },
+    "mmlu_humanities": {
+      "alias": "humanities",
+      "name": "mmlu_humanities",
+      "sample_len": 195,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.027616229780176946,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu": {
+      "alias": "mmlu",
+      "name": "mmlu",
+      "sample_len": 855,
+      "acc,none": 0.7649122807017544,
+      "acc_stderr,none": 0.014028124142725057,
+      "sample_count": {
+        "acc,none": 855
+      }
+    }
+  },
+  "groups": {
+    "mmlu_stem": {
+      "alias": "stem",
+      "name": "mmlu_stem",
+      "sample_len": 285,
+      "acc,none": 0.712280701754386,
+      "acc_stderr,none": 0.02592016736611107,
+      "sample_count": {
+        "acc,none": 285
+      }
+    },
+    "mmlu_other": {
+      "alias": "other",
+      "name": "mmlu_other",
+      "sample_len": 195,
+      "acc,none": 0.7487179487179487,
+      "acc_stderr,none": 0.03033887068256213,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu_social_sciences": {
+      "alias": "social sciences",
+      "name": "mmlu_social_sciences",
+      "sample_len": 180,
+      "acc,none": 0.8277777777777777,
+      "acc_stderr,none": 0.027936056994917528,
+      "sample_count": {
+        "acc,none": 180
+      }
+    },
+    "mmlu_humanities": {
+      "alias": "humanities",
+      "name": "mmlu_humanities",
+      "sample_len": 195,
+      "acc,none": 0.8,
+      "acc_stderr,none": 0.027616229780176946,
+      "sample_count": {
+        "acc,none": 195
+      }
+    },
+    "mmlu": {
+      "alias": "mmlu",
+      "name": "mmlu",
+      "sample_len": 855,
+      "acc,none": 0.7649122807017544,
+      "acc_stderr,none": 0.014028124142725057,
+      "sample_count": {
+        "acc,none": 855
+      }
+    }
+  },
+  "group_subtasks": {
+    "mmlu": [
+      "mmlu_stem",
+      "mmlu_other",
+      "mmlu_social_sciences",
+      "mmlu_humanities"
+    ],
+    "mmlu_stem": [
+      "mmlu_abstract_algebra",
+      "mmlu_anatomy",
+      "mmlu_astronomy",
+      "mmlu_college_biology",
+      "mmlu_college_chemistry",
+      "mmlu_college_computer_science",
+      "mmlu_college_mathematics",
+      "mmlu_college_physics",
+      "mmlu_computer_security",
+      "mmlu_conceptual_physics",
+      "mmlu_electrical_engineering",
+      "mmlu_elementary_mathematics",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_chemistry",
+      "mmlu_high_school_computer_science",
+      "mmlu_high_school_mathematics",
+      "mmlu_high_school_physics",
+      "mmlu_high_school_statistics",
+      "mmlu_machine_learning"
+    ],
+    "mmlu_other": [
+      "mmlu_business_ethics",
+      "mmlu_clinical_knowledge",
+      "mmlu_college_medicine",
+      "mmlu_global_facts",
+      "mmlu_human_aging",
+      "mmlu_management",
+      "mmlu_marketing",
+      "mmlu_medical_genetics",
+      "mmlu_miscellaneous",
+      "mmlu_nutrition",
+      "mmlu_professional_accounting",
+      "mmlu_professional_medicine",
+      "mmlu_virology"
+    ],
+    "mmlu_social_sciences": [
+      "mmlu_econometrics",
+      "mmlu_high_school_geography",
+      "mmlu_high_school_government_and_politics",
+      "mmlu_high_school_macroeconomics",
+      "mmlu_high_school_microeconomics",
+      "mmlu_high_school_psychology",
+      "mmlu_human_sexuality",
+      "mmlu_professional_psychology",
+      "mmlu_public_relations",
+      "mmlu_security_studies",
+      "mmlu_sociology",
+      "mmlu_us_foreign_policy"
+    ],
+    "mmlu_humanities": [
+      "mmlu_formal_logic",
+      "mmlu_high_school_european_history",
+      "mmlu_high_school_us_history",
+      "mmlu_high_school_world_history",
+      "mmlu_international_law",
+      "mmlu_jurisprudence",
+      "mmlu_logical_fallacies",
+      "mmlu_moral_disputes",
+      "mmlu_moral_scenarios",
+      "mmlu_philosophy",
+      "mmlu_prehistory",
+      "mmlu_professional_law",
+      "mmlu_world_religions"
+    ]
+  },
+  "configs": {
+    "mmlu_abstract_algebra": {
+      "task": "mmlu_abstract_algebra",
+      "task_alias": "abstract_algebra",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "abstract_algebra",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml"
+      }
+    },
+    "mmlu_anatomy": {
+      "task": "mmlu_anatomy",
+      "task_alias": "anatomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "anatomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml"
+      }
+    },
+    "mmlu_astronomy": {
+      "task": "mmlu_astronomy",
+      "task_alias": "astronomy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "astronomy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml"
+      }
+    },
+    "mmlu_business_ethics": {
+      "task": "mmlu_business_ethics",
+      "task_alias": "business_ethics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "business_ethics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml"
+      }
+    },
+    "mmlu_clinical_knowledge": {
+      "task": "mmlu_clinical_knowledge",
+      "task_alias": "clinical_knowledge",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "clinical_knowledge",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml"
+      }
+    },
+    "mmlu_college_biology": {
+      "task": "mmlu_college_biology",
+      "task_alias": "college_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml"
+      }
+    },
+    "mmlu_college_chemistry": {
+      "task": "mmlu_college_chemistry",
+      "task_alias": "college_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml"
+      }
+    },
+    "mmlu_college_computer_science": {
+      "task": "mmlu_college_computer_science",
+      "task_alias": "college_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml"
+      }
+    },
+    "mmlu_college_mathematics": {
+      "task": "mmlu_college_mathematics",
+      "task_alias": "college_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml"
+      }
+    },
+    "mmlu_college_medicine": {
+      "task": "mmlu_college_medicine",
+      "task_alias": "college_medicine",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_medicine",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml"
+      }
+    },
+    "mmlu_college_physics": {
+      "task": "mmlu_college_physics",
+      "task_alias": "college_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "college_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about college physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml"
+      }
+    },
+    "mmlu_computer_security": {
+      "task": "mmlu_computer_security",
+      "task_alias": "computer_security",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "computer_security",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about computer security.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml"
+      }
+    },
+    "mmlu_conceptual_physics": {
+      "task": "mmlu_conceptual_physics",
+      "task_alias": "conceptual_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "conceptual_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml"
+      }
+    },
+    "mmlu_econometrics": {
+      "task": "mmlu_econometrics",
+      "task_alias": "econometrics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "econometrics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml"
+      }
+    },
+    "mmlu_electrical_engineering": {
+      "task": "mmlu_electrical_engineering",
+      "task_alias": "electrical_engineering",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "electrical_engineering",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml"
+      }
+    },
+    "mmlu_elementary_mathematics": {
+      "task": "mmlu_elementary_mathematics",
+      "task_alias": "elementary_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "elementary_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml"
+      }
+    },
+    "mmlu_formal_logic": {
+      "task": "mmlu_formal_logic",
+      "task_alias": "formal_logic",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "formal_logic",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml"
+      }
+    },
+    "mmlu_global_facts": {
+      "task": "mmlu_global_facts",
+      "task_alias": "global_facts",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "global_facts",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about global facts.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml"
+      }
+    },
+    "mmlu_high_school_biology": {
+      "task": "mmlu_high_school_biology",
+      "task_alias": "high_school_biology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_biology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml"
+      }
+    },
+    "mmlu_high_school_chemistry": {
+      "task": "mmlu_high_school_chemistry",
+      "task_alias": "high_school_chemistry",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_chemistry",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml"
+      }
+    },
+    "mmlu_high_school_computer_science": {
+      "task": "mmlu_high_school_computer_science",
+      "task_alias": "high_school_computer_science",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_computer_science",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml"
+      }
+    },
+    "mmlu_high_school_european_history": {
+      "task": "mmlu_high_school_european_history",
+      "task_alias": "high_school_european_history",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_european_history",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml"
+      }
+    },
+    "mmlu_high_school_geography": {
+      "task": "mmlu_high_school_geography",
+      "task_alias": "high_school_geography",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_geography",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml"
+      }
+    },
+    "mmlu_high_school_government_and_politics": {
+      "task": "mmlu_high_school_government_and_politics",
+      "task_alias": "high_school_government_and_politics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_government_and_politics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml"
+      }
+    },
+    "mmlu_high_school_macroeconomics": {
+      "task": "mmlu_high_school_macroeconomics",
+      "task_alias": "high_school_macroeconomics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_macroeconomics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml"
+      }
+    },
+    "mmlu_high_school_mathematics": {
+      "task": "mmlu_high_school_mathematics",
+      "task_alias": "high_school_mathematics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_mathematics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml"
+      }
+    },
+    "mmlu_high_school_microeconomics": {
+      "task": "mmlu_high_school_microeconomics",
+      "task_alias": "high_school_microeconomics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_microeconomics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml"
+      }
+    },
+    "mmlu_high_school_physics": {
+      "task": "mmlu_high_school_physics",
+      "task_alias": "high_school_physics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_physics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml"
+      }
+    },
+    "mmlu_high_school_psychology": {
+      "task": "mmlu_high_school_psychology",
+      "task_alias": "high_school_psychology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_psychology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml"
+      }
+    },
+    "mmlu_high_school_statistics": {
+      "task": "mmlu_high_school_statistics",
+      "task_alias": "high_school_statistics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_statistics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml"
+      }
+    },
+    "mmlu_high_school_us_history": {
+      "task": "mmlu_high_school_us_history",
+      "task_alias": "high_school_us_history",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_us_history",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml"
+      }
+    },
+    "mmlu_high_school_world_history": {
+      "task": "mmlu_high_school_world_history",
+      "task_alias": "high_school_world_history",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "high_school_world_history",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml"
+      }
+    },
+    "mmlu_human_aging": {
+      "task": "mmlu_human_aging",
+      "task_alias": "human_aging",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "human_aging",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about human aging.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml"
+      }
+    },
+    "mmlu_human_sexuality": {
+      "task": "mmlu_human_sexuality",
+      "task_alias": "human_sexuality",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "human_sexuality",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml"
+      }
+    },
+    "mmlu_international_law": {
+      "task": "mmlu_international_law",
+      "task_alias": "international_law",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "international_law",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about international law.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml"
+      }
+    },
+    "mmlu_jurisprudence": {
+      "task": "mmlu_jurisprudence",
+      "task_alias": "jurisprudence",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "jurisprudence",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml"
+      }
+    },
+    "mmlu_logical_fallacies": {
+      "task": "mmlu_logical_fallacies",
+      "task_alias": "logical_fallacies",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "logical_fallacies",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml"
+      }
+    },
+    "mmlu_machine_learning": {
+      "task": "mmlu_machine_learning",
+      "task_alias": "machine_learning",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "machine_learning",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml"
+      }
+    },
+    "mmlu_management": {
+      "task": "mmlu_management",
+      "task_alias": "management",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "management",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about management.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_management.yaml"
+      }
+    },
+    "mmlu_marketing": {
+      "task": "mmlu_marketing",
+      "task_alias": "marketing",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "marketing",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about marketing.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml"
+      }
+    },
+    "mmlu_medical_genetics": {
+      "task": "mmlu_medical_genetics",
+      "task_alias": "medical_genetics",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "medical_genetics",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml"
+      }
+    },
+    "mmlu_miscellaneous": {
+      "task": "mmlu_miscellaneous",
+      "task_alias": "miscellaneous",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "miscellaneous",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml"
+      }
+    },
+    "mmlu_moral_disputes": {
+      "task": "mmlu_moral_disputes",
+      "task_alias": "moral_disputes",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "moral_disputes",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml"
+      }
+    },
+    "mmlu_moral_scenarios": {
+      "task": "mmlu_moral_scenarios",
+      "task_alias": "moral_scenarios",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "moral_scenarios",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml"
+      }
+    },
+    "mmlu_nutrition": {
+      "task": "mmlu_nutrition",
+      "task_alias": "nutrition",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "nutrition",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml"
+      }
+    },
+    "mmlu_philosophy": {
+      "task": "mmlu_philosophy",
+      "task_alias": "philosophy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "philosophy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml"
+      }
+    },
+    "mmlu_prehistory": {
+      "task": "mmlu_prehistory",
+      "task_alias": "prehistory",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "prehistory",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml"
+      }
+    },
+    "mmlu_professional_accounting": {
+      "task": "mmlu_professional_accounting",
+      "task_alias": "professional_accounting",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_accounting",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml"
+      }
+    },
+    "mmlu_professional_law": {
+      "task": "mmlu_professional_law",
+      "task_alias": "professional_law",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_law",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional law.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml"
+      }
+    },
+    "mmlu_professional_medicine": {
+      "task": "mmlu_professional_medicine",
+      "task_alias": "professional_medicine",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_medicine",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml"
+      }
+    },
+    "mmlu_professional_psychology": {
+      "task": "mmlu_professional_psychology",
+      "task_alias": "professional_psychology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "professional_psychology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml"
+      }
+    },
+    "mmlu_public_relations": {
+      "task": "mmlu_public_relations",
+      "task_alias": "public_relations",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "public_relations",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about public relations.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml"
+      }
+    },
+    "mmlu_security_studies": {
+      "task": "mmlu_security_studies",
+      "task_alias": "security_studies",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "security_studies",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about security studies.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml"
+      }
+    },
+    "mmlu_sociology": {
+      "task": "mmlu_sociology",
+      "task_alias": "sociology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "sociology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about sociology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml"
+      }
+    },
+    "mmlu_us_foreign_policy": {
+      "task": "mmlu_us_foreign_policy",
+      "task_alias": "us_foreign_policy",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "us_foreign_policy",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml"
+      }
+    },
+    "mmlu_virology": {
+      "task": "mmlu_virology",
+      "task_alias": "virology",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "virology",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about virology.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_virology.yaml"
+      }
+    },
+    "mmlu_world_religions": {
+      "task": "mmlu_world_religions",
+      "task_alias": "world_religions",
+      "dataset_path": "cais/mmlu",
+      "dataset_name": "world_religions",
+      "test_split": "test",
+      "fewshot_split": "dev",
+      "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "doc_to_choice": [
+        "A",
+        "B",
+        "C",
+        "D"
+      ],
+      "description": "The following are multiple choice questions (with answers) about world religions.\n\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": "dev",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+        "doc_to_choice": [
+          "A",
+          "B",
+          "C",
+          "D"
+        ],
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0,
+        "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+        "dtype": "bfloat16",
+        "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+        "config_source": "/usr/local/lib/python3.11/dist-packages/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml"
+      }
+    }
+  },
+  "versions": {
+    "mmlu": "2",
+    "mmlu_abstract_algebra": 1.0,
+    "mmlu_anatomy": 1.0,
+    "mmlu_astronomy": 1.0,
+    "mmlu_business_ethics": 1.0,
+    "mmlu_clinical_knowledge": 1.0,
+    "mmlu_college_biology": 1.0,
+    "mmlu_college_chemistry": 1.0,
+    "mmlu_college_computer_science": 1.0,
+    "mmlu_college_mathematics": 1.0,
+    "mmlu_college_medicine": 1.0,
+    "mmlu_college_physics": 1.0,
+    "mmlu_computer_security": 1.0,
+    "mmlu_conceptual_physics": 1.0,
+    "mmlu_econometrics": 1.0,
+    "mmlu_electrical_engineering": 1.0,
+    "mmlu_elementary_mathematics": 1.0,
+    "mmlu_formal_logic": 1.0,
+    "mmlu_global_facts": 1.0,
+    "mmlu_high_school_biology": 1.0,
+    "mmlu_high_school_chemistry": 1.0,
+    "mmlu_high_school_computer_science": 1.0,
+    "mmlu_high_school_european_history": 1.0,
+    "mmlu_high_school_geography": 1.0,
+    "mmlu_high_school_government_and_politics": 1.0,
+    "mmlu_high_school_macroeconomics": 1.0,
+    "mmlu_high_school_mathematics": 1.0,
+    "mmlu_high_school_microeconomics": 1.0,
+    "mmlu_high_school_physics": 1.0,
+    "mmlu_high_school_psychology": 1.0,
+    "mmlu_high_school_statistics": 1.0,
+    "mmlu_high_school_us_history": 1.0,
+    "mmlu_high_school_world_history": 1.0,
+    "mmlu_human_aging": 1.0,
+    "mmlu_human_sexuality": 1.0,
+    "mmlu_humanities": "2",
+    "mmlu_international_law": 1.0,
+    "mmlu_jurisprudence": 1.0,
+    "mmlu_logical_fallacies": 1.0,
+    "mmlu_machine_learning": 1.0,
+    "mmlu_management": 1.0,
+    "mmlu_marketing": 1.0,
+    "mmlu_medical_genetics": 1.0,
+    "mmlu_miscellaneous": 1.0,
+    "mmlu_moral_disputes": 1.0,
+    "mmlu_moral_scenarios": 1.0,
+    "mmlu_nutrition": 1.0,
+    "mmlu_other": "2",
+    "mmlu_philosophy": 1.0,
+    "mmlu_prehistory": 1.0,
+    "mmlu_professional_accounting": 1.0,
+    "mmlu_professional_law": 1.0,
+    "mmlu_professional_medicine": 1.0,
+    "mmlu_professional_psychology": 1.0,
+    "mmlu_public_relations": 1.0,
+    "mmlu_security_studies": 1.0,
+    "mmlu_social_sciences": "2",
+    "mmlu_sociology": 1.0,
+    "mmlu_stem": "2",
+    "mmlu_us_foreign_policy": 1.0,
+    "mmlu_virology": 1.0,
+    "mmlu_world_religions": 1.0
+  },
+  "n-shot": {
+    "mmlu_abstract_algebra": 5,
+    "mmlu_anatomy": 5,
+    "mmlu_astronomy": 5,
+    "mmlu_business_ethics": 5,
+    "mmlu_clinical_knowledge": 5,
+    "mmlu_college_biology": 5,
+    "mmlu_college_chemistry": 5,
+    "mmlu_college_computer_science": 5,
+    "mmlu_college_mathematics": 5,
+    "mmlu_college_medicine": 5,
+    "mmlu_college_physics": 5,
+    "mmlu_computer_security": 5,
+    "mmlu_conceptual_physics": 5,
+    "mmlu_econometrics": 5,
+    "mmlu_electrical_engineering": 5,
+    "mmlu_elementary_mathematics": 5,
+    "mmlu_formal_logic": 5,
+    "mmlu_global_facts": 5,
+    "mmlu_high_school_biology": 5,
+    "mmlu_high_school_chemistry": 5,
+    "mmlu_high_school_computer_science": 5,
+    "mmlu_high_school_european_history": 5,
+    "mmlu_high_school_geography": 5,
+    "mmlu_high_school_government_and_politics": 5,
+    "mmlu_high_school_macroeconomics": 5,
+    "mmlu_high_school_mathematics": 5,
+    "mmlu_high_school_microeconomics": 5,
+    "mmlu_high_school_physics": 5,
+    "mmlu_high_school_psychology": 5,
+    "mmlu_high_school_statistics": 5,
+    "mmlu_high_school_us_history": 5,
+    "mmlu_high_school_world_history": 5,
+    "mmlu_human_aging": 5,
+    "mmlu_human_sexuality": 5,
+    "mmlu_humanities": 5,
+    "mmlu_international_law": 5,
+    "mmlu_jurisprudence": 5,
+    "mmlu_logical_fallacies": 5,
+    "mmlu_machine_learning": 5,
+    "mmlu_management": 5,
+    "mmlu_marketing": 5,
+    "mmlu_medical_genetics": 5,
+    "mmlu_miscellaneous": 5,
+    "mmlu_moral_disputes": 5,
+    "mmlu_moral_scenarios": 5,
+    "mmlu_nutrition": 5,
+    "mmlu_other": 5,
+    "mmlu_philosophy": 5,
+    "mmlu_prehistory": 5,
+    "mmlu_professional_accounting": 5,
+    "mmlu_professional_law": 5,
+    "mmlu_professional_medicine": 5,
+    "mmlu_professional_psychology": 5,
+    "mmlu_public_relations": 5,
+    "mmlu_security_studies": 5,
+    "mmlu_social_sciences": 5,
+    "mmlu_sociology": 5,
+    "mmlu_stem": 5,
+    "mmlu_us_foreign_policy": 5,
+    "mmlu_virology": 5,
+    "mmlu_world_religions": 5
+  },
+  "higher_is_better": {
+    "mmlu_abstract_algebra": {
+      "acc": true
+    },
+    "mmlu_anatomy": {
+      "acc": true
+    },
+    "mmlu_astronomy": {
+      "acc": true
+    },
+    "mmlu_business_ethics": {
+      "acc": true
+    },
+    "mmlu_clinical_knowledge": {
+      "acc": true
+    },
+    "mmlu_college_biology": {
+      "acc": true
+    },
+    "mmlu_college_chemistry": {
+      "acc": true
+    },
+    "mmlu_college_computer_science": {
+      "acc": true
+    },
+    "mmlu_college_mathematics": {
+      "acc": true
+    },
+    "mmlu_college_medicine": {
+      "acc": true
+    },
+    "mmlu_college_physics": {
+      "acc": true
+    },
+    "mmlu_computer_security": {
+      "acc": true
+    },
+    "mmlu_conceptual_physics": {
+      "acc": true
+    },
+    "mmlu_econometrics": {
+      "acc": true
+    },
+    "mmlu_electrical_engineering": {
+      "acc": true
+    },
+    "mmlu_elementary_mathematics": {
+      "acc": true
+    },
+    "mmlu_formal_logic": {
+      "acc": true
+    },
+    "mmlu_global_facts": {
+      "acc": true
+    },
+    "mmlu_high_school_biology": {
+      "acc": true
+    },
+    "mmlu_high_school_chemistry": {
+      "acc": true
+    },
+    "mmlu_high_school_computer_science": {
+      "acc": true
+    },
+    "mmlu_high_school_european_history": {
+      "acc": true
+    },
+    "mmlu_high_school_geography": {
+      "acc": true
+    },
+    "mmlu_high_school_government_and_politics": {
+      "acc": true
+    },
+    "mmlu_high_school_macroeconomics": {
+      "acc": true
+    },
+    "mmlu_high_school_mathematics": {
+      "acc": true
+    },
+    "mmlu_high_school_microeconomics": {
+      "acc": true
+    },
+    "mmlu_high_school_physics": {
+      "acc": true
+    },
+    "mmlu_high_school_psychology": {
+      "acc": true
+    },
+    "mmlu_high_school_statistics": {
+      "acc": true
+    },
+    "mmlu_high_school_us_history": {
+      "acc": true
+    },
+    "mmlu_high_school_world_history": {
+      "acc": true
+    },
+    "mmlu_human_aging": {
+      "acc": true
+    },
+    "mmlu_human_sexuality": {
+      "acc": true
+    },
+    "mmlu_humanities": {
+      "acc": true
+    },
+    "mmlu_international_law": {
+      "acc": true
+    },
+    "mmlu_jurisprudence": {
+      "acc": true
+    },
+    "mmlu_logical_fallacies": {
+      "acc": true
+    },
+    "mmlu_machine_learning": {
+      "acc": true
+    },
+    "mmlu_management": {
+      "acc": true
+    },
+    "mmlu_marketing": {
+      "acc": true
+    },
+    "mmlu_medical_genetics": {
+      "acc": true
+    },
+    "mmlu_miscellaneous": {
+      "acc": true
+    },
+    "mmlu_moral_disputes": {
+      "acc": true
+    },
+    "mmlu_moral_scenarios": {
+      "acc": true
+    },
+    "mmlu_nutrition": {
+      "acc": true
+    },
+    "mmlu_other": {
+      "acc": true
+    },
+    "mmlu_philosophy": {
+      "acc": true
+    },
+    "mmlu_prehistory": {
+      "acc": true
+    },
+    "mmlu_professional_accounting": {
+      "acc": true
+    },
+    "mmlu_professional_law": {
+      "acc": true
+    },
+    "mmlu_professional_medicine": {
+      "acc": true
+    },
+    "mmlu_professional_psychology": {
+      "acc": true
+    },
+    "mmlu_public_relations": {
+      "acc": true
+    },
+    "mmlu_security_studies": {
+      "acc": true
+    },
+    "mmlu_social_sciences": {
+      "acc": true
+    },
+    "mmlu_sociology": {
+      "acc": true
+    },
+    "mmlu_stem": {
+      "acc": true
+    },
+    "mmlu_us_foreign_policy": {
+      "acc": true
+    },
+    "mmlu_virology": {
+      "acc": true
+    },
+    "mmlu_world_religions": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "mmlu_abstract_algebra": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_anatomy": {
+      "original": 135,
+      "effective": 15
+    },
+    "mmlu_astronomy": {
+      "original": 152,
+      "effective": 15
+    },
+    "mmlu_college_biology": {
+      "original": 144,
+      "effective": 15
+    },
+    "mmlu_college_chemistry": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_college_computer_science": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_college_mathematics": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_college_physics": {
+      "original": 102,
+      "effective": 15
+    },
+    "mmlu_computer_security": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_conceptual_physics": {
+      "original": 235,
+      "effective": 15
+    },
+    "mmlu_electrical_engineering": {
+      "original": 145,
+      "effective": 15
+    },
+    "mmlu_elementary_mathematics": {
+      "original": 378,
+      "effective": 15
+    },
+    "mmlu_high_school_biology": {
+      "original": 310,
+      "effective": 15
+    },
+    "mmlu_high_school_chemistry": {
+      "original": 203,
+      "effective": 15
+    },
+    "mmlu_high_school_computer_science": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_high_school_mathematics": {
+      "original": 270,
+      "effective": 15
+    },
+    "mmlu_high_school_physics": {
+      "original": 151,
+      "effective": 15
+    },
+    "mmlu_high_school_statistics": {
+      "original": 216,
+      "effective": 15
+    },
+    "mmlu_machine_learning": {
+      "original": 112,
+      "effective": 15
+    },
+    "mmlu_business_ethics": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_clinical_knowledge": {
+      "original": 265,
+      "effective": 15
+    },
+    "mmlu_college_medicine": {
+      "original": 173,
+      "effective": 15
+    },
+    "mmlu_global_facts": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_human_aging": {
+      "original": 223,
+      "effective": 15
+    },
+    "mmlu_management": {
+      "original": 103,
+      "effective": 15
+    },
+    "mmlu_marketing": {
+      "original": 234,
+      "effective": 15
+    },
+    "mmlu_medical_genetics": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_miscellaneous": {
+      "original": 783,
+      "effective": 15
+    },
+    "mmlu_nutrition": {
+      "original": 306,
+      "effective": 15
+    },
+    "mmlu_professional_accounting": {
+      "original": 282,
+      "effective": 15
+    },
+    "mmlu_professional_medicine": {
+      "original": 272,
+      "effective": 15
+    },
+    "mmlu_virology": {
+      "original": 166,
+      "effective": 15
+    },
+    "mmlu_econometrics": {
+      "original": 114,
+      "effective": 15
+    },
+    "mmlu_high_school_geography": {
+      "original": 198,
+      "effective": 15
+    },
+    "mmlu_high_school_government_and_politics": {
+      "original": 193,
+      "effective": 15
+    },
+    "mmlu_high_school_macroeconomics": {
+      "original": 390,
+      "effective": 15
+    },
+    "mmlu_high_school_microeconomics": {
+      "original": 238,
+      "effective": 15
+    },
+    "mmlu_high_school_psychology": {
+      "original": 545,
+      "effective": 15
+    },
+    "mmlu_human_sexuality": {
+      "original": 131,
+      "effective": 15
+    },
+    "mmlu_professional_psychology": {
+      "original": 612,
+      "effective": 15
+    },
+    "mmlu_public_relations": {
+      "original": 110,
+      "effective": 15
+    },
+    "mmlu_security_studies": {
+      "original": 245,
+      "effective": 15
+    },
+    "mmlu_sociology": {
+      "original": 201,
+      "effective": 15
+    },
+    "mmlu_us_foreign_policy": {
+      "original": 100,
+      "effective": 15
+    },
+    "mmlu_formal_logic": {
+      "original": 126,
+      "effective": 15
+    },
+    "mmlu_high_school_european_history": {
+      "original": 165,
+      "effective": 15
+    },
+    "mmlu_high_school_us_history": {
+      "original": 204,
+      "effective": 15
+    },
+    "mmlu_high_school_world_history": {
+      "original": 237,
+      "effective": 15
+    },
+    "mmlu_international_law": {
+      "original": 121,
+      "effective": 15
+    },
+    "mmlu_jurisprudence": {
+      "original": 108,
+      "effective": 15
+    },
+    "mmlu_logical_fallacies": {
+      "original": 163,
+      "effective": 15
+    },
+    "mmlu_moral_disputes": {
+      "original": 346,
+      "effective": 15
+    },
+    "mmlu_moral_scenarios": {
+      "original": 895,
+      "effective": 15
+    },
+    "mmlu_philosophy": {
+      "original": 311,
+      "effective": 15
+    },
+    "mmlu_prehistory": {
+      "original": 324,
+      "effective": 15
+    },
+    "mmlu_professional_law": {
+      "original": 1534,
+      "effective": 15
+    },
+    "mmlu_world_religions": {
+      "original": 171,
+      "effective": 15
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": {
+      "pretrained": "Qwen/Qwen2.5-7B-Instruct",
+      "dtype": "bfloat16",
+      "peft": "/root/steering-resistance/results/full_7b/m1_resist_adapter"
+    },
+    "model_num_parameters": 7653126656,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "a09a35458c702b33eeacc393d103063234e8bc28",
+    "peft_sha": "",
+    "batch_size": "2",
+    "batch_sizes": [],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": 15.0,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": {},
+    "random_seed": 0,
+    "numpy_seed": 0,
+    "torch_seed": 0,
+    "fewshot_seed": 0
+  },
+  "git_hash": "eb4f2be22f7baf6d268c3dd5e46d49d6bd2e74ae",
+  "date": 1784878150.8815203,
+  "pretty_env_info": "PyTorch version: 2.13.0+cu126\nIs debug build: False\nCUDA used to build PyTorch: 12.6\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.11.10 (main, Sep  7 2024, 18:35:41) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.8.0-49-generic-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3090\nNvidia driver version: 565.57.01\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: False\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               48\nOn-line CPU(s) list:                  0-47\nVendor ID:                            AuthenticAMD\nModel name:                           AMD EPYC 7402P 24-Core Processor\nCPU family:                           23\nModel:                                49\nThread(s) per core:                   2\nCore(s) per socket:                   24\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU max MHz:                          2800.0000\nCPU min MHz:                          1500.0000\nBogoMIPS:                             5600.34\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            768 KiB (24 instances)\nL1i cache:                            768 KiB (24 instances)\nL2 cache:                             12 MiB (24 instances)\nL3 cache:                             128 MiB (8 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-47\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.6.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.6.80\n[pip3] nvidia-cuda-nvrtc-cu12==12.6.85\n[pip3] nvidia-cuda-runtime-cu12==12.6.77\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.0.4\n[pip3] nvidia-curand-cu12==10.3.7.77\n[pip3] nvidia-cusolver-cu12==11.7.1.2\n[pip3] nvidia-cusparse-cu12==12.5.4.2\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.29.3\n[pip3] nvidia-nvjitlink-cu12==12.6.85\n[pip3] nvidia-nvtx-cu12==12.6.77\n[pip3] torch==2.13.0+cu126\n[pip3] triton==3.7.1\n[conda] Could not collect",
+  "transformers_version": "5.14.1",
+  "lm_eval_version": "0.4.12",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    "151645"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151645,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "/root/steering-resistance/results/full_7b/m1_resist_adapter",
+  "model_name_sanitized": "__root__steering-resistance__results__full_7b__m1_resist_adapter",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": true,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+  "total_evaluation_time_seconds": "270.4222217015922"
+}
\ No newline at end of file
diff --git a/run/invocations.jsonl b/run/invocations.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c854b65be50d50179c235c937b3fbb50249b34a1
--- /dev/null
+++ b/run/invocations.jsonl
@@ -0,0 +1 @@
+{"time": "2026-07-23T18:26:39+0000", "stages": ["vectors", "data", "eval_m0", "train", "eval_m1"], "commit": "none", "status": "success", "headline": "clean 100%->100% · steer_heldout@1.6 correct 0%->20%", "wandb_url": null}
diff --git a/run/run_meta.json b/run/run_meta.json
index c543abdca339021d6e78cd43e928be451f374516..e15906278d355fb6bef6c51e9d9abd5a71dc8e27 100644
--- a/run/run_meta.json
+++ b/run/run_meta.json
@@ -110,5 +110,51 @@
    "sha256": "fa2356571420fd8c2a444aee6e8c879b865e0cd21f5ade2938a0308effbed8f2"
   }
  },
- "status": "running"
+ "status": "success",
+ "wandb_url": null,
+ "headline": "clean 100%->100% · steer_heldout@1.6 correct 0%->20%",
+ "hub_url": "https://huggingface.co/JacoDuToit/steer-full_7b",
+ "finished_at": "2026-07-23T20:16:36+0000",
+ "artifacts": {
+  "eval_m0.jsonl": {
+   "sha256": "850d1c677802f170a8e4552ab3323b6438acca7274047b8a6590c21451e69d49",
+   "bytes": 4749617
+  },
+  "eval_m1.jsonl": {
+   "sha256": "0a1ccba3eea0950ffc11429f3d8d89e58b5533db3b3605a17e77e07f8c05416f",
+   "bytes": 4129347
+  },
+  "eval_questions.json": {
+   "sha256": "0685a496f1a91ac4c4e06f4f56486ed8c615ba482cde3691002cd5ffa6d47a2b",
+   "bytes": 30264
+  },
+  "m1_resist_adapter/README.md": {
+   "sha256": "3191b465069be139a907db620ecaaee6188deea1f5e14f8ec7281f71bba535d2",
+   "bytes": 5202
+  },
+  "m1_resist_adapter/adapter_config.json": {
+   "sha256": "9a907152004d1043d9167bedfa3ba620293502340a75b5a0353fb8a4631a3531",
+   "bytes": 1103
+  },
+  "m1_resist_adapter/adapter_model.safetensors": {
+   "sha256": "63031303ded8f07d12eb164746743fd2ee2aae4d97b6e25df947ef6f5776f5f6",
+   "bytes": 161533192
+  },
+  "summary.csv": {
+   "sha256": "40dbf0f76cf633f9312039bdd6792efc2d54b0b23ee483044a0b82052e9d2626",
+   "bytes": 2561
+  },
+  "summary.md": {
+   "sha256": "7bc571ac210c16f6a6decea771082a8f3af230aae8bdc1de405012d86f09bc1f",
+   "bytes": 2606
+  },
+  "train_examples.json": {
+   "sha256": "1ba9a7154e366cab28361a413378675c232d26b906fac8e05e82d67282d9e8dd",
+   "bytes": 301292
+  },
+  "vectors.pt": {
+   "sha256": "a4efa358fc4a9d785793739be44c5355e7d29cf5f8aa17927c1c80fcac19bc4d",
+   "bytes": 10884283
+  }
+ }
 }
\ No newline at end of file