Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+lm_eval_logs/mmlu_results.json/__egr__research-optml__wangc168__Muon_wmdp__rmu_wmdp__models__muon_v1_test_V1_K64_batches150_bs4_alpha1200-1200_steer15-15_muonlr2.6e-3_muonmom0.95_muonwd0.1_adamlr5e-5_seed42_layer7_layers5-6-7_params6/samples_mmlu_professional_law_2026-04-03T16-40-13.638747.jsonl filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "HuggingFaceH4/zephyr-7b-beta",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 2,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.43.0"
+}

lm_eval_logs/mmlu.log

@@ -0,0 +1,318 @@
+/egr/research-optml/wangc168/anaconda3/envs/SOUL/lib/python3.9/site-packages/transformers/utils/hub.py:127: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.
+  warnings.warn(
+2026-04-03:16:35:59,252 INFO     [__main__.py:279] Verbosity set to INFO
+2026-04-03:16:35:59,682 INFO     [__init__.py:491] `group` and `group_alias` keys in TaskConfigs are deprecated and will be removed in v0.4.5 of lm_eval. The new `tag` field will be used to allow for a shortcut to a group of tasks one does not wish to aggregate metrics across. `group`s which aggregate across subtasks must be only defined in a separate group config file, which will be the official way to create groups that support cross-task aggregation as in `mmlu`. Please see the v0.4.4 patch notes and our documentation: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/new_task_guide.md#advanced-group-configs for more information.
+2026-04-03:16:36:07,612 INFO     [__main__.py:376] Selected Tasks: ['mmlu']
+2026-04-03:16:36:07,615 INFO     [evaluator.py:161] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234
+2026-04-03:16:36:07,615 INFO     [evaluator.py:198] Initializing hf model, with arguments: {'pretrained': '/egr/research-optml/wangc168/Muon_wmdp/rmu_wmdp/models/muon_v1_test_V1_K64_batches150_bs4_alpha1200-1200_steer15-15_muonlr2.6e-3_muonmom0.95_muonwd0.1_adamlr5e-5_seed42_layer7_layers5-6-7_params6'}
+2026-04-03:16:36:07,760 INFO     [huggingface.py:130] Using device 'cuda:0'
+2026-04-03:16:36:07,992 INFO     [huggingface.py:366] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:0'}
+
+2026-04-03:16:37:53,518 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,518 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,518 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,519 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,520 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234
+2026-04-03:16:37:53,521 WARNING  [model.py:422] model.chat_template was called with the chat_template set to False or None. Therefore no chat template will be applied. Make sure this is an intended behavior.
+2026-04-03:16:37:53,529 INFO     [task.py:423] Building contexts for mmlu_high_school_mathematics on rank 0...
+
  0%|          | 0/270 [00:00<?, ?it/s]
 30%|███       | 82/270 [00:00<00:00, 819.04it/s]
 74%|███████▍  | 200/270 [00:00<00:00, 1029.54it/s]
+2026-04-03:16:37:53,804 INFO     [task.py:423] Building contexts for mmlu_college_chemistry on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:53,891 INFO     [task.py:423] Building contexts for mmlu_computer_security on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:53,978 INFO     [task.py:423] Building contexts for mmlu_conceptual_physics on rank 0...
+
  0%|          | 0/235 [00:00<?, ?it/s]
 51%|█████     | 120/235 [00:00<00:00, 1197.33it/s]
+2026-04-03:16:37:54,181 INFO     [task.py:423] Building contexts for mmlu_college_biology on rank 0...
+
  0%|          | 0/144 [00:00<?, ?it/s]
 83%|████████▎ | 120/144 [00:00<00:00, 1193.77it/s]
+2026-04-03:16:37:54,306 INFO     [task.py:423] Building contexts for mmlu_college_computer_science on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:54,392 INFO     [task.py:423] Building contexts for mmlu_abstract_algebra on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:54,478 INFO     [task.py:423] Building contexts for mmlu_machine_learning on rank 0...
+
  0%|          | 0/112 [00:00<?, ?it/s]
+2026-04-03:16:37:54,575 INFO     [task.py:423] Building contexts for mmlu_high_school_computer_science on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:54,661 INFO     [task.py:423] Building contexts for mmlu_college_physics on rank 0...
+
  0%|          | 0/102 [00:00<?, ?it/s]
+2026-04-03:16:37:54,750 INFO     [task.py:423] Building contexts for mmlu_high_school_statistics on rank 0...
+
  0%|          | 0/216 [00:00<?, ?it/s]
 55%|█████▌    | 119/216 [00:00<00:00, 1186.72it/s]
+2026-04-03:16:37:54,937 INFO     [task.py:423] Building contexts for mmlu_high_school_physics on rank 0...
+
  0%|          | 0/151 [00:00<?, ?it/s]
 79%|███████▉  | 119/151 [00:00<00:00, 1189.56it/s]
+2026-04-03:16:37:55,068 INFO     [task.py:423] Building contexts for mmlu_elementary_mathematics on rank 0...
+
  0%|          | 0/378 [00:00<?, ?it/s]
 31%|███       | 118/378 [00:00<00:00, 1179.87it/s]
 63%|██████▎   | 237/378 [00:00<00:00, 1185.01it/s]
 94%|█████████▍| 356/378 [00:00<00:00, 1184.24it/s]
+2026-04-03:16:37:55,396 INFO     [task.py:423] Building contexts for mmlu_electrical_engineering on rank 0...
+
  0%|          | 0/145 [00:00<?, ?it/s]
 83%|████████▎ | 120/145 [00:00<00:00, 1198.54it/s]
+2026-04-03:16:37:55,521 INFO     [task.py:423] Building contexts for mmlu_high_school_biology on rank 0...
+
  0%|          | 0/310 [00:00<?, ?it/s]
 39%|███▊      | 120/310 [00:00<00:00, 1194.88it/s]
 77%|███████▋  | 240/310 [00:00<00:00, 1189.11it/s]
+2026-04-03:16:37:55,790 INFO     [task.py:423] Building contexts for mmlu_astronomy on rank 0...
+
  0%|          | 0/152 [00:00<?, ?it/s]
 79%|███████▉  | 120/152 [00:00<00:00, 1194.28it/s]
+2026-04-03:16:37:55,921 INFO     [task.py:423] Building contexts for mmlu_anatomy on rank 0...
+
  0%|          | 0/135 [00:00<?, ?it/s]
 89%|████████▉ | 120/135 [00:00<00:00, 1199.04it/s]
+2026-04-03:16:37:56,038 INFO     [task.py:423] Building contexts for mmlu_college_mathematics on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:56,124 INFO     [task.py:423] Building contexts for mmlu_high_school_chemistry on rank 0...
+
  0%|          | 0/203 [00:00<?, ?it/s]
 59%|█████▉    | 120/203 [00:00<00:00, 1194.98it/s]
+2026-04-03:16:37:56,299 INFO     [task.py:423] Building contexts for mmlu_management on rank 0...
+
  0%|          | 0/103 [00:00<?, ?it/s]
+2026-04-03:16:37:56,388 INFO     [task.py:423] Building contexts for mmlu_clinical_knowledge on rank 0...
+
  0%|          | 0/265 [00:00<?, ?it/s]
 45%|████▌     | 120/265 [00:00<00:00, 1190.62it/s]
 91%|█████████ | 240/265 [00:00<00:00, 1188.99it/s]
+2026-04-03:16:37:56,618 INFO     [task.py:423] Building contexts for mmlu_professional_medicine on rank 0...
+
  0%|          | 0/272 [00:00<?, ?it/s]
 44%|████▍     | 120/272 [00:00<00:00, 1192.97it/s]
 88%|████████▊ | 240/272 [00:00<00:00, 1191.79it/s]
+2026-04-03:16:37:56,853 INFO     [task.py:423] Building contexts for mmlu_human_aging on rank 0...
+
  0%|          | 0/223 [00:00<?, ?it/s]
 33%|███▎      | 74/223 [00:00<00:00, 340.71it/s]
 87%|████████▋ | 193/223 [00:00<00:00, 676.52it/s]
+2026-04-03:16:37:57,201 INFO     [task.py:423] Building contexts for mmlu_professional_accounting on rank 0...
+
  0%|          | 0/282 [00:00<?, ?it/s]
 42%|████▏     | 119/282 [00:00<00:00, 1189.22it/s]
 84%|████████▍ | 238/282 [00:00<00:00, 1186.54it/s]
+2026-04-03:16:37:57,447 INFO     [task.py:423] Building contexts for mmlu_college_medicine on rank 0...
+
  0%|          | 0/173 [00:00<?, ?it/s]
 69%|██████▉   | 119/173 [00:00<00:00, 1189.56it/s]
+2026-04-03:16:37:57,596 INFO     [task.py:423] Building contexts for mmlu_medical_genetics on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:57,683 INFO     [task.py:423] Building contexts for mmlu_miscellaneous on rank 0...
+
  0%|          | 0/783 [00:00<?, ?it/s]
 15%|█▌        | 120/783 [00:00<00:00, 1197.11it/s]
 31%|███       | 240/783 [00:00<00:00, 1193.18it/s]
 46%|████▌     | 360/783 [00:00<00:00, 1190.30it/s]
 61%|██████▏   | 480/783 [00:00<00:00, 1191.35it/s]
 77%|███████▋  | 600/783 [00:00<00:00, 1189.42it/s]
 92%|█████████▏| 720/783 [00:00<00:00, 1190.78it/s]
+2026-04-03:16:37:58,358 INFO     [task.py:423] Building contexts for mmlu_global_facts on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:58,445 INFO     [task.py:423] Building contexts for mmlu_virology on rank 0...
+
  0%|          | 0/166 [00:00<?, ?it/s]
 72%|███████▏  | 120/166 [00:00<00:00, 1198.06it/s]
+2026-04-03:16:37:58,587 INFO     [task.py:423] Building contexts for mmlu_business_ethics on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:37:58,674 INFO     [task.py:423] Building contexts for mmlu_nutrition on rank 0...
+
  0%|          | 0/306 [00:00<?, ?it/s]
 39%|███▉      | 120/306 [00:00<00:00, 1198.40it/s]
 78%|███████▊  | 240/306 [00:00<00:00, 1194.25it/s]
+2026-04-03:16:37:58,937 INFO     [task.py:423] Building contexts for mmlu_marketing on rank 0...
+
  0%|          | 0/234 [00:00<?, ?it/s]
 51%|█████▏    | 120/234 [00:00<00:00, 1194.33it/s]
+2026-04-03:16:37:59,139 INFO     [task.py:423] Building contexts for mmlu_professional_psychology on rank 0...
+
  0%|          | 0/612 [00:00<?, ?it/s]
 20%|█▉        | 120/612 [00:00<00:00, 1190.56it/s]
 39%|███▉      | 240/612 [00:00<00:00, 1191.50it/s]
 59%|█████▉    | 360/612 [00:00<00:00, 1193.22it/s]
 78%|███████▊  | 480/612 [00:00<00:00, 1192.31it/s]
 98%|█████████▊| 600/612 [00:00<00:00, 1193.37it/s]
+2026-04-03:16:37:59,667 INFO     [task.py:423] Building contexts for mmlu_high_school_geography on rank 0...
+
  0%|          | 0/198 [00:00<?, ?it/s]
 61%|██████    | 120/198 [00:00<00:00, 1190.72it/s]
+2026-04-03:16:37:59,838 INFO     [task.py:423] Building contexts for mmlu_high_school_psychology on rank 0...
+
  0%|          | 0/545 [00:00<?, ?it/s]
 22%|██▏       | 121/545 [00:00<00:00, 1200.06it/s]
 44%|████▍     | 242/545 [00:00<00:00, 1191.97it/s]
 66%|██████▋   | 362/545 [00:00<00:00, 1192.51it/s]
 88%|████████▊ | 482/545 [00:00<00:00, 1191.55it/s]
+2026-04-03:16:38:00,309 INFO     [task.py:423] Building contexts for mmlu_us_foreign_policy on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
+2026-04-03:16:38:00,396 INFO     [task.py:423] Building contexts for mmlu_high_school_microeconomics on rank 0...
+
  0%|          | 0/238 [00:00<?, ?it/s]
 50%|█████     | 120/238 [00:00<00:00, 1191.20it/s]
+2026-04-03:16:38:00,602 INFO     [task.py:423] Building contexts for mmlu_sociology on rank 0...
+
  0%|          | 0/201 [00:00<?, ?it/s]
 60%|█████▉    | 120/201 [00:00<00:00, 1191.49it/s]
+2026-04-03:16:38:00,776 INFO     [task.py:423] Building contexts for mmlu_security_studies on rank 0...
+
  0%|          | 0/245 [00:00<?, ?it/s]
 49%|████▊     | 119/245 [00:00<00:00, 1188.33it/s]
 97%|█████████▋| 238/245 [00:00<00:00, 1187.83it/s]
+2026-04-03:16:38:00,989 INFO     [task.py:423] Building contexts for mmlu_public_relations on rank 0...
+
  0%|          | 0/110 [00:00<?, ?it/s]
+2026-04-03:16:38:01,083 INFO     [task.py:423] Building contexts for mmlu_human_sexuality on rank 0...
+
  0%|          | 0/131 [00:00<?, ?it/s]
 92%|█████████▏| 120/131 [00:00<00:00, 1198.52it/s]
+2026-04-03:16:38:01,196 INFO     [task.py:423] Building contexts for mmlu_high_school_government_and_politics on rank 0...
+
  0%|          | 0/193 [00:00<?, ?it/s]
 62%|██████▏   | 119/193 [00:00<00:00, 1188.54it/s]
+2026-04-03:16:38:01,364 INFO     [task.py:423] Building contexts for mmlu_econometrics on rank 0...
+
  0%|          | 0/114 [00:00<?, ?it/s]
+2026-04-03:16:38:01,462 INFO     [task.py:423] Building contexts for mmlu_high_school_macroeconomics on rank 0...
+
  0%|          | 0/390 [00:00<?, ?it/s]
 31%|███       | 121/390 [00:00<00:00, 1200.51it/s]
 62%|██████▏   | 242/390 [00:00<00:00, 1193.08it/s]
 93%|█████████▎| 362/390 [00:00<00:00, 1192.32it/s]
+2026-04-03:16:38:01,799 INFO     [task.py:423] Building contexts for mmlu_formal_logic on rank 0...
+
  0%|          | 0/126 [00:00<?, ?it/s]
 94%|█████████▍| 119/126 [00:00<00:00, 1187.81it/s]
+2026-04-03:16:38:01,908 INFO     [task.py:423] Building contexts for mmlu_moral_disputes on rank 0...
+
  0%|          | 0/346 [00:00<?, ?it/s]
 34%|███▍      | 119/346 [00:00<00:00, 1186.30it/s]
 69%|██████▉   | 239/346 [00:00<00:00, 1189.05it/s]
+2026-04-03:16:38:02,208 INFO     [task.py:423] Building contexts for mmlu_moral_scenarios on rank 0...
+
  0%|          | 0/895 [00:00<?, ?it/s]
 13%|█▎        | 120/895 [00:00<00:00, 1190.24it/s]
 27%|██▋       | 240/895 [00:00<00:00, 1191.14it/s]
 40%|████      | 360/895 [00:00<00:00, 1189.38it/s]
 54%|█████▎    | 480/895 [00:00<00:00, 1190.82it/s]
 67%|██████▋   | 600/895 [00:00<00:00, 1187.94it/s]
 80%|████████  | 719/895 [00:00<00:00, 1187.12it/s]
 94%|█████████▎| 839/895 [00:00<00:00, 1189.43it/s]
+2026-04-03:16:38:02,982 INFO     [task.py:423] Building contexts for mmlu_jurisprudence on rank 0...
+
  0%|          | 0/108 [00:00<?, ?it/s]
+2026-04-03:16:38:03,075 INFO     [task.py:423] Building contexts for mmlu_high_school_european_history on rank 0...
+
  0%|          | 0/165 [00:00<?, ?it/s]
 59%|█████▉    | 97/165 [00:00<00:00, 316.18it/s]
+2026-04-03:16:38:03,445 INFO     [task.py:423] Building contexts for mmlu_professional_law on rank 0...
+
  0%|          | 0/1534 [00:00<?, ?it/s]
  8%|▊         | 119/1534 [00:00<00:01, 1181.94it/s]
 16%|█▌        | 238/1534 [00:00<00:01, 1183.71it/s]
 23%|██▎       | 357/1534 [00:00<00:00, 1185.37it/s]
 31%|███       | 477/1534 [00:00<00:00, 1188.98it/s]
 39%|███▉      | 597/1534 [00:00<00:00, 1189.89it/s]
 47%|████▋     | 717/1534 [00:00<00:00, 1192.61it/s]
 55%|█████▍    | 837/1534 [00:00<00:00, 1191.78it/s]
 62%|██████▏   | 957/1534 [00:00<00:00, 1192.38it/s]
 70%|███████   | 1077/1534 [00:00<00:00, 1192.50it/s]
 78%|███████▊  | 1197/1534 [00:01<00:00, 1190.77it/s]
 86%|████████▌ | 1317/1534 [00:01<00:00, 1190.94it/s]
 94%|█████████▎| 1437/1534 [00:01<00:00, 1190.67it/s]
+2026-04-03:16:38:04,772 INFO     [task.py:423] Building contexts for mmlu_prehistory on rank 0...
+
  0%|          | 0/324 [00:00<?, ?it/s]
 37%|███▋      | 119/324 [00:00<00:00, 1181.62it/s]
 73%|███████▎  | 238/324 [00:00<00:00, 1184.06it/s]
+2026-04-03:16:38:05,054 INFO     [task.py:423] Building contexts for mmlu_world_religions on rank 0...
+
  0%|          | 0/171 [00:00<?, ?it/s]
 70%|███████   | 120/171 [00:00<00:00, 1191.27it/s]
+2026-04-03:16:38:05,202 INFO     [task.py:423] Building contexts for mmlu_high_school_world_history on rank 0...
+
  0%|          | 0/237 [00:00<?, ?it/s]
 50%|█████     | 119/237 [00:00<00:00, 1188.77it/s]
+2026-04-03:16:38:05,408 INFO     [task.py:423] Building contexts for mmlu_international_law on rank 0...
+
  0%|          | 0/121 [00:00<?, ?it/s]
 99%|█████████▉| 120/121 [00:00<00:00, 1193.52it/s]
+2026-04-03:16:38:05,513 INFO     [task.py:423] Building contexts for mmlu_logical_fallacies on rank 0...
+
  0%|          | 0/163 [00:00<?, ?it/s]
 74%|███████▍  | 121/163 [00:00<00:00, 1205.56it/s]
+2026-04-03:16:38:05,652 INFO     [task.py:423] Building contexts for mmlu_philosophy on rank 0...
+
  0%|          | 0/311 [00:00<?, ?it/s]
 39%|███▉      | 121/311 [00:00<00:00, 1201.64it/s]
 78%|███████▊  | 242/311 [00:00<00:00, 1195.51it/s]
+2026-04-03:16:38:05,920 INFO     [task.py:423] Building contexts for mmlu_high_school_us_history on rank 0...
+
  0%|          | 0/204 [00:00<?, ?it/s]
 58%|█████▊    | 118/204 [00:00<00:00, 1174.86it/s]
+2026-04-03:16:38:06,099 INFO     [evaluator.py:465] Running loglikelihood requests
+
+
+2026-04-03:16:40:08,272 WARNING  [huggingface.py:1344] Failed to get model SHA for /egr/research-optml/wangc168/Muon_wmdp/rmu_wmdp/models/muon_v1_test_V1_K64_batches150_bs4_alpha1200-1200_steer15-15_muonlr2.6e-3_muonmom0.95_muonwd0.1_adamlr5e-5_seed42_layer7_layers5-6-7_params6 at revision main. Error: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/egr/research-optml/wangc168/Muon_wmdp/rmu_wmdp/models/muon_v1_test_V1_K64_batches150_bs4_alpha1200-1200_steer15-15_muonlr2.6e-3_muonmom0.95_muonwd0.1_adamlr5e-5_seed42_layer7_layers5-6-7_params6'. Use `repo_type` argument if needed.
+2026-04-03:16:40:13,621 INFO     [evaluation_tracker.py:206] Saving results aggregated
+2026-04-03:16:40:13,641 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_abstract_algebra
+2026-04-03:16:40:13,711 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_anatomy
+2026-04-03:16:40:13,809 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_astronomy
+2026-04-03:16:40:13,927 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_business_ethics
+2026-04-03:16:40:14,011 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_clinical_knowledge
+2026-04-03:16:40:14,215 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_college_biology
+2026-04-03:16:40:14,331 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_college_chemistry
+2026-04-03:16:40:14,423 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_college_computer_science
+2026-04-03:16:40:14,517 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_college_mathematics
+2026-04-03:16:40:14,607 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_college_medicine
+2026-04-03:16:40:14,749 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_college_physics
+2026-04-03:16:40:14,828 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_computer_security
+2026-04-03:16:40:14,915 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_conceptual_physics
+2026-04-03:16:40:15,093 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_econometrics
+2026-04-03:16:40:15,183 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_electrical_engineering
+2026-04-03:16:40:15,308 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_elementary_mathematics
+2026-04-03:16:40:15,632 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_formal_logic
+2026-04-03:16:40:15,732 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_global_facts
+2026-04-03:16:40:15,808 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_biology
+2026-04-03:16:40:16,059 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_chemistry
+2026-04-03:16:40:16,266 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_computer_science
+2026-04-03:16:40:16,375 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_european_history
+2026-04-03:16:40:16,583 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_geography
+2026-04-03:16:40:16,806 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_government_and_politics
+2026-04-03:16:40:17,022 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_macroeconomics
+2026-04-03:16:40:17,385 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_mathematics
+2026-04-03:16:40:17,684 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_microeconomics
+2026-04-03:16:40:17,929 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_physics
+2026-04-03:16:40:18,109 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_psychology
+2026-04-03:16:40:18,727 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_statistics
+2026-04-03:16:40:18,908 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_us_history
+2026-04-03:16:40:19,537 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_high_school_world_history
+2026-04-03:16:40:19,726 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_human_aging
+2026-04-03:16:40:19,895 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_human_sexuality
+2026-04-03:16:40:20,007 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_international_law
+2026-04-03:16:40:20,097 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_jurisprudence
+2026-04-03:16:40:20,195 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_logical_fallacies
+2026-04-03:16:40:20,341 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_machine_learning
+2026-04-03:16:40:20,436 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_management
+2026-04-03:16:40:20,524 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_marketing
+2026-04-03:16:40:20,789 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_medical_genetics
+2026-04-03:16:40:20,897 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_miscellaneous
+2026-04-03:16:40:21,555 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_moral_disputes
+2026-04-03:16:40:21,850 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_moral_scenarios
+2026-04-03:16:40:22,676 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_nutrition
+2026-04-03:16:40:22,944 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_philosophy
+2026-04-03:16:40:23,198 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_prehistory
+2026-04-03:16:40:23,473 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_professional_accounting
+2026-04-03:16:40:23,719 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_professional_law
+2026-04-03:16:40:25,243 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_professional_medicine
+2026-04-03:16:40:25,522 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_professional_psychology
+2026-04-03:16:40:26,010 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_public_relations
+2026-04-03:16:40:26,094 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_security_studies
+2026-04-03:16:40:26,264 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_sociology
+2026-04-03:16:40:26,404 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_us_foreign_policy
+2026-04-03:16:40:26,475 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_virology
+2026-04-03:16:40:26,587 INFO     [evaluation_tracker.py:287] Saving per-sample results for: mmlu_world_religions
+hf (pretrained=/egr/research-optml/wangc168/Muon_wmdp/rmu_wmdp/models/muon_v1_test_V1_K64_batches150_bs4_alpha1200-1200_steer15-15_muonlr2.6e-3_muonmom0.95_muonwd0.1_adamlr5e-5_seed42_layer7_layers5-6-7_params6), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 16
+|                 Tasks                 |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|---------------------------------------|------:|------|-----:|------|---|-----:|---|-----:|
+|mmlu                                   |      2|none  |      |acc   |↑  |0.5713|±  |0.0040|
+| - humanities                          |      2|none  |      |acc   |↑  |0.5186|±  |0.0069|
+|  - formal_logic                       |      1|none  |     0|acc   |↑  |0.3810|±  |0.0434|
+|  - high_school_european_history       |      1|none  |     0|acc   |↑  |0.6970|±  |0.0359|
+|  - high_school_us_history             |      1|none  |     0|acc   |↑  |0.7500|±  |0.0304|
+|  - high_school_world_history          |      1|none  |     0|acc   |↑  |0.7384|±  |0.0286|
+|  - international_law                  |      1|none  |     0|acc   |↑  |0.7107|±  |0.0414|
+|  - jurisprudence                      |      1|none  |     0|acc   |↑  |0.6944|±  |0.0445|
+|  - logical_fallacies                  |      1|none  |     0|acc   |↑  |0.6810|±  |0.0366|
+|  - moral_disputes                     |      1|none  |     0|acc   |↑  |0.6590|±  |0.0255|
+|  - moral_scenarios                    |      1|none  |     0|acc   |↑  |0.2804|±  |0.0150|
+|  - philosophy                         |      1|none  |     0|acc   |↑  |0.6463|±  |0.0272|
+|  - prehistory                         |      1|none  |     0|acc   |↑  |0.6451|±  |0.0266|
+|  - professional_law                   |      1|none  |     0|acc   |↑  |0.4231|±  |0.0126|
+|  - world_religions                    |      1|none  |     0|acc   |↑  |0.8129|±  |0.0299|
+| - other                               |      2|none  |      |acc   |↑  |0.6350|±  |0.0082|
+|  - business_ethics                    |      1|none  |     0|acc   |↑  |0.5200|±  |0.0502|
+|  - clinical_knowledge                 |      1|none  |     0|acc   |↑  |0.6491|±  |0.0294|
+|  - college_medicine                   |      1|none  |     0|acc   |↑  |0.5896|±  |0.0375|
+|  - global_facts                       |      1|none  |     0|acc   |↑  |0.3000|±  |0.0461|
+|  - human_aging                        |      1|none  |     0|acc   |↑  |0.6368|±  |0.0323|
+|  - management                         |      1|none  |     0|acc   |↑  |0.7476|±  |0.0430|
+|  - marketing                          |      1|none  |     0|acc   |↑  |0.8248|±  |0.0249|
+|  - medical_genetics                   |      1|none  |     0|acc   |↑  |0.6400|±  |0.0482|
+|  - miscellaneous                      |      1|none  |     0|acc   |↑  |0.7778|±  |0.0149|
+|  - nutrition                          |      1|none  |     0|acc   |↑  |0.6601|±  |0.0271|
+|  - professional_accounting            |      1|none  |     0|acc   |↑  |0.4149|±  |0.0294|
+|  - professional_medicine              |      1|none  |     0|acc   |↑  |0.5699|±  |0.0301|
+|  - virology                           |      1|none  |     0|acc   |↑  |0.3494|±  |0.0371|
+| - social sciences                     |      2|none  |      |acc   |↑  |0.6783|±  |0.0082|
+|  - econometrics                       |      1|none  |     0|acc   |↑  |0.4298|±  |0.0466|
+|  - high_school_geography              |      1|none  |     0|acc   |↑  |0.7172|±  |0.0321|
+|  - high_school_government_and_politics|      1|none  |     0|acc   |↑  |0.8290|±  |0.0272|
+|  - high_school_macroeconomics         |      1|none  |     0|acc   |↑  |0.5795|±  |0.0250|
+|  - high_school_microeconomics         |      1|none  |     0|acc   |↑  |0.6050|±  |0.0318|
+|  - high_school_psychology             |      1|none  |     0|acc   |↑  |0.7890|±  |0.0175|
+|  - human_sexuality                    |      1|none  |     0|acc   |↑  |0.6565|±  |0.0416|
+|  - professional_psychology            |      1|none  |     0|acc   |↑  |0.5997|±  |0.0198|
+|  - public_relations                   |      1|none  |     0|acc   |↑  |0.6545|±  |0.0455|
+|  - security_studies                   |      1|none  |     0|acc   |↑  |0.6490|±  |0.0306|
+|  - sociology                          |      1|none  |     0|acc   |↑  |0.8557|±  |0.0248|
+|  - us_foreign_policy                  |      1|none  |     0|acc   |↑  |0.8000|±  |0.0402|
+| - stem                                |      2|none  |      |acc   |↑  |0.4827|±  |0.0087|
+|  - abstract_algebra                   |      1|none  |     0|acc   |↑  |0.3000|±  |0.0461|
+|  - anatomy                            |      1|none  |     0|acc   |↑  |0.5630|±  |0.0428|
+|  - astronomy                          |      1|none  |     0|acc   |↑  |0.5855|±  |0.0401|
+|  - college_biology                    |      1|none  |     0|acc   |↑  |0.6389|±  |0.0402|
+|  - college_chemistry                  |      1|none  |     0|acc   |↑  |0.4900|±  |0.0502|
+|  - college_computer_science           |      1|none  |     0|acc   |↑  |0.5100|±  |0.0502|
+|  - college_mathematics                |      1|none  |     0|acc   |↑  |0.3700|±  |0.0485|
+|  - college_physics                    |      1|none  |     0|acc   |↑  |0.5294|±  |0.0497|
+|  - computer_security                  |      1|none  |     0|acc   |↑  |0.4800|±  |0.0502|
+|  - conceptual_physics                 |      1|none  |     0|acc   |↑  |0.4936|±  |0.0327|
+|  - electrical_engineering             |      1|none  |     0|acc   |↑  |0.5310|±  |0.0416|
+|  - elementary_mathematics             |      1|none  |     0|acc   |↑  |0.3995|±  |0.0252|
+|  - high_school_biology                |      1|none  |     0|acc   |↑  |0.6581|±  |0.0270|
+|  - high_school_chemistry              |      1|none  |     0|acc   |↑  |0.4828|±  |0.0352|
+|  - high_school_computer_science       |      1|none  |     0|acc   |↑  |0.5700|±  |0.0498|
+|  - high_school_mathematics            |      1|none  |     0|acc   |↑  |0.3370|±  |0.0288|
+|  - high_school_physics                |      1|none  |     0|acc   |↑  |0.2914|±  |0.0371|
+|  - high_school_statistics             |      1|none  |     0|acc   |↑  |0.4907|±  |0.0341|
+|  - machine_learning                   |      1|none  |     0|acc   |↑  |0.4643|±  |0.0473|
+
+|      Groups      |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|------------------|------:|------|------|------|---|-----:|---|-----:|
+|mmlu              |      2|none  |      |acc   |↑  |0.5713|±  |0.0040|
+| - humanities     |      2|none  |      |acc   |↑  |0.5186|±  |0.0069|
+| - other          |      2|none  |      |acc   |↑  |0.6350|±  |0.0082|
+| - social sciences|      2|none  |      |acc   |↑  |0.6783|±  |0.0082|
+| - stem           |      2|none  |      |acc   |↑  |0.4827|±  |0.0087|
+