Round 4 oracle-fix results

.gitattributes

@@ -198,3 +198,156 @@ round3/Y_pred/mmlu_college_chemistry_topk8_global_ridge_math+code_N14/tokenizer.
 round3/Y_pred/mmlu_college_chemistry_topk8_global_ridge_math-only_N8/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 round3/Y_pred/mmlu_college_chemistry_topk8_global_ridge_science-only_N8/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 round3/Y_pred/mmlu_college_chemistry_topk8_pertensor_ridge_N24/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+figures/exp3_transfer_heatmap_r4.png filter=lfs diff=lfs merge=lfs -text
+round4/X/aqua_rat/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/arc_challenge/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/arc_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/gsm8k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/gsm8k_test_500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/gsm_hard/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/humaneval/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/math_algebra_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/math_counting_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/mbpp/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/mbpp_plus/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/mbpp_sanitized/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/mbpp_test_held/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/medmcqa_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/mmlu_elementary_math/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/mmlu_high_school_biology/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/mmlu_high_school_physics/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/multiarith/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/openbookqa/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/openbookqa_test/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/sciq/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/X/svamp/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/aqua_rat/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/arc_challenge/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/arc_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/gsm8k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/gsm8k_test_500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/gsm_hard/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/humaneval/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/math_algebra_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/math_counting_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/mbpp/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/mbpp_plus/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/mbpp_sanitized/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/mbpp_test_held/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/medmcqa_easy/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/mmlu_elementary_math/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/mmlu_high_school_biology/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/mmlu_high_school_physics/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/multiarith/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/openbookqa/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/openbookqa_test/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/sciq/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y/svamp/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_global_ridge_N5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_mean_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_pertensor_mlp_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_pertensor_pca_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_procrustes_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk12_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk12_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_global_ridge_all_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_global_ridge_code-only_N3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_global_ridge_math+code_N9/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_global_ridge_math-only_N6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_global_ridge_science-only_N7/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/arc_challenge_topk8_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_global_ridge_N5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_mean_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_pertensor_mlp_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_pertensor_pca_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_procrustes_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk12_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk12_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_global_ridge_all_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_global_ridge_code-only_N3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_global_ridge_math+code_N9/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_global_ridge_math-only_N6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_global_ridge_science-only_N7/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm8k_test_500_topk8_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_global_ridge_N5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_mean_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_pertensor_mlp_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_pertensor_pca_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_procrustes_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk12_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk12_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_global_ridge_all_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_global_ridge_code-only_N3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_global_ridge_math+code_N9/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_global_ridge_math-only_N6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_global_ridge_science-only_N7/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/gsm_hard_topk8_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_global_ridge_N5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_mean_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_pertensor_mlp_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_pertensor_pca_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_procrustes_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk12_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk12_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_global_ridge_all_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_global_ridge_code-only_N3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_global_ridge_math+code_N9/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_global_ridge_math-only_N6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_global_ridge_science-only_N7/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_plus_topk8_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_global_ridge_N5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_mean_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_pertensor_mlp_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_pertensor_pca_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_procrustes_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk12_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk12_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_global_ridge_all_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_global_ridge_code-only_N3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_global_ridge_math+code_N9/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_global_ridge_math-only_N6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_global_ridge_science-only_N7/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/mbpp_test_held_topk8_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_global_ridge_N5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_mean_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_pertensor_mlp_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_pertensor_pca_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_procrustes_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk12_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk12_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_global_ridge_N12/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_global_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_global_ridge_all_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_global_ridge_code-only_N3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_global_ridge_math+code_N9/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_global_ridge_math-only_N6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_global_ridge_science-only_N7/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+round4/Y_pred/openbookqa_test_topk8_pertensor_ridge_N16/tokenizer.json filter=lfs diff=lfs merge=lfs -text

ROUND4_REPORT.md

@@ -0,0 +1,64 @@
+# Cross-Model LoRA Adapter Translation — Round 4
+
+**Repo:** https://huggingface.co/CK0607/cross-model-lora-prediction-3b
+**Models:** X=`Qwen/Qwen2.5-3B-Instruct` → Y=`meta-llama/Llama-3.2-3B-Instruct`
+
+## Diff vs Round 3
+
+- Kept Round 3 3B model pair and mapping algorithms unchanged.
+- Replaced broken held-outs: `math_algebra_medium` → `gsm8k_test_500`, `humaneval_plus` → `mbpp_test_held`, `mmlu_college_chemistry` → `openbookqa_test`.
+- Retrained only the bounded Round 4 pool: 16 matched X/Y anchors plus 6 X held-out conditioning adapters and 6 Y oracle adapters.
+- Stronger recipe: LoRA r=16, alpha=32, targets=['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], epochs=3.0, train_per_task=1500, lr=0.0002, bf16, max_len=512.
+- Recomputed Top-K cosine selection from the new r=16/full-target X adapter space.
+
+## Experiment 1 — Main table
+
+Rows with `oracle - base_Y < 3 pp` are flagged as not usable for averages.
+
+| Domain | Task | base_Y | mean | global_ridge | pertensor_ridge | topk8_global_ridge | topk8_pertensor_ridge | pertensor_mlp | oracle | oracle_minus_base_pp | usable | gap_recovered |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| math | gsm_hard | 0.063 | 0.057 | 0.060 | 0.067 | 0.067 | 0.063 | 0.073 | 0.150 | 8.667 | True | 0.115 |
+| math | gsm8k_test_500 | 0.080 | 0.093 | 0.100 | 0.100 | 0.093 | 0.097 | 0.100 | 0.293 | 21.333 | True | 0.094 |
+| code | mbpp_test_held | 0.230 | 0.240 | 0.250 | 0.250 | 0.250 | 0.250 | 0.240 | 0.320 | 9.000 | True | 0.222 |
+| code | mbpp_plus | 0.217 | 0.213 | 0.280 | 0.270 | 0.270 | 0.267 | 0.210 | 0.450 | 23.333 | True | 0.271 |
+| science | arc_challenge | 0.716 | 0.732 | 0.736 | 0.729 | 0.736 | 0.729 | 0.739 | 0.722 | 0.669 | False | 5.000 |
+| science | openbookqa_test | 0.710 | 0.760 | 0.747 | 0.743 | 0.713 | 0.717 | 0.753 | 0.983 | 27.333 | True | 0.183 |
+
+
+## Headline
+
+- Best learned method minus mean baseline, averaged over usable held-outs: `0.0187`
+- Usable held-outs: `['gsm_hard', 'gsm8k_test_500', 'mbpp_test_held', 'mbpp_plus', 'openbookqa_test']`
+- Excluded held-outs: `['arc_challenge']`
+
+## Top-K selection log
+
+| Held-out | topk8_global_ridge | topk8_pertensor_ridge |
+|---|---|---|
+| gsm_hard | `['math_counting_easy', 'mbpp_sanitized', 'mmlu_high_school_physics', 'humaneval', 'multiarith', 'math_algebra_easy', 'mmlu_elementary_math', 'mmlu_high_school_biology']` | `['math_counting_easy', 'mbpp_sanitized', 'mmlu_high_school_physics', 'humaneval', 'multiarith', 'math_algebra_easy', 'mmlu_elementary_math', 'mmlu_high_school_biology']` |
+| gsm8k_test_500 | `['math_counting_easy', 'mbpp_sanitized', 'mmlu_high_school_physics', 'humaneval', 'multiarith', 'math_algebra_easy', 'mmlu_elementary_math', 'mmlu_high_school_biology']` | `['math_counting_easy', 'mbpp_sanitized', 'mmlu_high_school_physics', 'humaneval', 'multiarith', 'math_algebra_easy', 'mmlu_elementary_math', 'mmlu_high_school_biology']` |
+| mbpp_test_held | `['mbpp_sanitized', 'math_counting_easy', 'humaneval', 'mmlu_high_school_physics', 'multiarith', 'mmlu_high_school_biology', 'mmlu_elementary_math', 'math_algebra_easy']` | `['mbpp_sanitized', 'math_counting_easy', 'humaneval', 'mmlu_high_school_physics', 'multiarith', 'mmlu_high_school_biology', 'mmlu_elementary_math', 'math_algebra_easy']` |
+| mbpp_plus | `['mbpp_sanitized', 'humaneval', 'math_counting_easy', 'mmlu_high_school_physics', 'multiarith', 'mmlu_high_school_biology', 'mmlu_elementary_math', 'math_algebra_easy']` | `['mbpp_sanitized', 'humaneval', 'math_counting_easy', 'mmlu_high_school_physics', 'multiarith', 'mmlu_high_school_biology', 'mmlu_elementary_math', 'math_algebra_easy']` |
+| arc_challenge | `['mmlu_high_school_physics', 'mmlu_high_school_biology', 'mmlu_elementary_math', 'math_counting_easy', 'mbpp_sanitized', 'humaneval', 'multiarith', 'math_algebra_easy']` | `['mmlu_high_school_physics', 'mmlu_high_school_biology', 'mmlu_elementary_math', 'math_counting_easy', 'mbpp_sanitized', 'humaneval', 'multiarith', 'math_algebra_easy']` |
+| openbookqa_test | `['mmlu_high_school_physics', 'mmlu_high_school_biology', 'mbpp_sanitized', 'math_counting_easy', 'mmlu_elementary_math', 'humaneval', 'multiarith', 'math_algebra_easy']` | `['mmlu_high_school_physics', 'mmlu_high_school_biology', 'mbpp_sanitized', 'math_counting_easy', 'mmlu_elementary_math', 'humaneval', 'multiarith', 'math_algebra_easy']` |
+
+## Experiment 2 — Anchor-count + Top-K scaling
+
+![Anchor scaling](figures/exp2_anchor_scaling_r4.png)
+
+## Experiment 3 — Cross-domain transfer
+
+![Transfer heatmap](figures/exp3_transfer_heatmap_r4.png)
+
+| Held-out domain | Best anchor pool | Top-K actual selections (top-3) |
+|---|---|---|
+| math | science-only | `{'gsm_hard': ['mmlu_high_school_physics', 'mmlu_elementary_math', 'mmlu_high_school_biology'], 'gsm8k_test_500': ['mmlu_high_school_physics', 'mmlu_elementary_math', 'mmlu_high_school_biology']}` |
+| code | code-only | `{'mbpp_test_held': ['mbpp_sanitized', 'humaneval', 'mbpp'], 'mbpp_plus': ['mbpp_sanitized', 'humaneval', 'mbpp']}` |
+| science | science-only | `{'arc_challenge': ['mmlu_high_school_physics', 'mmlu_high_school_biology', 'mmlu_elementary_math'], 'openbookqa_test': ['mmlu_high_school_physics', 'mmlu_high_school_biology', 'mmlu_elementary_math']}` |
+
+## Honest failure modes
+
+- Excluded from averages: `arc_challenge` has oracle-base = 0.67 pp.
+- Code-task evaluation remains cheap answer-string/span matching, not sandboxed unit tests; code numbers are adapter-locality proxies, not pass@1.
+- Math uses numeric extraction/equality; formatting or non-numeric generations are counted wrong.
+- Top-K and ridge methods are exactly the prior mapping family; no new mapping method was added.

dataset_audit_round3.json

@@ -22,55 +22,11 @@
       },
       "ok": true
     },
-    "svamp": {
-      "domain": "math",
-      "kind": "math_num",
-      "dataset": "ChilleD/SVAMP",
-      "config": null,
-      "train_rows_sampled": 20,
-      "eval_rows_sampled": 10,
-      "labels": [],
-      "sample": {
-        "messages": [
-          {
-            "role": "user",
-            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Danny collects bottle caps. He found 63 bottle caps at the park while he threw away 51 old ones. Now he has 33 bottle caps in his collection. How many bottle caps did danny have at first?\n\nFinal answer:"
-          },
-          {
-            "role": "assistant",
-            "content": "21"
-          }
-        ]
-      },
-      "ok": true
-    },
-    "asdiv": {
-      "domain": "math",
-      "kind": "math_num",
-      "dataset": "EleutherAI/asdiv",
-      "config": null,
-      "train_rows_sampled": 20,
-      "eval_rows_sampled": 10,
-      "labels": [],
-      "sample": {
-        "messages": [
-          {
-            "role": "user",
-            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: A chef was making pies. He made 2 apple, 4 pecan and 7 pumpkin pies. How many pies did he make total?\n\nFinal answer:"
-          },
-          {
-            "role": "assistant",
-            "content": "13"
-          }
-        ]
-      },
-      "ok": true
-    },
-    "mawps": {
-      "domain": "math",
-      "kind": "math_num",
-      "dataset": "MU-NLPC/Calc-mawps",
-      "config": null,
+    "mbpp": {
+      "domain": "code",
+      "kind": "code",
+      "dataset": "google-research-datasets/mbpp",
+      "config": "sanitized",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [],
@@ -78,43 +34,52 @@
         "messages": [
           {
             "role": "user",
-            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Marilyn had 279 Beg. Daryl clasped some Beg. Now Marilyn has 59  Beg. How many did Daryl claspeds?\n\nFinal answer:"
+            "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to reverse words seperated by spaces in a given string.\n\nTests:\nassert reverse_words(\"python program\")==(\"program python\")\nassert reverse_words(\"java language\")==(\"language java\")\nassert reverse_words(\"indian man\")==(\"man indian\")\n\nCode:"
           },
           {
             "role": "assistant",
-            "content": "220"
+            "content": "def reverse_words(s):\n        return ' '.join(reversed(s.split()))"
           }
         ]
       },
       "ok": true
     },
-    "multiarith": {
-      "domain": "math",
-      "kind": "math_num",
-      "dataset": "ChilleD/MultiArith",
+    "sciq": {
+      "domain": "science",
+      "kind": "mcq",
+      "dataset": "allenai/sciq",
       "config": null,
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
-      "labels": [],
+      "labels": [
+        "A",
+        "B",
+        "C",
+        "D",
+        "E",
+        "F",
+        "G",
+        "H"
+      ],
       "sample": {
         "messages": [
           {
             "role": "user",
-            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: A florist had 11 roses. If she sold 2 of them and then later picked 32 more, how many roses would she have?\n\nFinal answer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhat kind of reasoning involves formulating generalizations inferred from careful observation and the analysis of a large amount of data?\nA. skepticism\nB. reflexive\nC. inductive\nD. deductive\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "41"
+            "content": "D"
           }
         ]
       },
       "ok": true
     },
-    "aqua_rat": {
-      "domain": "math",
+    "arc_easy": {
+      "domain": "science",
       "kind": "mcq",
-      "dataset": "deepmind/aqua_rat",
-      "config": "raw",
+      "dataset": "allenai/ai2_arc",
+      "config": "ARC-Easy",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [
@@ -131,86 +96,51 @@
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nThere are r red ball, b blue ball and w white ball in a bag. What is the ratio of the number of blue ball to the total no. of ball in terms of r, b and w.?\nA. r / (r + b + w)\nB. r * (r + b + w)\nC. (r + b + w)\nD. r / (r + b )\nE. r / (b + w)\nAnswer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhich of the following materials would best slow the transfer of heat?\nA. aluminum\nB. copper\nC. glass\nD. wood\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "A"
+            "content": "D"
           }
         ]
       },
       "ok": true
     },
-    "math_algebra_easy": {
-      "domain": "math",
-      "kind": "math_solution",
-      "dataset": "EleutherAI/hendrycks_math",
-      "config": "algebra",
+    "openbookqa": {
+      "domain": "science",
+      "kind": "mcq",
+      "dataset": "allenai/openbookqa",
+      "config": "main",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
-      "labels": [],
+      "labels": [
+        "A",
+        "B",
+        "C",
+        "D",
+        "E",
+        "F",
+        "G",
+        "H"
+      ],
       "sample": {
         "messages": [
           {
             "role": "user",
-            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Given that $2^x+ 2^x+ 2^x+ 2^x= 512$, what is the value of $x$?\n\nFinal answer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWith the addition of thrusters your forward momentum will\nA. stop\nB. increase\nC. decrease\nD. stall\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "7"
+            "content": "B"
           }
         ]
       },
       "ok": true
     },
-    "math_counting_easy": {
+    "svamp": {
       "domain": "math",
-      "kind": "math_solution",
-      "dataset": "EleutherAI/hendrycks_math",
-      "config": "counting_and_probability",
-      "train_rows_sampled": 20,
-      "eval_rows_sampled": 10,
-      "labels": [],
-      "sample": {
-        "messages": [
-          {
-            "role": "user",
-            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Alex has 10 different kinds of lunch meat and 9 different kinds of cheese. If he wants to make a sandwich with one kind of meat and two kinds of cheese, how many different sandwiches could he make?  (It does not matter in which order he chooses the two types of cheese.)\n\nFinal answer:"
-          },
-          {
-            "role": "assistant",
-            "content": "360"
-          }
-        ]
-      },
-      "ok": true
-    },
-    "mbpp": {
-      "domain": "code",
-      "kind": "code",
-      "dataset": "google-research-datasets/mbpp",
-      "config": "sanitized",
-      "train_rows_sampled": 20,
-      "eval_rows_sampled": 10,
-      "labels": [],
-      "sample": {
-        "messages": [
-          {
-            "role": "user",
-            "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to reverse words seperated by spaces in a given string.\n\nTests:\nassert reverse_words(\"python program\")==(\"program python\")\nassert reverse_words(\"java language\")==(\"language java\")\nassert reverse_words(\"indian man\")==(\"man indian\")\n\nCode:"
-          },
-          {
-            "role": "assistant",
-            "content": "def reverse_words(s):\n        return ' '.join(reversed(s.split()))"
-          }
-        ]
-      },
-      "ok": true
-    },
-    "humaneval": {
-      "domain": "code",
-      "kind": "code",
-      "dataset": "openai/openai_humaneval",
+      "kind": "math_num",
+      "dataset": "ChilleD/SVAMP",
       "config": null,
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
@@ -219,20 +149,20 @@
         "messages": [
           {
             "role": "user",
-            "content": "Complete the following Python function. Respond with only the completion code.\n\ndef below_threshold(l: list, t: int):\n    \"\"\"Return True if all numbers in the list l are below threshold t.\n    >>> below_threshold([1, 2, 4, 10], 100)\n    True\n    >>> below_threshold([1, 20, 4, 10], 5)\n    False\n    \"\"\""
+            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Danny collects bottle caps. He found 63 bottle caps at the park while he threw away 51 old ones. Now he has 33 bottle caps in his collection. How many bottle caps did danny have at first?\n\nFinal answer:"
           },
           {
             "role": "assistant",
-            "content": "for e in l:\n        if e >= t:\n            return False\n    return True"
+            "content": "21"
           }
         ]
       },
       "ok": true
     },
-    "codealpaca_mini": {
-      "domain": "code",
-      "kind": "code",
-      "dataset": "HuggingFaceH4/CodeAlpaca_20K",
+    "multiarith": {
+      "domain": "math",
+      "kind": "math_num",
+      "dataset": "ChilleD/MultiArith",
       "config": null,
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
@@ -241,54 +171,52 @@
         "messages": [
           {
             "role": "user",
-            "content": "Write code for the following instruction. Respond with only code.\n\nInstruction: Create a Sass mixin to create a \"toast\" style.\n\nCode:"
+            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: A florist had 11 roses. If she sold 2 of them and then later picked 32 more, how many roses would she have?\n\nFinal answer:"
           },
           {
             "role": "assistant",
-            "content": "@mixin toast-style {\n    background-color: #F2DEDE;\n    color: #A94442;\n    border: 1px solid #EBCCD1;\n    font-size-base: 1rem;\n    font-family: sans-serif;\n}"
+            "content": "41"
           }
         ]
       },
       "ok": true
     },
-    "mbpp_sanitized": {
-      "domain": "code",
-      "kind": "code",
-      "dataset": "google-research-datasets/mbpp",
-      "config": "sanitized",
+    "mmlu_high_school_biology": {
+      "domain": "science",
+      "kind": "mcq",
+      "dataset": "cais/mmlu",
+      "config": "high_school_biology",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
-      "labels": [],
+      "labels": [
+        "A",
+        "B",
+        "C",
+        "D",
+        "E",
+        "F",
+        "G",
+        "H"
+      ],
       "sample": {
         "messages": [
           {
             "role": "user",
-            "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to reverse words seperated by spaces in a given string.\n\nTests:\nassert reverse_words(\"python program\")==(\"program python\")\nassert reverse_words(\"java language\")==(\"language java\")\nassert reverse_words(\"indian man\")==(\"man indian\")\n\nCode:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhich of the following is (are) required when performing a polymerase chain reaction (P C R) on a DNA sample?\nA. Reverse transcriptase\nB. A vacuum chamber\nC. Primers complementary to certain portions of the DNA\nD. DNA ligase\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "def reverse_words(s):\n        return ' '.join(reversed(s.split()))"
+            "content": "C"
           }
         ]
       },
       "ok": true
     },
-    "apps_introductory": {
-      "domain": "code",
-      "kind": "code",
-      "dataset": "codeparrot/apps",
-      "config": "all",
-      "train_rows_sampled": 0,
-      "eval_rows_sampled": 0,
-      "labels": [],
-      "sample": null,
-      "ok": false
-    },
-    "conala_curated": {
-      "domain": "code",
-      "kind": "code",
-      "dataset": "codeparrot/conala-mined-curated",
-      "config": null,
+    "math_counting_easy": {
+      "domain": "math",
+      "kind": "math_solution",
+      "dataset": "EleutherAI/hendrycks_math",
+      "config": "counting_and_probability",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [],
@@ -296,31 +224,20 @@
         "messages": [
           {
             "role": "user",
-            "content": "Write a Python code snippet for this intent. Respond with only code.\n\nIntent: print all contacts in class all_contacts\n\nCode:"
+            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Alex has 10 different kinds of lunch meat and 9 different kinds of cheese. If he wants to make a sandwich with one kind of meat and two kinds of cheese, how many different sandwiches could he make?  (It does not matter in which order he chooses the two types of cheese.)\n\nFinal answer:"
           },
           {
             "role": "assistant",
-            "content": "all_contacts = []"
+            "content": "360"
           }
         ]
       },
       "ok": true
     },
-    "codecontests_easy": {
-      "domain": "code",
-      "kind": "code",
-      "dataset": "deepmind/code_contests",
-      "config": null,
-      "train_rows_sampled": 0,
-      "eval_rows_sampled": 10,
-      "labels": [],
-      "sample": null,
-      "ok": false
-    },
-    "livecodebench_easy": {
+    "humaneval": {
       "domain": "code",
       "kind": "code",
-      "dataset": "HuggingFaceH4/CodeAlpaca_20K",
+      "dataset": "openai/openai_humaneval",
       "config": null,
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
@@ -329,52 +246,21 @@
         "messages": [
           {
             "role": "user",
-            "content": "Write code for the following instruction. Respond with only code.\n\nInstruction: What is the syntax of creating a variable in Java?\n\nCode:"
-          },
-          {
-            "role": "assistant",
-            "content": "The syntax for creating a variable in Java is:\n<data_type> <variable_name> = <variable_value>;\n\nFor example:\n\nint myVar = 5;"
-          }
-        ]
-      },
-      "ok": true
-    },
-    "sciq": {
-      "domain": "science",
-      "kind": "mcq",
-      "dataset": "allenai/sciq",
-      "config": null,
-      "train_rows_sampled": 20,
-      "eval_rows_sampled": 10,
-      "labels": [
-        "A",
-        "B",
-        "C",
-        "D",
-        "E",
-        "F",
-        "G",
-        "H"
-      ],
-      "sample": {
-        "messages": [
-          {
-            "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhat kind of reasoning involves formulating generalizations inferred from careful observation and the analysis of a large amount of data?\nA. skepticism\nB. reflexive\nC. inductive\nD. deductive\nAnswer:"
+            "content": "Complete the following Python function. Respond with only the completion code.\n\ndef below_threshold(l: list, t: int):\n    \"\"\"Return True if all numbers in the list l are below threshold t.\n    >>> below_threshold([1, 2, 4, 10], 100)\n    True\n    >>> below_threshold([1, 20, 4, 10], 5)\n    False\n    \"\"\""
           },
           {
             "role": "assistant",
-            "content": "D"
+            "content": "for e in l:\n        if e >= t:\n            return False\n    return True"
           }
         ]
       },
       "ok": true
     },
-    "arc_easy": {
+    "mmlu_high_school_physics": {
       "domain": "science",
       "kind": "mcq",
-      "dataset": "allenai/ai2_arc",
-      "config": "ARC-Easy",
+      "dataset": "cais/mmlu",
+      "config": "high_school_physics",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [
@@ -391,52 +277,43 @@
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhich of the following materials would best slow the transfer of heat?\nA. aluminum\nB. copper\nC. glass\nD. wood\nAnswer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nThe mass of the Earth is 5.97 \u00d7 10^24 kg. The Moon, whose center is 3.84 \u00d7 10^8 m from the Earth\u2019s center, has mass 7.35 \u00d7 10^22 kg. Which of the following is the best estimate of the gravitational force of the Earth on the Moon?\nA. 10^39 N\nB. 10^29 N\nC. 10^19 N\nD. 10^9 N\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "D"
+            "content": "C"
           }
         ]
       },
       "ok": true
     },
-    "openbookqa": {
-      "domain": "science",
-      "kind": "mcq",
-      "dataset": "allenai/openbookqa",
-      "config": "main",
+    "mbpp_sanitized": {
+      "domain": "code",
+      "kind": "code",
+      "dataset": "google-research-datasets/mbpp",
+      "config": "sanitized",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
-      "labels": [
-        "A",
-        "B",
-        "C",
-        "D",
-        "E",
-        "F",
-        "G",
-        "H"
-      ],
+      "labels": [],
       "sample": {
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWith the addition of thrusters your forward momentum will\nA. stop\nB. increase\nC. decrease\nD. stall\nAnswer:"
+            "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to reverse words seperated by spaces in a given string.\n\nTests:\nassert reverse_words(\"python program\")==(\"program python\")\nassert reverse_words(\"java language\")==(\"language java\")\nassert reverse_words(\"indian man\")==(\"man indian\")\n\nCode:"
           },
           {
             "role": "assistant",
-            "content": "B"
+            "content": "def reverse_words(s):\n        return ' '.join(reversed(s.split()))"
           }
         ]
       },
       "ok": true
     },
-    "medmcqa_easy": {
+    "mmlu_elementary_math": {
       "domain": "science",
       "kind": "mcq",
-      "dataset": "openlifescienceai/medmcqa",
-      "config": null,
+      "dataset": "cais/mmlu",
+      "config": "elementary_mathematics",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [
@@ -453,52 +330,43 @@
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nApolipoprotein A-I (Apo A-I) is found in which of the following lipid components?\nA. LDL\nB. HDL\nC. VLDL\nD. Chylomicron\nAnswer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nFind the median of the set of data 13, 35, 26, 8, 24, 10, 22, 10, 32.\nA. 23\nB. 20\nC. 22\nD. 27\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "B"
+            "content": "C"
           }
         ]
       },
       "ok": true
     },
-    "mmlu_elementary_math": {
-      "domain": "science",
-      "kind": "mcq",
-      "dataset": "cais/mmlu",
-      "config": "elementary_mathematics",
+    "math_algebra_easy": {
+      "domain": "math",
+      "kind": "math_solution",
+      "dataset": "EleutherAI/hendrycks_math",
+      "config": "algebra",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
-      "labels": [
-        "A",
-        "B",
-        "C",
-        "D",
-        "E",
-        "F",
-        "G",
-        "H"
-      ],
+      "labels": [],
       "sample": {
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nFind the median of the set of data 13, 35, 26, 8, 24, 10, 22, 10, 32.\nA. 23\nB. 20\nC. 22\nD. 27\nAnswer:"
+            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Given that $2^x+ 2^x+ 2^x+ 2^x= 512$, what is the value of $x$?\n\nFinal answer:"
           },
           {
             "role": "assistant",
-            "content": "C"
+            "content": "7"
           }
         ]
       },
       "ok": true
     },
-    "mmlu_high_school_biology": {
-      "domain": "science",
+    "aqua_rat": {
+      "domain": "math",
       "kind": "mcq",
-      "dataset": "cais/mmlu",
-      "config": "high_school_biology",
+      "dataset": "deepmind/aqua_rat",
+      "config": "raw",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [
@@ -515,21 +383,21 @@
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhich of the following is (are) required when performing a polymerase chain reaction (P C R) on a DNA sample?\nA. Reverse transcriptase\nB. A vacuum chamber\nC. Primers complementary to certain portions of the DNA\nD. DNA ligase\nAnswer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nThere are r red ball, b blue ball and w white ball in a bag. What is the ratio of the number of blue ball to the total no. of ball in terms of r, b and w.?\nA. r / (r + b + w)\nB. r * (r + b + w)\nC. (r + b + w)\nD. r / (r + b )\nE. r / (b + w)\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "C"
+            "content": "A"
           }
         ]
       },
       "ok": true
     },
-    "mmlu_high_school_physics": {
+    "medmcqa_easy": {
       "domain": "science",
       "kind": "mcq",
-      "dataset": "cais/mmlu",
-      "config": "high_school_physics",
+      "dataset": "openlifescienceai/medmcqa",
+      "config": null,
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [
@@ -546,37 +414,11 @@
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nThe mass of the Earth is 5.97 \u00d7 10^24 kg. The Moon, whose center is 3.84 \u00d7 10^8 m from the Earth\u2019s center, has mass 7.35 \u00d7 10^22 kg. Which of the following is the best estimate of the gravitational force of the Earth on the Moon?\nA. 10^39 N\nB. 10^29 N\nC. 10^19 N\nD. 10^9 N\nAnswer:"
-          },
-          {
-            "role": "assistant",
-            "content": "C"
-          }
-        ]
-      },
-      "ok": true
-    },
-    "pubmedqa_pqal": {
-      "domain": "science",
-      "kind": "short",
-      "dataset": "qiaojin/PubMedQA",
-      "config": "pqa_labeled",
-      "train_rows_sampled": 20,
-      "eval_rows_sampled": 10,
-      "labels": [
-        "yes",
-        "no",
-        "maybe"
-      ],
-      "sample": {
-        "messages": [
-          {
-            "role": "user",
-            "content": "Answer the biomedical question with only one of: yes, no, maybe.\n\nContext: To assess whether eligibility to an adjuvant chemotherapy protocol in itself represents a good prognostic factor after radical cystectomy for bladder cancer. Between April 1984 and May 1989, our institution entered 35 patients with invasive bladder cancer into the Swiss Group for Clinical and Epidemiological Cancer Research (SAKK) study 09/84. They were randomly assigned to either observation or three postoperative courses of cisplatin monotherapy after cystectomy. This study had a negative result. The outcome of these 35 patients (protocol group) was compared with an age- and tumor-stage-matched cohort (matched group; n = 35) who also underwent cystectomy during the same period, but were not entered into the SAKK study, as well as the remaining 57 patients treated during the study period for the same indication (remaining group). Median overall survival decreased from 76.3 months in the protocol group to 52.1 months in the matched group and to 20.3 months in the remaining group. The respective times of median recurrence-free survival were 67.2, 16.0, and 9.4 months. Tumor progression occurred in 46% of the protocol group compared with 69% in the matched group and 65% in the remaining group (P<.05). Cancer-related death was noted in 40% of the protocol group, 57% in the matched group, and 56% in the remaining group.\n\nQuestion: Is eligibility for a chemotherapy protocol a good prognostic factor for invasive bladder cancer after radical cystectomy?\n\nAnswer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nApolipoprotein A-I (Apo A-I) is found in which of the following lipid components?\nA. LDL\nB. HDL\nC. VLDL\nD. Chylomicron\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "yes"
+            "content": "B"
           }
         ]
       },
@@ -604,11 +446,11 @@
       },
       "ok": true
     },
-    "math_algebra_medium": {
+    "gsm8k_test_500": {
       "domain": "math",
-      "kind": "math_solution",
-      "dataset": "EleutherAI/hendrycks_math",
-      "config": "algebra",
+      "kind": "math_num",
+      "dataset": "openai/gsm8k",
+      "config": "main",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [],
@@ -616,21 +458,21 @@
         "messages": [
           {
             "role": "user",
-            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: A super ball is dropped from 100 feet and rebounds half the distance it falls each time it bounces. How many feet will the ball have traveled when it hits the ground the fourth time?\n\nFinal answer:"
+            "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Benny saw a 10-foot shark with 2 6-inch remoras attached to it. What percentage of the shark's body length is the combined length of the remoras?\n\nFinal answer:"
           },
           {
             "role": "assistant",
-            "content": "275"
+            "content": "10"
           }
         ]
       },
       "ok": true
     },
-    "humaneval_plus": {
+    "mbpp_test_held": {
       "domain": "code",
       "kind": "code",
-      "dataset": "evalplus/humanevalplus",
-      "config": null,
+      "dataset": "google-research-datasets/mbpp",
+      "config": "sanitized",
       "train_rows_sampled": 20,
       "eval_rows_sampled": 10,
       "labels": [],
@@ -638,11 +480,11 @@
         "messages": [
           {
             "role": "user",
-            "content": "Complete the following Python function. Respond with only the completion code.\n\ndef below_threshold(l: list, t: int):\n    \"\"\"Return True if all numbers in the list l are below threshold t.\n    >>> below_threshold([1, 2, 4, 10], 100)\n    True\n    >>> below_threshold([1, 20, 4, 10], 5)\n    False\n    \"\"\""
+            "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to convert a given tuple of positive integers into a single integer.\n\nTests:\nassert tuple_to_int((1,2,3))==123\nassert tuple_to_int((4,5,6))==456\nassert tuple_to_int((5,6,7))==567\n\nCode:"
           },
           {
             "role": "assistant",
-            "content": "return all(x < t for x in l)"
+            "content": "def tuple_to_int(nums):\n    result = int(''.join(map(str,nums)))\n    return result"
           }
         ]
       },
@@ -701,13 +543,13 @@
       },
       "ok": true
     },
-    "mmlu_college_chemistry": {
+    "openbookqa_test": {
       "domain": "science",
       "kind": "mcq",
-      "dataset": "cais/mmlu",
-      "config": "college_chemistry",
+      "dataset": "allenai/openbookqa",
+      "config": "main",
       "train_rows_sampled": 20,
-      "eval_rows_sampled": 8,
+      "eval_rows_sampled": 10,
       "labels": [
         "A",
         "B",
@@ -722,54 +564,45 @@
         "messages": [
           {
             "role": "user",
-            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nOf the following ionic substances, which has the greatest lattice enthalpy?\nA. MgO\nB. MgS\nC. NaF\nD. NaCl\nAnswer:"
+            "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nLive birth is exemplified in\nA. snakes slithering out of eggs\nB. a calf emerging from a mother giraffe\nC. owlets pecking out of their encasement\nD. sea turtles emerging from their shells\nAnswer:"
           },
           {
             "role": "assistant",
-            "content": "A"
+            "content": "B"
           }
         ]
       },
       "ok": true
     }
   },
-  "dropped": [
-    "apps_introductory",
-    "codecontests_easy"
-  ],
+  "dropped": [],
   "kept": [
     "gsm8k",
-    "svamp",
-    "asdiv",
-    "mawps",
-    "multiarith",
-    "aqua_rat",
-    "math_algebra_easy",
-    "math_counting_easy",
     "mbpp",
-    "humaneval",
-    "codealpaca_mini",
-    "mbpp_sanitized",
-    "conala_curated",
-    "livecodebench_easy",
     "sciq",
     "arc_easy",
     "openbookqa",
-    "medmcqa_easy",
-    "mmlu_elementary_math",
+    "svamp",
+    "multiarith",
     "mmlu_high_school_biology",
+    "math_counting_easy",
+    "humaneval",
     "mmlu_high_school_physics",
-    "pubmedqa_pqal",
+    "mbpp_sanitized",
+    "mmlu_elementary_math",
+    "math_algebra_easy",
+    "aqua_rat",
+    "medmcqa_easy",
     "gsm_hard",
-    "math_algebra_medium",
-    "humaneval_plus",
+    "gsm8k_test_500",
+    "mbpp_test_held",
     "mbpp_plus",
     "arc_challenge",
-    "mmlu_college_chemistry"
+    "openbookqa_test"
   ],
   "anchor_domain_counts": {
-    "math": 8,
-    "code": 6,
-    "science": 8
+    "math": 6,
+    "code": 3,
+    "science": 7
   }
 }

experiment1_round4.json

@@ -0,0 +1,482 @@
+{
+  "tasks": {
+    "gsm_hard": {
+      "domain": "math",
+      "selected": {
+        "topk8_global_ridge": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "mmlu_high_school_physics",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ],
+        "topk8_pertensor_ridge": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "mmlu_high_school_physics",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ]
+      },
+      "metrics": {
+        "base_Y": 0.06333333333333334,
+        "oracle": 0.15,
+        "mean": 0.056666666666666664,
+        "mean__cos": 0.8538880348205566,
+        "global_ridge": 0.06,
+        "global_ridge__cos": 0.9063830375671387,
+        "pertensor_ridge": 0.06666666666666667,
+        "pertensor_ridge__cos": 0.904435396194458,
+        "pertensor_pca": 0.06666666666666667,
+        "pertensor_pca__cos": 0.9048259854316711,
+        "pertensor_mlp": 0.07333333333333333,
+        "pertensor_mlp__cos": 0.90220707654953,
+        "procrustes": 0.07,
+        "procrustes__cos": 0.89919513463974,
+        "topk8_global_ridge": 0.06666666666666667,
+        "topk8_global_ridge__cos": 0.9051854014396667,
+        "topk8_pertensor_ridge": 0.06333333333333334,
+        "topk8_pertensor_ridge__cos": 0.903511106967926
+      },
+      "main_row": {
+        "Domain": "math",
+        "Task": "gsm_hard",
+        "base_Y": 0.06333333333333334,
+        "mean": 0.056666666666666664,
+        "global_ridge": 0.06,
+        "pertensor_ridge": 0.06666666666666667,
+        "topk8_global_ridge": 0.06666666666666667,
+        "topk8_pertensor_ridge": 0.06333333333333334,
+        "pertensor_mlp": 0.07333333333333333,
+        "oracle": 0.15,
+        "oracle_minus_base_pp": 8.666666666666666,
+        "usable": true,
+        "gap_recovered": 0.11538461538461534
+      }
+    },
+    "gsm8k_test_500": {
+      "domain": "math",
+      "selected": {
+        "topk8_global_ridge": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "mmlu_high_school_physics",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ],
+        "topk8_pertensor_ridge": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "mmlu_high_school_physics",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ]
+      },
+      "metrics": {
+        "base_Y": 0.08,
+        "oracle": 0.29333333333333333,
+        "mean": 0.09333333333333334,
+        "mean__cos": 0.9122087955474854,
+        "global_ridge": 0.1,
+        "global_ridge__cos": 0.9690855145454407,
+        "pertensor_ridge": 0.1,
+        "pertensor_ridge__cos": 0.9674875736236572,
+        "pertensor_pca": 0.1,
+        "pertensor_pca__cos": 0.9675261974334717,
+        "pertensor_mlp": 0.1,
+        "pertensor_mlp__cos": 0.9635453224182129,
+        "procrustes": 0.09666666666666666,
+        "procrustes__cos": 0.9608618021011353,
+        "topk8_global_ridge": 0.09333333333333334,
+        "topk8_global_ridge__cos": 0.9682586789131165,
+        "topk8_pertensor_ridge": 0.09666666666666666,
+        "topk8_pertensor_ridge__cos": 0.9668623805046082
+      },
+      "main_row": {
+        "Domain": "math",
+        "Task": "gsm8k_test_500",
+        "base_Y": 0.08,
+        "mean": 0.09333333333333334,
+        "global_ridge": 0.1,
+        "pertensor_ridge": 0.1,
+        "topk8_global_ridge": 0.09333333333333334,
+        "topk8_pertensor_ridge": 0.09666666666666666,
+        "pertensor_mlp": 0.1,
+        "oracle": 0.29333333333333333,
+        "oracle_minus_base_pp": 21.333333333333332,
+        "usable": true,
+        "gap_recovered": 0.09375000000000003
+      }
+    },
+    "mbpp_test_held": {
+      "domain": "code",
+      "selected": {
+        "topk8_global_ridge": [
+          "mbpp_sanitized",
+          "math_counting_easy",
+          "humaneval",
+          "mmlu_high_school_physics",
+          "multiarith",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math",
+          "math_algebra_easy"
+        ],
+        "topk8_pertensor_ridge": [
+          "mbpp_sanitized",
+          "math_counting_easy",
+          "humaneval",
+          "mmlu_high_school_physics",
+          "multiarith",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math",
+          "math_algebra_easy"
+        ]
+      },
+      "metrics": {
+        "base_Y": 0.23,
+        "oracle": 0.32,
+        "mean": 0.24,
+        "mean__cos": 0.9324136972427368,
+        "global_ridge": 0.25,
+        "global_ridge__cos": 0.9998323917388916,
+        "pertensor_ridge": 0.25,
+        "pertensor_ridge__cos": 0.9997650980949402,
+        "pertensor_pca": 0.25,
+        "pertensor_pca__cos": 0.9902871251106262,
+        "pertensor_mlp": 0.24,
+        "pertensor_mlp__cos": 0.9861795902252197,
+        "procrustes": 0.24,
+        "procrustes__cos": 0.9862682819366455,
+        "topk8_global_ridge": 0.25,
+        "topk8_global_ridge__cos": 0.9998441338539124,
+        "topk8_pertensor_ridge": 0.25,
+        "topk8_pertensor_ridge__cos": 0.9995993375778198
+      },
+      "main_row": {
+        "Domain": "code",
+        "Task": "mbpp_test_held",
+        "base_Y": 0.23,
+        "mean": 0.24,
+        "global_ridge": 0.25,
+        "pertensor_ridge": 0.25,
+        "topk8_global_ridge": 0.25,
+        "topk8_pertensor_ridge": 0.25,
+        "pertensor_mlp": 0.24,
+        "oracle": 0.32,
+        "oracle_minus_base_pp": 9.0,
+        "usable": true,
+        "gap_recovered": 0.22222222222222213
+      }
+    },
+    "mbpp_plus": {
+      "domain": "code",
+      "selected": {
+        "topk8_global_ridge": [
+          "mbpp_sanitized",
+          "humaneval",
+          "math_counting_easy",
+          "mmlu_high_school_physics",
+          "multiarith",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math",
+          "math_algebra_easy"
+        ],
+        "topk8_pertensor_ridge": [
+          "mbpp_sanitized",
+          "humaneval",
+          "math_counting_easy",
+          "mmlu_high_school_physics",
+          "multiarith",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math",
+          "math_algebra_easy"
+        ]
+      },
+      "metrics": {
+        "base_Y": 0.21666666666666667,
+        "oracle": 0.45,
+        "mean": 0.21333333333333335,
+        "mean__cos": 0.9124462008476257,
+        "global_ridge": 0.28,
+        "global_ridge__cos": 0.9848983287811279,
+        "pertensor_ridge": 0.27,
+        "pertensor_ridge__cos": 0.9846793413162231,
+        "pertensor_pca": 0.21,
+        "pertensor_pca__cos": 0.9684830904006958,
+        "pertensor_mlp": 0.21,
+        "pertensor_mlp__cos": 0.9642786383628845,
+        "procrustes": 0.22666666666666666,
+        "procrustes__cos": 0.9647142887115479,
+        "topk8_global_ridge": 0.27,
+        "topk8_global_ridge__cos": 0.9848766922950745,
+        "topk8_pertensor_ridge": 0.26666666666666666,
+        "topk8_pertensor_ridge__cos": 0.9846369624137878
+      },
+      "main_row": {
+        "Domain": "code",
+        "Task": "mbpp_plus",
+        "base_Y": 0.21666666666666667,
+        "mean": 0.21333333333333335,
+        "global_ridge": 0.28,
+        "pertensor_ridge": 0.27,
+        "topk8_global_ridge": 0.27,
+        "topk8_pertensor_ridge": 0.26666666666666666,
+        "pertensor_mlp": 0.21,
+        "oracle": 0.45,
+        "oracle_minus_base_pp": 23.333333333333332,
+        "usable": true,
+        "gap_recovered": 0.2714285714285715
+      }
+    },
+    "arc_challenge": {
+      "domain": "science",
+      "selected": {
+        "topk8_global_ridge": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math",
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy"
+        ],
+        "topk8_pertensor_ridge": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math",
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy"
+        ]
+      },
+      "metrics": {
+        "base_Y": 0.7157190635451505,
+        "oracle": 0.7224080267558528,
+        "mean": 0.7324414715719063,
+        "mean__cos": 0.8707941174507141,
+        "global_ridge": 0.7357859531772575,
+        "global_ridge__cos": 0.9247814416885376,
+        "pertensor_ridge": 0.7290969899665551,
+        "pertensor_ridge__cos": 0.9244466423988342,
+        "pertensor_pca": 0.7357859531772575,
+        "pertensor_pca__cos": 0.9224098920822144,
+        "pertensor_mlp": 0.7391304347826086,
+        "pertensor_mlp__cos": 0.920215368270874,
+        "procrustes": 0.7491638795986622,
+        "procrustes__cos": 0.9195225238800049,
+        "topk8_global_ridge": 0.7357859531772575,
+        "topk8_global_ridge__cos": 0.9233748316764832,
+        "topk8_pertensor_ridge": 0.7290969899665551,
+        "topk8_pertensor_ridge__cos": 0.9231237173080444
+      },
+      "main_row": {
+        "Domain": "science",
+        "Task": "arc_challenge",
+        "base_Y": 0.7157190635451505,
+        "mean": 0.7324414715719063,
+        "global_ridge": 0.7357859531772575,
+        "pertensor_ridge": 0.7290969899665551,
+        "topk8_global_ridge": 0.7357859531772575,
+        "topk8_pertensor_ridge": 0.7290969899665551,
+        "pertensor_mlp": 0.7391304347826086,
+        "oracle": 0.7224080267558528,
+        "oracle_minus_base_pp": 0.6688963210702337,
+        "usable": false,
+        "gap_recovered": 5.0
+      }
+    },
+    "openbookqa_test": {
+      "domain": "science",
+      "selected": {
+        "topk8_global_ridge": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mbpp_sanitized",
+          "math_counting_easy",
+          "mmlu_elementary_math",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy"
+        ],
+        "topk8_pertensor_ridge": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mbpp_sanitized",
+          "math_counting_easy",
+          "mmlu_elementary_math",
+          "humaneval",
+          "multiarith",
+          "math_algebra_easy"
+        ]
+      },
+      "metrics": {
+        "base_Y": 0.71,
+        "oracle": 0.9833333333333333,
+        "mean": 0.76,
+        "mean__cos": 0.9152830243110657,
+        "global_ridge": 0.7466666666666667,
+        "global_ridge__cos": 0.9711560010910034,
+        "pertensor_ridge": 0.7433333333333333,
+        "pertensor_ridge__cos": 0.9709675312042236,
+        "pertensor_pca": 0.76,
+        "pertensor_pca__cos": 0.9696671366691589,
+        "pertensor_mlp": 0.7533333333333333,
+        "pertensor_mlp__cos": 0.9671377539634705,
+        "procrustes": 0.7366666666666667,
+        "procrustes__cos": 0.9668078422546387,
+        "topk8_global_ridge": 0.7133333333333334,
+        "topk8_global_ridge__cos": 0.969284176826477,
+        "topk8_pertensor_ridge": 0.7166666666666667,
+        "topk8_pertensor_ridge__cos": 0.9690907597541809
+      },
+      "main_row": {
+        "Domain": "science",
+        "Task": "openbookqa_test",
+        "base_Y": 0.71,
+        "mean": 0.76,
+        "global_ridge": 0.7466666666666667,
+        "pertensor_ridge": 0.7433333333333333,
+        "topk8_global_ridge": 0.7133333333333334,
+        "topk8_pertensor_ridge": 0.7166666666666667,
+        "pertensor_mlp": 0.7533333333333333,
+        "oracle": 0.9833333333333333,
+        "oracle_minus_base_pp": 27.333333333333332,
+        "usable": true,
+        "gap_recovered": 0.18292682926829285
+      }
+    }
+  },
+  "main_table": [
+    {
+      "Domain": "math",
+      "Task": "gsm_hard",
+      "base_Y": 0.06333333333333334,
+      "mean": 0.056666666666666664,
+      "global_ridge": 0.06,
+      "pertensor_ridge": 0.06666666666666667,
+      "topk8_global_ridge": 0.06666666666666667,
+      "topk8_pertensor_ridge": 0.06333333333333334,
+      "pertensor_mlp": 0.07333333333333333,
+      "oracle": 0.15,
+      "oracle_minus_base_pp": 8.666666666666666,
+      "usable": true,
+      "gap_recovered": 0.11538461538461534
+    },
+    {
+      "Domain": "math",
+      "Task": "gsm8k_test_500",
+      "base_Y": 0.08,
+      "mean": 0.09333333333333334,
+      "global_ridge": 0.1,
+      "pertensor_ridge": 0.1,
+      "topk8_global_ridge": 0.09333333333333334,
+      "topk8_pertensor_ridge": 0.09666666666666666,
+      "pertensor_mlp": 0.1,
+      "oracle": 0.29333333333333333,
+      "oracle_minus_base_pp": 21.333333333333332,
+      "usable": true,
+      "gap_recovered": 0.09375000000000003
+    },
+    {
+      "Domain": "code",
+      "Task": "mbpp_test_held",
+      "base_Y": 0.23,
+      "mean": 0.24,
+      "global_ridge": 0.25,
+      "pertensor_ridge": 0.25,
+      "topk8_global_ridge": 0.25,
+      "topk8_pertensor_ridge": 0.25,
+      "pertensor_mlp": 0.24,
+      "oracle": 0.32,
+      "oracle_minus_base_pp": 9.0,
+      "usable": true,
+      "gap_recovered": 0.22222222222222213
+    },
+    {
+      "Domain": "code",
+      "Task": "mbpp_plus",
+      "base_Y": 0.21666666666666667,
+      "mean": 0.21333333333333335,
+      "global_ridge": 0.28,
+      "pertensor_ridge": 0.27,
+      "topk8_global_ridge": 0.27,
+      "topk8_pertensor_ridge": 0.26666666666666666,
+      "pertensor_mlp": 0.21,
+      "oracle": 0.45,
+      "oracle_minus_base_pp": 23.333333333333332,
+      "usable": true,
+      "gap_recovered": 0.2714285714285715
+    },
+    {
+      "Domain": "science",
+      "Task": "arc_challenge",
+      "base_Y": 0.7157190635451505,
+      "mean": 0.7324414715719063,
+      "global_ridge": 0.7357859531772575,
+      "pertensor_ridge": 0.7290969899665551,
+      "topk8_global_ridge": 0.7357859531772575,
+      "topk8_pertensor_ridge": 0.7290969899665551,
+      "pertensor_mlp": 0.7391304347826086,
+      "oracle": 0.7224080267558528,
+      "oracle_minus_base_pp": 0.6688963210702337,
+      "usable": false,
+      "gap_recovered": 5.0
+    },
+    {
+      "Domain": "science",
+      "Task": "openbookqa_test",
+      "base_Y": 0.71,
+      "mean": 0.76,
+      "global_ridge": 0.7466666666666667,
+      "pertensor_ridge": 0.7433333333333333,
+      "topk8_global_ridge": 0.7133333333333334,
+      "topk8_pertensor_ridge": 0.7166666666666667,
+      "pertensor_mlp": 0.7533333333333333,
+      "oracle": 0.9833333333333333,
+      "oracle_minus_base_pp": 27.333333333333332,
+      "usable": true,
+      "gap_recovered": 0.18292682926829285
+    }
+  ],
+  "anchor_names": [
+    "gsm8k",
+    "svamp",
+    "multiarith",
+    "aqua_rat",
+    "math_algebra_easy",
+    "math_counting_easy",
+    "mbpp",
+    "humaneval",
+    "mbpp_sanitized",
+    "sciq",
+    "arc_easy",
+    "openbookqa",
+    "medmcqa_easy",
+    "mmlu_elementary_math",
+    "mmlu_high_school_biology",
+    "mmlu_high_school_physics"
+  ],
+  "heldout_names": [
+    "gsm_hard",
+    "gsm8k_test_500",
+    "mbpp_test_held",
+    "mbpp_plus",
+    "arc_challenge",
+    "openbookqa_test"
+  ]
+}

experiment2_round4.json

@@ -0,0 +1,212 @@
+{
+  "curves": {
+    "N5_global_ridge": {
+      "math": -0.010817307692307696,
+      "code": 0.026984126984126878,
+      "science": 1.5304878048780488
+    },
+    "N12_global_ridge": {
+      "math": -0.03425480769230772,
+      "code": 0.2539682539682539,
+      "science": 2.073170731707317
+    },
+    "N12_topk8_global_ridge": {
+      "math": -0.010817307692307696,
+      "code": 0.2738095238095238,
+      "science": 1.3353658536585367
+    },
+    "N12_topk12_global_ridge": {
+      "math": -0.018629807692307716,
+      "code": 0.23253968253968244,
+      "science": 1.829268292682927
+    },
+    "N16_global_ridge": {
+      "math": 0.027644230769230737,
+      "code": 0.24682539682539684,
+      "science": 1.5670731707317074
+    },
+    "N16_topk8_global_ridge": {
+      "math": 0.050480769230769204,
+      "code": 0.22539682539682537,
+      "science": 1.50609756097561
+    },
+    "N16_topk12_global_ridge": {
+      "math": 0.03906249999999999,
+      "code": 0.2396825396825396,
+      "science": 2.3170731707317076
+    }
+  },
+  "details": {
+    "N5_anchors": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy"
+    ],
+    "gsm_hard": {
+      "N5_global_ridge": {
+        "acc": 0.05333333333333334,
+        "gap_recovered": -0.11538461538461542
+      },
+      "N12_global_ridge": {
+        "acc": 0.05333333333333334,
+        "gap_recovered": -0.11538461538461542
+      },
+      "N12_topk8_global_ridge": {
+        "acc": 0.05333333333333334,
+        "gap_recovered": -0.11538461538461542
+      },
+      "N12_topk12_global_ridge": {
+        "acc": 0.05333333333333334,
+        "gap_recovered": -0.11538461538461542
+      },
+      "N16_topk12_global_ridge": {
+        "acc": 0.06333333333333334,
+        "gap_recovered": 0.0
+      }
+    },
+    "gsm8k_test_500": {
+      "N5_global_ridge": {
+        "acc": 0.1,
+        "gap_recovered": 0.09375000000000003
+      },
+      "N12_global_ridge": {
+        "acc": 0.09,
+        "gap_recovered": 0.04687499999999998
+      },
+      "N12_topk8_global_ridge": {
+        "acc": 0.1,
+        "gap_recovered": 0.09375000000000003
+      },
+      "N12_topk12_global_ridge": {
+        "acc": 0.09666666666666666,
+        "gap_recovered": 0.07812499999999999
+      },
+      "N16_topk12_global_ridge": {
+        "acc": 0.09666666666666666,
+        "gap_recovered": 0.07812499999999999
+      }
+    },
+    "mbpp_test_held": {
+      "N5_global_ridge": {
+        "acc": 0.24,
+        "gap_recovered": 0.11111111111111091
+      },
+      "N12_global_ridge": {
+        "acc": 0.25,
+        "gap_recovered": 0.22222222222222213
+      },
+      "N12_topk8_global_ridge": {
+        "acc": 0.26,
+        "gap_recovered": 0.3333333333333333
+      },
+      "N12_topk12_global_ridge": {
+        "acc": 0.25,
+        "gap_recovered": 0.22222222222222213
+      },
+      "N16_topk12_global_ridge": {
+        "acc": 0.25,
+        "gap_recovered": 0.22222222222222213
+      }
+    },
+    "mbpp_plus": {
+      "N5_global_ridge": {
+        "acc": 0.20333333333333334,
+        "gap_recovered": -0.057142857142857155
+      },
+      "N12_global_ridge": {
+        "acc": 0.2833333333333333,
+        "gap_recovered": 0.28571428571428564
+      },
+      "N12_topk8_global_ridge": {
+        "acc": 0.26666666666666666,
+        "gap_recovered": 0.21428571428571425
+      },
+      "N12_topk12_global_ridge": {
+        "acc": 0.2733333333333333,
+        "gap_recovered": 0.24285714285714274
+      },
+      "N16_topk12_global_ridge": {
+        "acc": 0.27666666666666667,
+        "gap_recovered": 0.2571428571428571
+      }
+    },
+    "arc_challenge": {
+      "N5_global_ridge": {
+        "acc": 0.7357859531772575,
+        "gap_recovered": 3.0
+      },
+      "N12_global_ridge": {
+        "acc": 0.7424749163879598,
+        "gap_recovered": 4.0
+      },
+      "N12_topk8_global_ridge": {
+        "acc": 0.7324414715719063,
+        "gap_recovered": 2.5
+      },
+      "N12_topk12_global_ridge": {
+        "acc": 0.7391304347826086,
+        "gap_recovered": 3.5
+      },
+      "N16_topk12_global_ridge": {
+        "acc": 0.745819397993311,
+        "gap_recovered": 4.5
+      }
+    },
+    "openbookqa_test": {
+      "N5_global_ridge": {
+        "acc": 0.7266666666666667,
+        "gap_recovered": 0.060975609756097754
+      },
+      "N12_global_ridge": {
+        "acc": 0.75,
+        "gap_recovered": 0.14634146341463428
+      },
+      "N12_topk8_global_ridge": {
+        "acc": 0.7566666666666667,
+        "gap_recovered": 0.17073170731707346
+      },
+      "N12_topk12_global_ridge": {
+        "acc": 0.7533333333333333,
+        "gap_recovered": 0.15853658536585366
+      },
+      "N16_topk12_global_ridge": {
+        "acc": 0.7466666666666667,
+        "gap_recovered": 0.1341463414634149
+      }
+    },
+    "N12_anchors": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy",
+      "math_counting_easy",
+      "mbpp",
+      "humaneval",
+      "mbpp_sanitized",
+      "sciq",
+      "arc_easy",
+      "openbookqa"
+    ],
+    "N16_anchors": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy",
+      "math_counting_easy",
+      "mbpp",
+      "humaneval",
+      "mbpp_sanitized",
+      "sciq",
+      "arc_easy",
+      "openbookqa",
+      "medmcqa_easy",
+      "mmlu_elementary_math",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_physics"
+    ]
+  }
+}

experiment3_round4.json

@@ -0,0 +1,317 @@
+{
+  "pools": {
+    "math-only": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy",
+      "math_counting_easy"
+    ],
+    "code-only": [
+      "mbpp",
+      "humaneval",
+      "mbpp_sanitized"
+    ],
+    "science-only": [
+      "sciq",
+      "arc_easy",
+      "openbookqa",
+      "medmcqa_easy",
+      "mmlu_elementary_math",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_physics"
+    ],
+    "math+code": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy",
+      "math_counting_easy",
+      "mbpp",
+      "humaneval",
+      "mbpp_sanitized"
+    ],
+    "all": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy",
+      "math_counting_easy",
+      "mbpp",
+      "humaneval",
+      "mbpp_sanitized",
+      "sciq",
+      "arc_easy",
+      "openbookqa",
+      "medmcqa_easy",
+      "mmlu_elementary_math",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_physics"
+    ]
+  },
+  "heatmap": {
+    "math-only": {
+      "gsm_hard": -0.038461538461538554,
+      "gsm8k_test_500": 0.07812499999999999,
+      "mbpp_test_held": 0.0,
+      "mbpp_plus": -0.04285714285714289,
+      "arc_challenge": 2.0,
+      "openbookqa_test": 0.10975609756097571
+    },
+    "code-only": {
+      "gsm_hard": -0.15384615384615388,
+      "gsm8k_test_500": -0.03125000000000001,
+      "mbpp_test_held": 0.44444444444444453,
+      "mbpp_plus": 0.24285714285714274,
+      "arc_challenge": -1.5,
+      "openbookqa_test": 0.012195121951219795
+    },
+    "science-only": {
+      "gsm_hard": -0.038461538461538554,
+      "gsm8k_test_500": 0.171875,
+      "mbpp_test_held": 0.11111111111111091,
+      "mbpp_plus": -0.01428571428571426,
+      "arc_challenge": 3.5,
+      "openbookqa_test": 0.10975609756097571
+    },
+    "math+code": {
+      "gsm_hard": -0.15384615384615388,
+      "gsm8k_test_500": 0.09375000000000003,
+      "mbpp_test_held": 0.22222222222222213,
+      "mbpp_plus": 0.21428571428571425,
+      "arc_challenge": 0.5,
+      "openbookqa_test": 0.09756097560975632
+    },
+    "all": {
+      "gsm_hard": 0.038461538461538394,
+      "gsm8k_test_500": 0.06250000000000001,
+      "mbpp_test_held": 0.22222222222222213,
+      "mbpp_plus": 0.22857142857142862,
+      "arc_challenge": 3.0,
+      "openbookqa_test": 0.012195121951219795
+    }
+  },
+  "selected_top3": {
+    "math-only": {
+      "gsm_hard": [
+        "math_counting_easy",
+        "multiarith",
+        "math_algebra_easy"
+      ],
+      "gsm8k_test_500": [
+        "math_counting_easy",
+        "multiarith",
+        "math_algebra_easy"
+      ],
+      "mbpp_test_held": [
+        "math_counting_easy",
+        "multiarith",
+        "math_algebra_easy"
+      ],
+      "mbpp_plus": [
+        "math_counting_easy",
+        "multiarith",
+        "math_algebra_easy"
+      ],
+      "arc_challenge": [
+        "math_counting_easy",
+        "multiarith",
+        "math_algebra_easy"
+      ],
+      "openbookqa_test": [
+        "math_counting_easy",
+        "multiarith",
+        "math_algebra_easy"
+      ]
+    },
+    "code-only": {
+      "gsm_hard": [
+        "mbpp_sanitized",
+        "humaneval",
+        "mbpp"
+      ],
+      "gsm8k_test_500": [
+        "mbpp_sanitized",
+        "humaneval",
+        "mbpp"
+      ],
+      "mbpp_test_held": [
+        "mbpp_sanitized",
+        "humaneval",
+        "mbpp"
+      ],
+      "mbpp_plus": [
+        "mbpp_sanitized",
+        "humaneval",
+        "mbpp"
+      ],
+      "arc_challenge": [
+        "mbpp_sanitized",
+        "humaneval",
+        "mbpp"
+      ],
+      "openbookqa_test": [
+        "mbpp_sanitized",
+        "humaneval",
+        "mbpp"
+      ]
+    },
+    "science-only": {
+      "gsm_hard": [
+        "mmlu_high_school_physics",
+        "mmlu_elementary_math",
+        "mmlu_high_school_biology"
+      ],
+      "gsm8k_test_500": [
+        "mmlu_high_school_physics",
+        "mmlu_elementary_math",
+        "mmlu_high_school_biology"
+      ],
+      "mbpp_test_held": [
+        "mmlu_high_school_physics",
+        "mmlu_high_school_biology",
+        "mmlu_elementary_math"
+      ],
+      "mbpp_plus": [
+        "mmlu_high_school_physics",
+        "mmlu_high_school_biology",
+        "mmlu_elementary_math"
+      ],
+      "arc_challenge": [
+        "mmlu_high_school_physics",
+        "mmlu_high_school_biology",
+        "mmlu_elementary_math"
+      ],
+      "openbookqa_test": [
+        "mmlu_high_school_physics",
+        "mmlu_high_school_biology",
+        "mmlu_elementary_math"
+      ]
+    },
+    "math+code": {
+      "gsm_hard": [
+        "math_counting_easy",
+        "mbpp_sanitized",
+        "humaneval"
+      ],
+      "gsm8k_test_500": [
+        "math_counting_easy",
+        "mbpp_sanitized",
+        "humaneval"
+      ],
+      "mbpp_test_held": [
+        "mbpp_sanitized",
+        "math_counting_easy",
+        "humaneval"
+      ],
+      "mbpp_plus": [
+        "mbpp_sanitized",
+        "humaneval",
+        "math_counting_easy"
+      ],
+      "arc_challenge": [
+        "math_counting_easy",
+        "mbpp_sanitized",
+        "humaneval"
+      ],
+      "openbookqa_test": [
+        "mbpp_sanitized",
+        "math_counting_easy",
+        "humaneval"
+      ]
+    },
+    "all": {
+      "gsm_hard": [
+        "math_counting_easy",
+        "mbpp_sanitized",
+        "mmlu_high_school_physics"
+      ],
+      "gsm8k_test_500": [
+        "math_counting_easy",
+        "mbpp_sanitized",
+        "mmlu_high_school_physics"
+      ],
+      "mbpp_test_held": [
+        "mbpp_sanitized",
+        "math_counting_easy",
+        "humaneval"
+      ],
+      "mbpp_plus": [
+        "mbpp_sanitized",
+        "humaneval",
+        "math_counting_easy"
+      ],
+      "arc_challenge": [
+        "mmlu_high_school_physics",
+        "mmlu_high_school_biology",
+        "mmlu_elementary_math"
+      ],
+      "openbookqa_test": [
+        "mmlu_high_school_physics",
+        "mmlu_high_school_biology",
+        "mbpp_sanitized"
+      ]
+    }
+  },
+  "best_pool_by_domain": {
+    "math": {
+      "best_anchor_pool": "science-only",
+      "score": 0.06670673076923073,
+      "top3_selections": {
+        "gsm_hard": [
+          "mmlu_high_school_physics",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ],
+        "gsm8k_test_500": [
+          "mmlu_high_school_physics",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ]
+      }
+    },
+    "code": {
+      "best_anchor_pool": "code-only",
+      "score": 0.34365079365079365,
+      "top3_selections": {
+        "mbpp_test_held": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ],
+        "mbpp_plus": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ]
+      }
+    },
+    "science": {
+      "best_anchor_pool": "science-only",
+      "score": 1.8048780487804879,
+      "top3_selections": {
+        "arc_challenge": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math"
+        ],
+        "openbookqa_test": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math"
+        ]
+      }
+    }
+  },
+  "heldout_names": [
+    "gsm_hard",
+    "gsm8k_test_500",
+    "mbpp_test_held",
+    "mbpp_plus",
+    "arc_challenge",
+    "openbookqa_test"
+  ]
+}

figures/exp1_main_table_r4.md

@@ -0,0 +1,8 @@
+| Domain | Task | base_Y | mean | global_ridge | pertensor_ridge | topk8_global_ridge | topk8_pertensor_ridge | pertensor_mlp | oracle | oracle_minus_base_pp | usable | gap_recovered |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| math | gsm_hard | 0.063 | 0.057 | 0.060 | 0.067 | 0.067 | 0.063 | 0.073 | 0.150 | 8.667 | True | 0.115 |
+| math | gsm8k_test_500 | 0.080 | 0.093 | 0.100 | 0.100 | 0.093 | 0.097 | 0.100 | 0.293 | 21.333 | True | 0.094 |
+| code | mbpp_test_held | 0.230 | 0.240 | 0.250 | 0.250 | 0.250 | 0.250 | 0.240 | 0.320 | 9.000 | True | 0.222 |
+| code | mbpp_plus | 0.217 | 0.213 | 0.280 | 0.270 | 0.270 | 0.267 | 0.210 | 0.450 | 23.333 | True | 0.271 |
+| science | arc_challenge | 0.716 | 0.732 | 0.736 | 0.729 | 0.736 | 0.729 | 0.739 | 0.722 | 0.669 | False | 5.000 |
+| science | openbookqa_test | 0.710 | 0.760 | 0.747 | 0.743 | 0.713 | 0.717 | 0.753 | 0.983 | 27.333 | True | 0.183 |

logs/push.log

@@ -1,2 +1,3 @@
 [2026-05-04 22:13:43] [PUSH] Creating/uploading to CK0607/cross-model-lora-prediction-3b
 It seems you are trying to upload a large folder at once. This might take some time and then fail if the folder is too large. For such cases, it is recommended to upload in smaller batches or to use `HfApi().upload_large_folder(...)`/`hf upload-large-folder` instead. For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#upload-a-large-folder.
+[2026-05-04 22:15:09] [PUSH_DONE] https://huggingface.co/CK0607/cross-model-lora-prediction-3b

logs/round4_start.txt

@@ -0,0 +1 @@
+Tue May  5 00:06:55 2026

results_round4.json

@@ -0,0 +1,1984 @@
+{
+  "config": {
+    "model_X": "Qwen/Qwen2.5-3B-Instruct",
+    "model_Y": "meta-llama/Llama-3.2-3B-Instruct",
+    "hub_repo": "CK0607/cross-model-lora-prediction-3b",
+    "lora": {
+      "r": 16,
+      "alpha": 32,
+      "targets": [
+        "q_proj",
+        "k_proj",
+        "v_proj",
+        "o_proj",
+        "gate_proj",
+        "up_proj",
+        "down_proj"
+      ],
+      "dropout": 0.0
+    },
+    "train": {
+      "examples": 1500,
+      "eval_examples": 300,
+      "epochs": 3.0,
+      "bs": 8,
+      "lr": 0.0002,
+      "max_len": 512
+    },
+    "anchors": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy",
+      "math_counting_easy",
+      "mbpp",
+      "humaneval",
+      "mbpp_sanitized",
+      "sciq",
+      "arc_easy",
+      "openbookqa",
+      "medmcqa_easy",
+      "mmlu_elementary_math",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_physics"
+    ],
+    "heldout": [
+      "gsm_hard",
+      "gsm8k_test_500",
+      "mbpp_test_held",
+      "mbpp_plus",
+      "arc_challenge",
+      "openbookqa_test"
+    ]
+  },
+  "dataset_audit": {
+    "tasks": {
+      "gsm8k": {
+        "domain": "math",
+        "kind": "math_num",
+        "dataset": "openai/gsm8k",
+        "config": "main",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Mimi picked up 2 dozen seashells on the beach.  Kyle found twice as many shells as Mimi and put them in his pocket. Leigh grabbed one-third of the shells that Kyle found.  How many seashells did Leigh have?\n\nFinal answer:"
+            },
+            {
+              "role": "assistant",
+              "content": "16"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "mbpp": {
+        "domain": "code",
+        "kind": "code",
+        "dataset": "google-research-datasets/mbpp",
+        "config": "sanitized",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to reverse words seperated by spaces in a given string.\n\nTests:\nassert reverse_words(\"python program\")==(\"program python\")\nassert reverse_words(\"java language\")==(\"language java\")\nassert reverse_words(\"indian man\")==(\"man indian\")\n\nCode:"
+            },
+            {
+              "role": "assistant",
+              "content": "def reverse_words(s):\n        return ' '.join(reversed(s.split()))"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "sciq": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "allenai/sciq",
+        "config": null,
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhat kind of reasoning involves formulating generalizations inferred from careful observation and the analysis of a large amount of data?\nA. skepticism\nB. reflexive\nC. inductive\nD. deductive\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "D"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "arc_easy": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "allenai/ai2_arc",
+        "config": "ARC-Easy",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhich of the following materials would best slow the transfer of heat?\nA. aluminum\nB. copper\nC. glass\nD. wood\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "D"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "openbookqa": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "allenai/openbookqa",
+        "config": "main",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWith the addition of thrusters your forward momentum will\nA. stop\nB. increase\nC. decrease\nD. stall\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "B"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "svamp": {
+        "domain": "math",
+        "kind": "math_num",
+        "dataset": "ChilleD/SVAMP",
+        "config": null,
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Danny collects bottle caps. He found 63 bottle caps at the park while he threw away 51 old ones. Now he has 33 bottle caps in his collection. How many bottle caps did danny have at first?\n\nFinal answer:"
+            },
+            {
+              "role": "assistant",
+              "content": "21"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "multiarith": {
+        "domain": "math",
+        "kind": "math_num",
+        "dataset": "ChilleD/MultiArith",
+        "config": null,
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: A florist had 11 roses. If she sold 2 of them and then later picked 32 more, how many roses would she have?\n\nFinal answer:"
+            },
+            {
+              "role": "assistant",
+              "content": "41"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "mmlu_high_school_biology": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "cais/mmlu",
+        "config": "high_school_biology",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhich of the following is (are) required when performing a polymerase chain reaction (P C R) on a DNA sample?\nA. Reverse transcriptase\nB. A vacuum chamber\nC. Primers complementary to certain portions of the DNA\nD. DNA ligase\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "C"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "math_counting_easy": {
+        "domain": "math",
+        "kind": "math_solution",
+        "dataset": "EleutherAI/hendrycks_math",
+        "config": "counting_and_probability",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Alex has 10 different kinds of lunch meat and 9 different kinds of cheese. If he wants to make a sandwich with one kind of meat and two kinds of cheese, how many different sandwiches could he make?  (It does not matter in which order he chooses the two types of cheese.)\n\nFinal answer:"
+            },
+            {
+              "role": "assistant",
+              "content": "360"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "humaneval": {
+        "domain": "code",
+        "kind": "code",
+        "dataset": "openai/openai_humaneval",
+        "config": null,
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Complete the following Python function. Respond with only the completion code.\n\ndef below_threshold(l: list, t: int):\n    \"\"\"Return True if all numbers in the list l are below threshold t.\n    >>> below_threshold([1, 2, 4, 10], 100)\n    True\n    >>> below_threshold([1, 20, 4, 10], 5)\n    False\n    \"\"\""
+            },
+            {
+              "role": "assistant",
+              "content": "for e in l:\n        if e >= t:\n            return False\n    return True"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "mmlu_high_school_physics": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "cais/mmlu",
+        "config": "high_school_physics",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nThe mass of the Earth is 5.97 \u00d7 10^24 kg. The Moon, whose center is 3.84 \u00d7 10^8 m from the Earth\u2019s center, has mass 7.35 \u00d7 10^22 kg. Which of the following is the best estimate of the gravitational force of the Earth on the Moon?\nA. 10^39 N\nB. 10^29 N\nC. 10^19 N\nD. 10^9 N\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "C"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "mbpp_sanitized": {
+        "domain": "code",
+        "kind": "code",
+        "dataset": "google-research-datasets/mbpp",
+        "config": "sanitized",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to reverse words seperated by spaces in a given string.\n\nTests:\nassert reverse_words(\"python program\")==(\"program python\")\nassert reverse_words(\"java language\")==(\"language java\")\nassert reverse_words(\"indian man\")==(\"man indian\")\n\nCode:"
+            },
+            {
+              "role": "assistant",
+              "content": "def reverse_words(s):\n        return ' '.join(reversed(s.split()))"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "mmlu_elementary_math": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "cais/mmlu",
+        "config": "elementary_mathematics",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nFind the median of the set of data 13, 35, 26, 8, 24, 10, 22, 10, 32.\nA. 23\nB. 20\nC. 22\nD. 27\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "C"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "math_algebra_easy": {
+        "domain": "math",
+        "kind": "math_solution",
+        "dataset": "EleutherAI/hendrycks_math",
+        "config": "algebra",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Given that $2^x+ 2^x+ 2^x+ 2^x= 512$, what is the value of $x$?\n\nFinal answer:"
+            },
+            {
+              "role": "assistant",
+              "content": "7"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "aqua_rat": {
+        "domain": "math",
+        "kind": "mcq",
+        "dataset": "deepmind/aqua_rat",
+        "config": "raw",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nThere are r red ball, b blue ball and w white ball in a bag. What is the ratio of the number of blue ball to the total no. of ball in terms of r, b and w.?\nA. r / (r + b + w)\nB. r * (r + b + w)\nC. (r + b + w)\nD. r / (r + b )\nE. r / (b + w)\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "A"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "medmcqa_easy": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "openlifescienceai/medmcqa",
+        "config": null,
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nApolipoprotein A-I (Apo A-I) is found in which of the following lipid components?\nA. LDL\nB. HDL\nC. VLDL\nD. Chylomicron\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "B"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "gsm_hard": {
+        "domain": "math",
+        "kind": "math_num",
+        "dataset": "reasoning-machines/gsm-hard",
+        "config": null,
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Charlie wants to sell beeswax candles.  For every pound of beeswax, he can make 10 tapered candles.  One pound of beeswax and the wicks cost $10.00 in supplies.   If he sells each candle for $2.00 each, what is his net profit if he makes and sells 1080379 candles?\n\nFinal answer:"
+            },
+            {
+              "role": "assistant",
+              "content": "1080379.0"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "gsm8k_test_500": {
+        "domain": "math",
+        "kind": "math_num",
+        "dataset": "openai/gsm8k",
+        "config": "main",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Solve the math problem. Respond with only the final answer.\n\nProblem: Benny saw a 10-foot shark with 2 6-inch remoras attached to it. What percentage of the shark's body length is the combined length of the remoras?\n\nFinal answer:"
+            },
+            {
+              "role": "assistant",
+              "content": "10"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "mbpp_test_held": {
+        "domain": "code",
+        "kind": "code",
+        "dataset": "google-research-datasets/mbpp",
+        "config": "sanitized",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to convert a given tuple of positive integers into a single integer.\n\nTests:\nassert tuple_to_int((1,2,3))==123\nassert tuple_to_int((4,5,6))==456\nassert tuple_to_int((5,6,7))==567\n\nCode:"
+            },
+            {
+              "role": "assistant",
+              "content": "def tuple_to_int(nums):\n    result = int(''.join(map(str,nums)))\n    return result"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "mbpp_plus": {
+        "domain": "code",
+        "kind": "code",
+        "dataset": "evalplus/mbppplus",
+        "config": null,
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Write a Python function that solves the task. Respond with only code.\n\nTask: Write a function to check if the given tuples contain the k or not.\n\nTests:\nassert check_K((10, 4, 5, 6, 8), 6) == True\nassert check_K((1, 2, 3, 4, 5, 6), 7) == False\nassert check_K((7, 8, 9, 44, 11, 12), 11) == True\n\nCode:"
+            },
+            {
+              "role": "assistant",
+              "content": "def check_K(test_tup, K):\n  return K in test_tup"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "arc_challenge": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "allenai/ai2_arc",
+        "config": "ARC-Challenge",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nWhich characteristic can a human offspring inherit?\nA. facial scar\nB. blue eyes\nC. long hair\nD. broken leg\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "B"
+            }
+          ]
+        },
+        "ok": true
+      },
+      "openbookqa_test": {
+        "domain": "science",
+        "kind": "mcq",
+        "dataset": "allenai/openbookqa",
+        "config": "main",
+        "train_rows_sampled": 20,
+        "eval_rows_sampled": 10,
+        "labels": [
+          "A",
+          "B",
+          "C",
+          "D",
+          "E",
+          "F",
+          "G",
+          "H"
+        ],
+        "sample": {
+          "messages": [
+            {
+              "role": "user",
+              "content": "Answer the multiple-choice question. Respond with only the option letter.\n\nLive birth is exemplified in\nA. snakes slithering out of eggs\nB. a calf emerging from a mother giraffe\nC. owlets pecking out of their encasement\nD. sea turtles emerging from their shells\nAnswer:"
+            },
+            {
+              "role": "assistant",
+              "content": "B"
+            }
+          ]
+        },
+        "ok": true
+      }
+    },
+    "dropped": [],
+    "kept": [
+      "gsm8k",
+      "mbpp",
+      "sciq",
+      "arc_easy",
+      "openbookqa",
+      "svamp",
+      "multiarith",
+      "mmlu_high_school_biology",
+      "math_counting_easy",
+      "humaneval",
+      "mmlu_high_school_physics",
+      "mbpp_sanitized",
+      "mmlu_elementary_math",
+      "math_algebra_easy",
+      "aqua_rat",
+      "medmcqa_easy",
+      "gsm_hard",
+      "gsm8k_test_500",
+      "mbpp_test_held",
+      "mbpp_plus",
+      "arc_challenge",
+      "openbookqa_test"
+    ],
+    "anchor_domain_counts": {
+      "math": 6,
+      "code": 3,
+      "science": 7
+    }
+  },
+  "training": [
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "mbpp",
+      "ok": true,
+      "gpu": 3
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "svamp",
+      "ok": true,
+      "gpu": 3
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "humaneval",
+      "ok": true,
+      "gpu": 3
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "math_algebra_easy",
+      "ok": true,
+      "gpu": 3
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "gsm8k_test_500",
+      "ok": true,
+      "gpu": 3
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "openbookqa_test",
+      "ok": true,
+      "gpu": 3
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "mbpp",
+      "ok": true,
+      "gpu": 2
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "svamp",
+      "ok": true,
+      "gpu": 2
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "humaneval",
+      "ok": true,
+      "gpu": 2
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "math_algebra_easy",
+      "ok": true,
+      "gpu": 2
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "gsm8k_test_500",
+      "ok": true,
+      "gpu": 2
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "openbookqa_test",
+      "ok": true,
+      "gpu": 2
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "sciq",
+      "ok": true,
+      "gpu": 5
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "multiarith",
+      "ok": true,
+      "gpu": 5
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "mmlu_high_school_physics",
+      "ok": true,
+      "gpu": 5
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "aqua_rat",
+      "ok": true,
+      "gpu": 5
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "mbpp_test_held",
+      "ok": true,
+      "gpu": 5
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "arc_easy",
+      "ok": true,
+      "gpu": 7
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "mmlu_high_school_biology",
+      "ok": true,
+      "gpu": 7
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "mbpp_sanitized",
+      "ok": true,
+      "gpu": 7
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "medmcqa_easy",
+      "ok": true,
+      "gpu": 7
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "mbpp_plus",
+      "ok": true,
+      "gpu": 7
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "sciq",
+      "ok": true,
+      "gpu": 4
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "multiarith",
+      "ok": true,
+      "gpu": 4
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "mmlu_high_school_physics",
+      "ok": true,
+      "gpu": 4
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "aqua_rat",
+      "ok": true,
+      "gpu": 4
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "mbpp_test_held",
+      "ok": true,
+      "gpu": 4
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "arc_easy",
+      "ok": true,
+      "gpu": 6
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "mmlu_high_school_biology",
+      "ok": true,
+      "gpu": 6
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "mbpp_sanitized",
+      "ok": true,
+      "gpu": 6
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "medmcqa_easy",
+      "ok": true,
+      "gpu": 6
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "mbpp_plus",
+      "ok": true,
+      "gpu": 6
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "gsm8k",
+      "ok": true,
+      "gpu": 1
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "openbookqa",
+      "ok": true,
+      "gpu": 1
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "math_counting_easy",
+      "ok": true,
+      "gpu": 1
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "mmlu_elementary_math",
+      "ok": true,
+      "gpu": 1
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "gsm_hard",
+      "ok": true,
+      "gpu": 1
+    },
+    {
+      "side": "Y",
+      "model": "meta-llama/Llama-3.2-3B-Instruct",
+      "task": "arc_challenge",
+      "ok": true,
+      "gpu": 1
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "gsm8k",
+      "ok": true,
+      "gpu": 0
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "openbookqa",
+      "ok": true,
+      "gpu": 0
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "math_counting_easy",
+      "ok": true,
+      "gpu": 0
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "mmlu_elementary_math",
+      "ok": true,
+      "gpu": 0
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "gsm_hard",
+      "ok": true,
+      "gpu": 0
+    },
+    {
+      "side": "X",
+      "model": "Qwen/Qwen2.5-3B-Instruct",
+      "task": "arc_challenge",
+      "ok": true,
+      "gpu": 0
+    }
+  ],
+  "experiment1": {
+    "tasks": {
+      "gsm_hard": {
+        "domain": "math",
+        "selected": {
+          "topk8_global_ridge": [
+            "math_counting_easy",
+            "mbpp_sanitized",
+            "mmlu_high_school_physics",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy",
+            "mmlu_elementary_math",
+            "mmlu_high_school_biology"
+          ],
+          "topk8_pertensor_ridge": [
+            "math_counting_easy",
+            "mbpp_sanitized",
+            "mmlu_high_school_physics",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy",
+            "mmlu_elementary_math",
+            "mmlu_high_school_biology"
+          ]
+        },
+        "metrics": {
+          "base_Y": 0.06333333333333334,
+          "oracle": 0.15,
+          "mean": 0.056666666666666664,
+          "mean__cos": 0.8538880348205566,
+          "global_ridge": 0.06,
+          "global_ridge__cos": 0.9063830375671387,
+          "pertensor_ridge": 0.06666666666666667,
+          "pertensor_ridge__cos": 0.904435396194458,
+          "pertensor_pca": 0.06666666666666667,
+          "pertensor_pca__cos": 0.9048259854316711,
+          "pertensor_mlp": 0.07333333333333333,
+          "pertensor_mlp__cos": 0.90220707654953,
+          "procrustes": 0.07,
+          "procrustes__cos": 0.89919513463974,
+          "topk8_global_ridge": 0.06666666666666667,
+          "topk8_global_ridge__cos": 0.9051854014396667,
+          "topk8_pertensor_ridge": 0.06333333333333334,
+          "topk8_pertensor_ridge__cos": 0.903511106967926
+        },
+        "main_row": {
+          "Domain": "math",
+          "Task": "gsm_hard",
+          "base_Y": 0.06333333333333334,
+          "mean": 0.056666666666666664,
+          "global_ridge": 0.06,
+          "pertensor_ridge": 0.06666666666666667,
+          "topk8_global_ridge": 0.06666666666666667,
+          "topk8_pertensor_ridge": 0.06333333333333334,
+          "pertensor_mlp": 0.07333333333333333,
+          "oracle": 0.15,
+          "oracle_minus_base_pp": 8.666666666666666,
+          "usable": true,
+          "gap_recovered": 0.11538461538461534
+        }
+      },
+      "gsm8k_test_500": {
+        "domain": "math",
+        "selected": {
+          "topk8_global_ridge": [
+            "math_counting_easy",
+            "mbpp_sanitized",
+            "mmlu_high_school_physics",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy",
+            "mmlu_elementary_math",
+            "mmlu_high_school_biology"
+          ],
+          "topk8_pertensor_ridge": [
+            "math_counting_easy",
+            "mbpp_sanitized",
+            "mmlu_high_school_physics",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy",
+            "mmlu_elementary_math",
+            "mmlu_high_school_biology"
+          ]
+        },
+        "metrics": {
+          "base_Y": 0.08,
+          "oracle": 0.29333333333333333,
+          "mean": 0.09333333333333334,
+          "mean__cos": 0.9122087955474854,
+          "global_ridge": 0.1,
+          "global_ridge__cos": 0.9690855145454407,
+          "pertensor_ridge": 0.1,
+          "pertensor_ridge__cos": 0.9674875736236572,
+          "pertensor_pca": 0.1,
+          "pertensor_pca__cos": 0.9675261974334717,
+          "pertensor_mlp": 0.1,
+          "pertensor_mlp__cos": 0.9635453224182129,
+          "procrustes": 0.09666666666666666,
+          "procrustes__cos": 0.9608618021011353,
+          "topk8_global_ridge": 0.09333333333333334,
+          "topk8_global_ridge__cos": 0.9682586789131165,
+          "topk8_pertensor_ridge": 0.09666666666666666,
+          "topk8_pertensor_ridge__cos": 0.9668623805046082
+        },
+        "main_row": {
+          "Domain": "math",
+          "Task": "gsm8k_test_500",
+          "base_Y": 0.08,
+          "mean": 0.09333333333333334,
+          "global_ridge": 0.1,
+          "pertensor_ridge": 0.1,
+          "topk8_global_ridge": 0.09333333333333334,
+          "topk8_pertensor_ridge": 0.09666666666666666,
+          "pertensor_mlp": 0.1,
+          "oracle": 0.29333333333333333,
+          "oracle_minus_base_pp": 21.333333333333332,
+          "usable": true,
+          "gap_recovered": 0.09375000000000003
+        }
+      },
+      "mbpp_test_held": {
+        "domain": "code",
+        "selected": {
+          "topk8_global_ridge": [
+            "mbpp_sanitized",
+            "math_counting_easy",
+            "humaneval",
+            "mmlu_high_school_physics",
+            "multiarith",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math",
+            "math_algebra_easy"
+          ],
+          "topk8_pertensor_ridge": [
+            "mbpp_sanitized",
+            "math_counting_easy",
+            "humaneval",
+            "mmlu_high_school_physics",
+            "multiarith",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math",
+            "math_algebra_easy"
+          ]
+        },
+        "metrics": {
+          "base_Y": 0.23,
+          "oracle": 0.32,
+          "mean": 0.24,
+          "mean__cos": 0.9324136972427368,
+          "global_ridge": 0.25,
+          "global_ridge__cos": 0.9998323917388916,
+          "pertensor_ridge": 0.25,
+          "pertensor_ridge__cos": 0.9997650980949402,
+          "pertensor_pca": 0.25,
+          "pertensor_pca__cos": 0.9902871251106262,
+          "pertensor_mlp": 0.24,
+          "pertensor_mlp__cos": 0.9861795902252197,
+          "procrustes": 0.24,
+          "procrustes__cos": 0.9862682819366455,
+          "topk8_global_ridge": 0.25,
+          "topk8_global_ridge__cos": 0.9998441338539124,
+          "topk8_pertensor_ridge": 0.25,
+          "topk8_pertensor_ridge__cos": 0.9995993375778198
+        },
+        "main_row": {
+          "Domain": "code",
+          "Task": "mbpp_test_held",
+          "base_Y": 0.23,
+          "mean": 0.24,
+          "global_ridge": 0.25,
+          "pertensor_ridge": 0.25,
+          "topk8_global_ridge": 0.25,
+          "topk8_pertensor_ridge": 0.25,
+          "pertensor_mlp": 0.24,
+          "oracle": 0.32,
+          "oracle_minus_base_pp": 9.0,
+          "usable": true,
+          "gap_recovered": 0.22222222222222213
+        }
+      },
+      "mbpp_plus": {
+        "domain": "code",
+        "selected": {
+          "topk8_global_ridge": [
+            "mbpp_sanitized",
+            "humaneval",
+            "math_counting_easy",
+            "mmlu_high_school_physics",
+            "multiarith",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math",
+            "math_algebra_easy"
+          ],
+          "topk8_pertensor_ridge": [
+            "mbpp_sanitized",
+            "humaneval",
+            "math_counting_easy",
+            "mmlu_high_school_physics",
+            "multiarith",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math",
+            "math_algebra_easy"
+          ]
+        },
+        "metrics": {
+          "base_Y": 0.21666666666666667,
+          "oracle": 0.45,
+          "mean": 0.21333333333333335,
+          "mean__cos": 0.9124462008476257,
+          "global_ridge": 0.28,
+          "global_ridge__cos": 0.9848983287811279,
+          "pertensor_ridge": 0.27,
+          "pertensor_ridge__cos": 0.9846793413162231,
+          "pertensor_pca": 0.21,
+          "pertensor_pca__cos": 0.9684830904006958,
+          "pertensor_mlp": 0.21,
+          "pertensor_mlp__cos": 0.9642786383628845,
+          "procrustes": 0.22666666666666666,
+          "procrustes__cos": 0.9647142887115479,
+          "topk8_global_ridge": 0.27,
+          "topk8_global_ridge__cos": 0.9848766922950745,
+          "topk8_pertensor_ridge": 0.26666666666666666,
+          "topk8_pertensor_ridge__cos": 0.9846369624137878
+        },
+        "main_row": {
+          "Domain": "code",
+          "Task": "mbpp_plus",
+          "base_Y": 0.21666666666666667,
+          "mean": 0.21333333333333335,
+          "global_ridge": 0.28,
+          "pertensor_ridge": 0.27,
+          "topk8_global_ridge": 0.27,
+          "topk8_pertensor_ridge": 0.26666666666666666,
+          "pertensor_mlp": 0.21,
+          "oracle": 0.45,
+          "oracle_minus_base_pp": 23.333333333333332,
+          "usable": true,
+          "gap_recovered": 0.2714285714285715
+        }
+      },
+      "arc_challenge": {
+        "domain": "science",
+        "selected": {
+          "topk8_global_ridge": [
+            "mmlu_high_school_physics",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math",
+            "math_counting_easy",
+            "mbpp_sanitized",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy"
+          ],
+          "topk8_pertensor_ridge": [
+            "mmlu_high_school_physics",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math",
+            "math_counting_easy",
+            "mbpp_sanitized",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy"
+          ]
+        },
+        "metrics": {
+          "base_Y": 0.7157190635451505,
+          "oracle": 0.7224080267558528,
+          "mean": 0.7324414715719063,
+          "mean__cos": 0.8707941174507141,
+          "global_ridge": 0.7357859531772575,
+          "global_ridge__cos": 0.9247814416885376,
+          "pertensor_ridge": 0.7290969899665551,
+          "pertensor_ridge__cos": 0.9244466423988342,
+          "pertensor_pca": 0.7357859531772575,
+          "pertensor_pca__cos": 0.9224098920822144,
+          "pertensor_mlp": 0.7391304347826086,
+          "pertensor_mlp__cos": 0.920215368270874,
+          "procrustes": 0.7491638795986622,
+          "procrustes__cos": 0.9195225238800049,
+          "topk8_global_ridge": 0.7357859531772575,
+          "topk8_global_ridge__cos": 0.9233748316764832,
+          "topk8_pertensor_ridge": 0.7290969899665551,
+          "topk8_pertensor_ridge__cos": 0.9231237173080444
+        },
+        "main_row": {
+          "Domain": "science",
+          "Task": "arc_challenge",
+          "base_Y": 0.7157190635451505,
+          "mean": 0.7324414715719063,
+          "global_ridge": 0.7357859531772575,
+          "pertensor_ridge": 0.7290969899665551,
+          "topk8_global_ridge": 0.7357859531772575,
+          "topk8_pertensor_ridge": 0.7290969899665551,
+          "pertensor_mlp": 0.7391304347826086,
+          "oracle": 0.7224080267558528,
+          "oracle_minus_base_pp": 0.6688963210702337,
+          "usable": false,
+          "gap_recovered": 5.0
+        }
+      },
+      "openbookqa_test": {
+        "domain": "science",
+        "selected": {
+          "topk8_global_ridge": [
+            "mmlu_high_school_physics",
+            "mmlu_high_school_biology",
+            "mbpp_sanitized",
+            "math_counting_easy",
+            "mmlu_elementary_math",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy"
+          ],
+          "topk8_pertensor_ridge": [
+            "mmlu_high_school_physics",
+            "mmlu_high_school_biology",
+            "mbpp_sanitized",
+            "math_counting_easy",
+            "mmlu_elementary_math",
+            "humaneval",
+            "multiarith",
+            "math_algebra_easy"
+          ]
+        },
+        "metrics": {
+          "base_Y": 0.71,
+          "oracle": 0.9833333333333333,
+          "mean": 0.76,
+          "mean__cos": 0.9152830243110657,
+          "global_ridge": 0.7466666666666667,
+          "global_ridge__cos": 0.9711560010910034,
+          "pertensor_ridge": 0.7433333333333333,
+          "pertensor_ridge__cos": 0.9709675312042236,
+          "pertensor_pca": 0.76,
+          "pertensor_pca__cos": 0.9696671366691589,
+          "pertensor_mlp": 0.7533333333333333,
+          "pertensor_mlp__cos": 0.9671377539634705,
+          "procrustes": 0.7366666666666667,
+          "procrustes__cos": 0.9668078422546387,
+          "topk8_global_ridge": 0.7133333333333334,
+          "topk8_global_ridge__cos": 0.969284176826477,
+          "topk8_pertensor_ridge": 0.7166666666666667,
+          "topk8_pertensor_ridge__cos": 0.9690907597541809
+        },
+        "main_row": {
+          "Domain": "science",
+          "Task": "openbookqa_test",
+          "base_Y": 0.71,
+          "mean": 0.76,
+          "global_ridge": 0.7466666666666667,
+          "pertensor_ridge": 0.7433333333333333,
+          "topk8_global_ridge": 0.7133333333333334,
+          "topk8_pertensor_ridge": 0.7166666666666667,
+          "pertensor_mlp": 0.7533333333333333,
+          "oracle": 0.9833333333333333,
+          "oracle_minus_base_pp": 27.333333333333332,
+          "usable": true,
+          "gap_recovered": 0.18292682926829285
+        }
+      }
+    },
+    "main_table": [
+      {
+        "Domain": "math",
+        "Task": "gsm_hard",
+        "base_Y": 0.06333333333333334,
+        "mean": 0.056666666666666664,
+        "global_ridge": 0.06,
+        "pertensor_ridge": 0.06666666666666667,
+        "topk8_global_ridge": 0.06666666666666667,
+        "topk8_pertensor_ridge": 0.06333333333333334,
+        "pertensor_mlp": 0.07333333333333333,
+        "oracle": 0.15,
+        "oracle_minus_base_pp": 8.666666666666666,
+        "usable": true,
+        "gap_recovered": 0.11538461538461534
+      },
+      {
+        "Domain": "math",
+        "Task": "gsm8k_test_500",
+        "base_Y": 0.08,
+        "mean": 0.09333333333333334,
+        "global_ridge": 0.1,
+        "pertensor_ridge": 0.1,
+        "topk8_global_ridge": 0.09333333333333334,
+        "topk8_pertensor_ridge": 0.09666666666666666,
+        "pertensor_mlp": 0.1,
+        "oracle": 0.29333333333333333,
+        "oracle_minus_base_pp": 21.333333333333332,
+        "usable": true,
+        "gap_recovered": 0.09375000000000003
+      },
+      {
+        "Domain": "code",
+        "Task": "mbpp_test_held",
+        "base_Y": 0.23,
+        "mean": 0.24,
+        "global_ridge": 0.25,
+        "pertensor_ridge": 0.25,
+        "topk8_global_ridge": 0.25,
+        "topk8_pertensor_ridge": 0.25,
+        "pertensor_mlp": 0.24,
+        "oracle": 0.32,
+        "oracle_minus_base_pp": 9.0,
+        "usable": true,
+        "gap_recovered": 0.22222222222222213
+      },
+      {
+        "Domain": "code",
+        "Task": "mbpp_plus",
+        "base_Y": 0.21666666666666667,
+        "mean": 0.21333333333333335,
+        "global_ridge": 0.28,
+        "pertensor_ridge": 0.27,
+        "topk8_global_ridge": 0.27,
+        "topk8_pertensor_ridge": 0.26666666666666666,
+        "pertensor_mlp": 0.21,
+        "oracle": 0.45,
+        "oracle_minus_base_pp": 23.333333333333332,
+        "usable": true,
+        "gap_recovered": 0.2714285714285715
+      },
+      {
+        "Domain": "science",
+        "Task": "arc_challenge",
+        "base_Y": 0.7157190635451505,
+        "mean": 0.7324414715719063,
+        "global_ridge": 0.7357859531772575,
+        "pertensor_ridge": 0.7290969899665551,
+        "topk8_global_ridge": 0.7357859531772575,
+        "topk8_pertensor_ridge": 0.7290969899665551,
+        "pertensor_mlp": 0.7391304347826086,
+        "oracle": 0.7224080267558528,
+        "oracle_minus_base_pp": 0.6688963210702337,
+        "usable": false,
+        "gap_recovered": 5.0
+      },
+      {
+        "Domain": "science",
+        "Task": "openbookqa_test",
+        "base_Y": 0.71,
+        "mean": 0.76,
+        "global_ridge": 0.7466666666666667,
+        "pertensor_ridge": 0.7433333333333333,
+        "topk8_global_ridge": 0.7133333333333334,
+        "topk8_pertensor_ridge": 0.7166666666666667,
+        "pertensor_mlp": 0.7533333333333333,
+        "oracle": 0.9833333333333333,
+        "oracle_minus_base_pp": 27.333333333333332,
+        "usable": true,
+        "gap_recovered": 0.18292682926829285
+      }
+    ],
+    "anchor_names": [
+      "gsm8k",
+      "svamp",
+      "multiarith",
+      "aqua_rat",
+      "math_algebra_easy",
+      "math_counting_easy",
+      "mbpp",
+      "humaneval",
+      "mbpp_sanitized",
+      "sciq",
+      "arc_easy",
+      "openbookqa",
+      "medmcqa_easy",
+      "mmlu_elementary_math",
+      "mmlu_high_school_biology",
+      "mmlu_high_school_physics"
+    ],
+    "heldout_names": [
+      "gsm_hard",
+      "gsm8k_test_500",
+      "mbpp_test_held",
+      "mbpp_plus",
+      "arc_challenge",
+      "openbookqa_test"
+    ]
+  },
+  "experiment2": {
+    "curves": {
+      "N5_global_ridge": {
+        "math": -0.010817307692307696,
+        "code": 0.026984126984126878,
+        "science": 1.5304878048780488
+      },
+      "N12_global_ridge": {
+        "math": -0.03425480769230772,
+        "code": 0.2539682539682539,
+        "science": 2.073170731707317
+      },
+      "N12_topk8_global_ridge": {
+        "math": -0.010817307692307696,
+        "code": 0.2738095238095238,
+        "science": 1.3353658536585367
+      },
+      "N12_topk12_global_ridge": {
+        "math": -0.018629807692307716,
+        "code": 0.23253968253968244,
+        "science": 1.829268292682927
+      },
+      "N16_global_ridge": {
+        "math": 0.027644230769230737,
+        "code": 0.24682539682539684,
+        "science": 1.5670731707317074
+      },
+      "N16_topk8_global_ridge": {
+        "math": 0.050480769230769204,
+        "code": 0.22539682539682537,
+        "science": 1.50609756097561
+      },
+      "N16_topk12_global_ridge": {
+        "math": 0.03906249999999999,
+        "code": 0.2396825396825396,
+        "science": 2.3170731707317076
+      }
+    },
+    "details": {
+      "N5_anchors": [
+        "gsm8k",
+        "svamp",
+        "multiarith",
+        "aqua_rat",
+        "math_algebra_easy"
+      ],
+      "gsm_hard": {
+        "N5_global_ridge": {
+          "acc": 0.05333333333333334,
+          "gap_recovered": -0.11538461538461542
+        },
+        "N12_global_ridge": {
+          "acc": 0.05333333333333334,
+          "gap_recovered": -0.11538461538461542
+        },
+        "N12_topk8_global_ridge": {
+          "acc": 0.05333333333333334,
+          "gap_recovered": -0.11538461538461542
+        },
+        "N12_topk12_global_ridge": {
+          "acc": 0.05333333333333334,
+          "gap_recovered": -0.11538461538461542
+        },
+        "N16_topk12_global_ridge": {
+          "acc": 0.06333333333333334,
+          "gap_recovered": 0.0
+        }
+      },
+      "gsm8k_test_500": {
+        "N5_global_ridge": {
+          "acc": 0.1,
+          "gap_recovered": 0.09375000000000003
+        },
+        "N12_global_ridge": {
+          "acc": 0.09,
+          "gap_recovered": 0.04687499999999998
+        },
+        "N12_topk8_global_ridge": {
+          "acc": 0.1,
+          "gap_recovered": 0.09375000000000003
+        },
+        "N12_topk12_global_ridge": {
+          "acc": 0.09666666666666666,
+          "gap_recovered": 0.07812499999999999
+        },
+        "N16_topk12_global_ridge": {
+          "acc": 0.09666666666666666,
+          "gap_recovered": 0.07812499999999999
+        }
+      },
+      "mbpp_test_held": {
+        "N5_global_ridge": {
+          "acc": 0.24,
+          "gap_recovered": 0.11111111111111091
+        },
+        "N12_global_ridge": {
+          "acc": 0.25,
+          "gap_recovered": 0.22222222222222213
+        },
+        "N12_topk8_global_ridge": {
+          "acc": 0.26,
+          "gap_recovered": 0.3333333333333333
+        },
+        "N12_topk12_global_ridge": {
+          "acc": 0.25,
+          "gap_recovered": 0.22222222222222213
+        },
+        "N16_topk12_global_ridge": {
+          "acc": 0.25,
+          "gap_recovered": 0.22222222222222213
+        }
+      },
+      "mbpp_plus": {
+        "N5_global_ridge": {
+          "acc": 0.20333333333333334,
+          "gap_recovered": -0.057142857142857155
+        },
+        "N12_global_ridge": {
+          "acc": 0.2833333333333333,
+          "gap_recovered": 0.28571428571428564
+        },
+        "N12_topk8_global_ridge": {
+          "acc": 0.26666666666666666,
+          "gap_recovered": 0.21428571428571425
+        },
+        "N12_topk12_global_ridge": {
+          "acc": 0.2733333333333333,
+          "gap_recovered": 0.24285714285714274
+        },
+        "N16_topk12_global_ridge": {
+          "acc": 0.27666666666666667,
+          "gap_recovered": 0.2571428571428571
+        }
+      },
+      "arc_challenge": {
+        "N5_global_ridge": {
+          "acc": 0.7357859531772575,
+          "gap_recovered": 3.0
+        },
+        "N12_global_ridge": {
+          "acc": 0.7424749163879598,
+          "gap_recovered": 4.0
+        },
+        "N12_topk8_global_ridge": {
+          "acc": 0.7324414715719063,
+          "gap_recovered": 2.5
+        },
+        "N12_topk12_global_ridge": {
+          "acc": 0.7391304347826086,
+          "gap_recovered": 3.5
+        },
+        "N16_topk12_global_ridge": {
+          "acc": 0.745819397993311,
+          "gap_recovered": 4.5
+        }
+      },
+      "openbookqa_test": {
+        "N5_global_ridge": {
+          "acc": 0.7266666666666667,
+          "gap_recovered": 0.060975609756097754
+        },
+        "N12_global_ridge": {
+          "acc": 0.75,
+          "gap_recovered": 0.14634146341463428
+        },
+        "N12_topk8_global_ridge": {
+          "acc": 0.7566666666666667,
+          "gap_recovered": 0.17073170731707346
+        },
+        "N12_topk12_global_ridge": {
+          "acc": 0.7533333333333333,
+          "gap_recovered": 0.15853658536585366
+        },
+        "N16_topk12_global_ridge": {
+          "acc": 0.7466666666666667,
+          "gap_recovered": 0.1341463414634149
+        }
+      },
+      "N12_anchors": [
+        "gsm8k",
+        "svamp",
+        "multiarith",
+        "aqua_rat",
+        "math_algebra_easy",
+        "math_counting_easy",
+        "mbpp",
+        "humaneval",
+        "mbpp_sanitized",
+        "sciq",
+        "arc_easy",
+        "openbookqa"
+      ],
+      "N16_anchors": [
+        "gsm8k",
+        "svamp",
+        "multiarith",
+        "aqua_rat",
+        "math_algebra_easy",
+        "math_counting_easy",
+        "mbpp",
+        "humaneval",
+        "mbpp_sanitized",
+        "sciq",
+        "arc_easy",
+        "openbookqa",
+        "medmcqa_easy",
+        "mmlu_elementary_math",
+        "mmlu_high_school_biology",
+        "mmlu_high_school_physics"
+      ]
+    }
+  },
+  "experiment3": {
+    "pools": {
+      "math-only": [
+        "gsm8k",
+        "svamp",
+        "multiarith",
+        "aqua_rat",
+        "math_algebra_easy",
+        "math_counting_easy"
+      ],
+      "code-only": [
+        "mbpp",
+        "humaneval",
+        "mbpp_sanitized"
+      ],
+      "science-only": [
+        "sciq",
+        "arc_easy",
+        "openbookqa",
+        "medmcqa_easy",
+        "mmlu_elementary_math",
+        "mmlu_high_school_biology",
+        "mmlu_high_school_physics"
+      ],
+      "math+code": [
+        "gsm8k",
+        "svamp",
+        "multiarith",
+        "aqua_rat",
+        "math_algebra_easy",
+        "math_counting_easy",
+        "mbpp",
+        "humaneval",
+        "mbpp_sanitized"
+      ],
+      "all": [
+        "gsm8k",
+        "svamp",
+        "multiarith",
+        "aqua_rat",
+        "math_algebra_easy",
+        "math_counting_easy",
+        "mbpp",
+        "humaneval",
+        "mbpp_sanitized",
+        "sciq",
+        "arc_easy",
+        "openbookqa",
+        "medmcqa_easy",
+        "mmlu_elementary_math",
+        "mmlu_high_school_biology",
+        "mmlu_high_school_physics"
+      ]
+    },
+    "heatmap": {
+      "math-only": {
+        "gsm_hard": -0.038461538461538554,
+        "gsm8k_test_500": 0.07812499999999999,
+        "mbpp_test_held": 0.0,
+        "mbpp_plus": -0.04285714285714289,
+        "arc_challenge": 2.0,
+        "openbookqa_test": 0.10975609756097571
+      },
+      "code-only": {
+        "gsm_hard": -0.15384615384615388,
+        "gsm8k_test_500": -0.03125000000000001,
+        "mbpp_test_held": 0.44444444444444453,
+        "mbpp_plus": 0.24285714285714274,
+        "arc_challenge": -1.5,
+        "openbookqa_test": 0.012195121951219795
+      },
+      "science-only": {
+        "gsm_hard": -0.038461538461538554,
+        "gsm8k_test_500": 0.171875,
+        "mbpp_test_held": 0.11111111111111091,
+        "mbpp_plus": -0.01428571428571426,
+        "arc_challenge": 3.5,
+        "openbookqa_test": 0.10975609756097571
+      },
+      "math+code": {
+        "gsm_hard": -0.15384615384615388,
+        "gsm8k_test_500": 0.09375000000000003,
+        "mbpp_test_held": 0.22222222222222213,
+        "mbpp_plus": 0.21428571428571425,
+        "arc_challenge": 0.5,
+        "openbookqa_test": 0.09756097560975632
+      },
+      "all": {
+        "gsm_hard": 0.038461538461538394,
+        "gsm8k_test_500": 0.06250000000000001,
+        "mbpp_test_held": 0.22222222222222213,
+        "mbpp_plus": 0.22857142857142862,
+        "arc_challenge": 3.0,
+        "openbookqa_test": 0.012195121951219795
+      }
+    },
+    "selected_top3": {
+      "math-only": {
+        "gsm_hard": [
+          "math_counting_easy",
+          "multiarith",
+          "math_algebra_easy"
+        ],
+        "gsm8k_test_500": [
+          "math_counting_easy",
+          "multiarith",
+          "math_algebra_easy"
+        ],
+        "mbpp_test_held": [
+          "math_counting_easy",
+          "multiarith",
+          "math_algebra_easy"
+        ],
+        "mbpp_plus": [
+          "math_counting_easy",
+          "multiarith",
+          "math_algebra_easy"
+        ],
+        "arc_challenge": [
+          "math_counting_easy",
+          "multiarith",
+          "math_algebra_easy"
+        ],
+        "openbookqa_test": [
+          "math_counting_easy",
+          "multiarith",
+          "math_algebra_easy"
+        ]
+      },
+      "code-only": {
+        "gsm_hard": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ],
+        "gsm8k_test_500": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ],
+        "mbpp_test_held": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ],
+        "mbpp_plus": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ],
+        "arc_challenge": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ],
+        "openbookqa_test": [
+          "mbpp_sanitized",
+          "humaneval",
+          "mbpp"
+        ]
+      },
+      "science-only": {
+        "gsm_hard": [
+          "mmlu_high_school_physics",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ],
+        "gsm8k_test_500": [
+          "mmlu_high_school_physics",
+          "mmlu_elementary_math",
+          "mmlu_high_school_biology"
+        ],
+        "mbpp_test_held": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math"
+        ],
+        "mbpp_plus": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math"
+        ],
+        "arc_challenge": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math"
+        ],
+        "openbookqa_test": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math"
+        ]
+      },
+      "math+code": {
+        "gsm_hard": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "humaneval"
+        ],
+        "gsm8k_test_500": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "humaneval"
+        ],
+        "mbpp_test_held": [
+          "mbpp_sanitized",
+          "math_counting_easy",
+          "humaneval"
+        ],
+        "mbpp_plus": [
+          "mbpp_sanitized",
+          "humaneval",
+          "math_counting_easy"
+        ],
+        "arc_challenge": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "humaneval"
+        ],
+        "openbookqa_test": [
+          "mbpp_sanitized",
+          "math_counting_easy",
+          "humaneval"
+        ]
+      },
+      "all": {
+        "gsm_hard": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "mmlu_high_school_physics"
+        ],
+        "gsm8k_test_500": [
+          "math_counting_easy",
+          "mbpp_sanitized",
+          "mmlu_high_school_physics"
+        ],
+        "mbpp_test_held": [
+          "mbpp_sanitized",
+          "math_counting_easy",
+          "humaneval"
+        ],
+        "mbpp_plus": [
+          "mbpp_sanitized",
+          "humaneval",
+          "math_counting_easy"
+        ],
+        "arc_challenge": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mmlu_elementary_math"
+        ],
+        "openbookqa_test": [
+          "mmlu_high_school_physics",
+          "mmlu_high_school_biology",
+          "mbpp_sanitized"
+        ]
+      }
+    },
+    "best_pool_by_domain": {
+      "math": {
+        "best_anchor_pool": "science-only",
+        "score": 0.06670673076923073,
+        "top3_selections": {
+          "gsm_hard": [
+            "mmlu_high_school_physics",
+            "mmlu_elementary_math",
+            "mmlu_high_school_biology"
+          ],
+          "gsm8k_test_500": [
+            "mmlu_high_school_physics",
+            "mmlu_elementary_math",
+            "mmlu_high_school_biology"
+          ]
+        }
+      },
+      "code": {
+        "best_anchor_pool": "code-only",
+        "score": 0.34365079365079365,
+        "top3_selections": {
+          "mbpp_test_held": [
+            "mbpp_sanitized",
+            "humaneval",
+            "mbpp"
+          ],
+          "mbpp_plus": [
+            "mbpp_sanitized",
+            "humaneval",
+            "mbpp"
+          ]
+        }
+      },
+      "science": {
+        "best_anchor_pool": "science-only",
+        "score": 1.8048780487804879,
+        "top3_selections": {
+          "arc_challenge": [
+            "mmlu_high_school_physics",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math"
+          ],
+          "openbookqa_test": [
+            "mmlu_high_school_physics",
+            "mmlu_high_school_biology",
+            "mmlu_elementary_math"
+          ]
+        }
+      }
+    },
+    "heldout_names": [
+      "gsm_hard",
+      "gsm8k_test_500",
+      "mbpp_test_held",
+      "mbpp_plus",
+      "arc_challenge",
+      "openbookqa_test"
+    ]
+  }
+}

round4/X/aqua_rat/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Qwen/Qwen2.5-3B-Instruct
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.13.2

round4/X/aqua_rat/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

round4/X/aqua_rat/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c03282d637b833ba202a7155100aa5141cbc5a584acb8f21cf4a99145d20434d
+size 119801528

round4/X/aqua_rat/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

round4/X/aqua_rat/recipe_round4.json

@@ -0,0 +1,21 @@
+{
+  "round": "round4",
+  "model": "Qwen/Qwen2.5-3B-Instruct",
+  "task": "aqua_rat",
+  "lora_r": 16,
+  "lora_alpha": 32,
+  "targets": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "epochs": 3.0,
+  "train_per_task": 1500,
+  "lr": 0.0002,
+  "bs": 8,
+  "max_len": 512
+}

round4/X/aqua_rat/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

round4/X/aqua_rat/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962b8d8c521fefa934665afddae177326e974ddd6a26e69ff31ad6bccbb5593b
+size 11421994

round4/X/aqua_rat/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

round4/X/arc_challenge/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Qwen/Qwen2.5-3B-Instruct
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.13.2

round4/X/arc_challenge/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

round4/X/arc_challenge/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb4f248bcf1c56c56ad251498e69fd1c2565b226edabc8176f8c6316dafdf08f
+size 119801528

round4/X/arc_challenge/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

round4/X/arc_challenge/recipe_round4.json

@@ -0,0 +1,21 @@
+{
+  "round": "round4",
+  "model": "Qwen/Qwen2.5-3B-Instruct",
+  "task": "arc_challenge",
+  "lora_r": 16,
+  "lora_alpha": 32,
+  "targets": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "epochs": 3.0,
+  "train_per_task": 1500,
+  "lr": 0.0002,
+  "bs": 8,
+  "max_len": 512
+}

round4/X/arc_challenge/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

round4/X/arc_challenge/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962b8d8c521fefa934665afddae177326e974ddd6a26e69ff31ad6bccbb5593b
+size 11421994

round4/X/arc_challenge/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

round4/X/arc_easy/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Qwen/Qwen2.5-3B-Instruct
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.13.2

round4/X/arc_easy/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

round4/X/arc_easy/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3369fca8643a63f9ef2dc74c29ca88321118c357379d54d8ae86e6c30130011
+size 119801528

round4/X/arc_easy/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

round4/X/arc_easy/recipe_round4.json

@@ -0,0 +1,21 @@
+{
+  "round": "round4",
+  "model": "Qwen/Qwen2.5-3B-Instruct",
+  "task": "arc_easy",
+  "lora_r": 16,
+  "lora_alpha": 32,
+  "targets": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "epochs": 3.0,
+  "train_per_task": 1500,
+  "lr": 0.0002,
+  "bs": 8,
+  "max_len": 512
+}

round4/X/arc_easy/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

round4/X/arc_easy/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962b8d8c521fefa934665afddae177326e974ddd6a26e69ff31ad6bccbb5593b
+size 11421994

round4/X/arc_easy/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

round4/X/gsm8k/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Qwen/Qwen2.5-3B-Instruct
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.13.2

round4/X/gsm8k/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

round4/X/gsm8k/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a594d554338856ad98365776766a70174c874483265d676e976693a63717f82d
+size 119801528

round4/X/gsm8k/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

round4/X/gsm8k/recipe_round4.json

@@ -0,0 +1,21 @@
+{
+  "round": "round4",
+  "model": "Qwen/Qwen2.5-3B-Instruct",
+  "task": "gsm8k",
+  "lora_r": 16,
+  "lora_alpha": 32,
+  "targets": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "epochs": 3.0,
+  "train_per_task": 1500,
+  "lr": 0.0002,
+  "bs": 8,
+  "max_len": 512
+}

round4/X/gsm8k/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}