CBE / data /models.json
jizej's picture
CBE Results Explorer: public results page
edc830f verified
Raw History Blame Contribute Delete
5.26 kB
[{"model":"GPT-4.1","group":"API models","acc_original":0.9306,"no_answer_rate":0.0006,"acc_equivalent":0.8983,"both_correct":0.8533,"agreement":0.8778},{"model":"Qwen3.5-27B (thinking)","group":"Open-weight models","acc_original":0.87,"no_answer_rate":0.1678,"acc_equivalent":0.7911,"both_correct":0.7544,"agreement":0.7544,"note":"Thinking enabled (24k-token budget), 100 item pairs per size. Unfinished answers count as wrong here."},{"model":"GPT-5","group":"API models","acc_original":0.9411,"no_answer_rate":0.0022,"acc_equivalent":0.8706,"both_correct":0.8328,"agreement":0.8533},{"model":"Gemma-4-31B","group":"Open-weight models","acc_original":0.8456,"no_answer_rate":0.1803,"acc_equivalent":0.7506,"both_correct":0.6967,"agreement":0.6994,"note":"Often starts reasoning instead of answering under the 20-token answer cap at large n; those count as wrong here."},{"model":"Qwen3.5-9B + GRPO-IG 位=0.5 (seed 42)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.8344,"no_answer_rate":0.0006,"acc_equivalent":0.7622,"both_correct":0.6739,"agreement":0.7511,"note":"The paper reports GRPO-IG as means over twelve held-out relations plus a front/behind range (Appendix G)."},{"model":"Qwen2.5-32B-Instruct","group":"Open-weight models","acc_original":0.8267,"no_answer_rate":0.0,"acc_equivalent":0.745,"both_correct":0.6856,"agreement":0.7994},{"model":"Qwen3.5-9B + GRPO (seed 42)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.8122,"no_answer_rate":0.0,"acc_equivalent":0.7506,"both_correct":0.6478,"agreement":0.7328},{"model":"Qwen3.5-9B + GRPO-IG 位=0.5 (seed 43)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.8094,"no_answer_rate":0.0008,"acc_equivalent":0.7611,"both_correct":0.62,"agreement":0.6689,"note":"The paper reports GRPO-IG as means over twelve held-out relations plus a front/behind range (Appendix G)."},{"model":"Qwen3.5-9B + GRPO-IG 位=0 (seed 42)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.7944,"no_answer_rate":0.0006,"acc_equivalent":0.7433,"both_correct":0.6183,"agreement":0.6983,"note":"The paper reports GRPO-IG as means over twelve held-out relations plus a front/behind range (Appendix G)."},{"model":"Qwen3.5-27B","group":"Open-weight models","acc_original":0.8139,"no_answer_rate":0.0,"acc_equivalent":0.7294,"both_correct":0.6322,"agreement":0.7211},{"model":"GPT-5.4 (effort none)","group":"API models","acc_original":0.7711,"no_answer_rate":0.0,"acc_equivalent":0.7489,"both_correct":0.6506,"agreement":0.7811,"note":"Reasoning effort none. Its reworded results here use the base rewording (paper, Appendix B); the paper's reasoning-effort table uses a different rewording."},{"model":"Qwen3.5-9B + GRPO (seed 43)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.7811,"no_answer_rate":0.0,"acc_equivalent":0.7672,"both_correct":0.6106,"agreement":0.6728},{"model":"Qwen3.5-9B + GRPO-IG 位=0 (seed 43)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.785,"no_answer_rate":0.0,"acc_equivalent":0.7456,"both_correct":0.5933,"agreement":0.6561,"note":"The paper reports GRPO-IG as means over twelve held-out relations plus a front/behind range (Appendix G)."},{"model":"GPT-4.1-mini","group":"API models","acc_original":0.7589,"no_answer_rate":0.0,"acc_equivalent":0.7139,"both_correct":0.5989,"agreement":0.725},{"model":"GPT-5-mini","group":"API models","acc_original":0.7,"no_answer_rate":0.0,"acc_equivalent":0.72,"both_correct":0.5456,"agreement":0.6711},{"model":"Qwen3.6-35B-A3B","group":"Open-weight models","acc_original":0.6439,"no_answer_rate":0.0003,"acc_equivalent":0.6094,"both_correct":0.4256,"agreement":0.5978},{"model":"Qwen3.5-9B + SFT (pairs)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.545,"no_answer_rate":0.0022,"acc_equivalent":0.5661,"both_correct":0.345,"agreement":0.5772},{"model":"Qwen3.5-9B + SFT (cross-relation)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.5639,"no_answer_rate":0.0022,"acc_equivalent":0.55,"both_correct":0.3394,"agreement":0.5633},{"model":"Qwen3.5-9B + SFT (pairs + JS)","group":"Fine-tuned Qwen3.5-9B","acc_original":0.5617,"no_answer_rate":0.0039,"acc_equivalent":0.545,"both_correct":0.3444,"agreement":0.5789},{"model":"Qwen3.5-9B","group":"Open-weight models","acc_original":0.5561,"no_answer_rate":0.0014,"acc_equivalent":0.5328,"both_correct":0.3311,"agreement":0.5717},{"model":"GPT-5.4 (effort high)","group":"API models","acc_original":0.9989,"no_answer_rate":0.0,"acc_equivalent":null,"both_correct":null,"agreement":null,"note":"Reasoning effort high. Original wording only on this page; its reworded results use a different rewording and appear in the paper's reasoning-effort table."},{"model":"GPT-5.4 (effort medium)","group":"API models","acc_original":0.9944,"no_answer_rate":0.0,"acc_equivalent":null,"both_correct":null,"agreement":null,"note":"Reasoning effort medium. Original wording only on this page; its reworded results use a different rewording and appear in the paper's reasoning-effort table."},{"model":"GPT-5.4 (effort low)","group":"API models","acc_original":0.9772,"no_answer_rate":0.0,"acc_equivalent":null,"both_correct":null,"agreement":null,"note":"Reasoning effort low. Original wording only on this page; its reworded results use a different rewording and appear in the paper's reasoning-effort table."}]