{ "fixture": { "purpose": "End-to-end replay of the existing Figure 1 family, selected by its role in the paper; uses both models and arms at base five, without choosing outputs by success.", "sample_id": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e", "base_count": 5, "reference_rule_pairs": [ [ "ex", "azw" ], [ "wa", "y" ] ], "task_pair": { "CONTROL": { "inputs": [ "gud", "czua", "bwcied", "dzexa", "yf", "iiiexa" ], "outputs": [ "gud", "czua", "bwcied", "dzazy", "yf", "iiiazy" ] }, "SEPARATION": { "inputs": [ "gud", "czua", "bwcied", "dzexa", "yf", "wbawex" ], "outputs": [ "gud", "czua", "bwcied", "dzazy", "yf", "wbayzw" ] } }, "request_ids": [ "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL", "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION", "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL", "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION" ], "expected": { "status": "PASS", "family": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e", "base_count": 5, "certificates": { "CONTROL": { "complete_one_rule_solutions": [ [ "exa", "azy" ] ], "minimum_length": 1, "two_rule_reference_passes": true }, "SEPARATION": { "complete_one_rule_solutions": [], "minimum_length": 2, "two_rule_reference_passes": true } }, "saved_outputs": { "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": { "selected_rules": null, "legal_selected_program": false, "targets": { "displayed": { "hits": 0, "inputs": 6, "all_pass": false }, "common_nine": { "hits": 0, "inputs": 9, "all_pass": false }, "wholly_unshown_pair": { "hits": 0, "inputs": 2, "all_pass": false }, "uniform": { "hits": 0, "inputs": 32, "all_pass": false }, "active": { "hits": 0, "inputs": 32, "all_pass": false } } }, "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": { "selected_rules": [ [ "a", "y" ], [ "e", "a" ], [ "x", "z" ] ], "legal_selected_program": true, "targets": { "displayed": { "hits": 4, "inputs": 6, "all_pass": false }, "common_nine": { "hits": 5, "inputs": 9, "all_pass": false }, "wholly_unshown_pair": { "hits": 1, "inputs": 2, "all_pass": false }, "uniform": { "hits": 11, "inputs": 32, "all_pass": false }, "active": { "hits": 5, "inputs": 32, "all_pass": false } } }, "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": { "selected_rules": [ [ "exa", "azy" ], [ "wex", "yzw" ] ], "legal_selected_program": true, "targets": { "displayed": { "hits": 6, "inputs": 6, "all_pass": true }, "common_nine": { "hits": 9, "inputs": 9, "all_pass": true }, "wholly_unshown_pair": { "hits": 2, "inputs": 2, "all_pass": true }, "uniform": { "hits": 32, "inputs": 32, "all_pass": true }, "active": { "hits": 0, "inputs": 32, "all_pass": false } } }, "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": { "selected_rules": null, "legal_selected_program": false, "targets": { "displayed": { "hits": 0, "inputs": 6, "all_pass": false }, "common_nine": { "hits": 0, "inputs": 9, "all_pass": false }, "wholly_unshown_pair": { "hits": 0, "inputs": 2, "all_pass": false }, "uniform": { "hits": 0, "inputs": 32, "all_pass": false }, "active": { "hits": 0, "inputs": 32, "all_pass": false } } } }, "model_inference_calls": 0 } }, "panel": [ { "model": "Qwen3-8B", "base_count": "3", "displayed_examples": "4", "primary_family_N": "119", "CONTROL_displayed_successes": "59", "SEPARATION_displayed_successes": "15", "displayed_delta_pp_and_95CI": "-36.97 [-46.22,-27.73]", "unshown_pair_delta_pp_and_95CI": "+13.45 [+7.56,+20.17]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-36.67 [-45.83,-27.50]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" }, { "model": "Qwen3-8B", "base_count": "4", "displayed_examples": "5", "primary_family_N": "119", "CONTROL_displayed_successes": "61", "SEPARATION_displayed_successes": "15", "displayed_delta_pp_and_95CI": "-38.66 [-47.90,-28.57]", "unshown_pair_delta_pp_and_95CI": "+10.92 [+5.88,+16.81]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-39.17 [-48.33,-29.17]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" }, { "model": "Qwen3-8B", "base_count": "5", "displayed_examples": "6", "primary_family_N": "120", "CONTROL_displayed_successes": "67", "SEPARATION_displayed_successes": "12", "displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]", "unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" }, { "model": "Qwen3-8B", "base_count": "6", "displayed_examples": "7", "primary_family_N": "120", "CONTROL_displayed_successes": "68", "SEPARATION_displayed_successes": "19", "displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]", "unshown_pair_delta_pp_and_95CI": "+12.50 [+6.67,+18.33]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" }, { "model": "Olmo-3-7B-Think", "base_count": "3", "displayed_examples": "4", "primary_family_N": "119", "CONTROL_displayed_successes": "53", "SEPARATION_displayed_successes": "14", "displayed_delta_pp_and_95CI": "-32.77 [-42.86,-22.69]", "unshown_pair_delta_pp_and_95CI": "+8.40 [+3.36,+13.45]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-33.33 [-43.33,-23.33]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" }, { "model": "Olmo-3-7B-Think", "base_count": "4", "displayed_examples": "5", "primary_family_N": "119", "CONTROL_displayed_successes": "47", "SEPARATION_displayed_successes": "19", "displayed_delta_pp_and_95CI": "-23.53 [-34.45,-12.61]", "unshown_pair_delta_pp_and_95CI": "+11.76 [+5.88,+17.65]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-23.33 [-34.17,-12.50]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" }, { "model": "Olmo-3-7B-Think", "base_count": "5", "displayed_examples": "6", "primary_family_N": "120", "CONTROL_displayed_successes": "52", "SEPARATION_displayed_successes": "15", "displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]", "unshown_pair_delta_pp_and_95CI": "+11.67 [+6.67,+17.50]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" }, { "model": "Olmo-3-7B-Think", "base_count": "6", "displayed_examples": "7", "primary_family_N": "120", "CONTROL_displayed_successes": "54", "SEPARATION_displayed_successes": "13", "displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]", "unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]", "all_legal_N": "120", "all_legal_displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]", "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED" } ] }