File size: 2,415 Bytes
63f41a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | {
"columns": [
"Eval set / metric",
"What it measures",
"Split-integrity rule",
"Statistics",
"Promotion threshold",
"Source",
"Confidence",
"Status"
],
"widths": [
36,
40,
48,
40,
36,
36,
10,
14
],
"rows": [
[
"miniF2F-valid (226, FILTERED: 13 unsupported, 4 elab failures, 1 overlap)",
"Contrast decisions K1-K4, averaging, stop rule",
"Driver refuses test-split scoring (test_guard); every selection logged",
"pass@8 dev; promotions confirmed with fresh-seed n=32",
"Heuristic gates only (+3 is not significant)",
"1c483cc; scripts/p2/expert_iteration.py",
"High",
"Done (built)"
],
[
"ProofNet-dev (to build)",
"Keeps selection from being miniF2F-only",
"Pinned, decontaminated",
"pass@8",
"-",
"docs/SOTA_PLAN.md Phase 0",
"Med",
"Planned"
],
[
"PRIMARY: miniF2F-test class A",
"Final candidate vs anchor 202/244",
"Touched once (--allow-test-selection --rounds 1)",
"n=k=32 solved/244; exact two-sided McNemar; wins/losses; paired CI; mean pass@1",
"Significant needs ~+9 net (losses 4 -> 13 wins) ~ 211/244; 207 is descriptive only",
"docs/SOTA_PLAN.md section 1",
"High",
"Planned"
],
[
"SECONDARY: class B miniF2F-test, class A ProofNet-test",
"Correction mode; transfer",
"Same harness both arms; trajectory ledger",
"Holm-corrected McNemar",
"-",
"docs/SOTA_PLAN.md section 1",
"High",
"Planned"
],
[
"EXPLORATORY: class C, PutnamBench class A pass@8",
"Agentic; hard transfer",
"Protocol tuple; compared only to agentic systems / own anchor",
"descriptive",
"-",
"docs/SOTA_PLAN.md section 1",
"Med",
"Planned"
],
[
"Verifier budget",
"Proof check wall-clock",
"300 s both arms; 1,200 s sensitivity; timeouts undecided",
"timeout counts per arm",
"-",
"1cc55e4",
"High",
"Done (protocol)"
],
[
"Rewritten-statement count",
"problem_check substitutes the reference statement",
"Reported per arm",
"count of accepted samples whose displayed statement differs",
"must not differ materially between arms",
"docs/P2.md 8c item 18",
"High",
"Planned"
],
[
"Verifier false-accept suite",
"Reward hacking resistance",
"Adversarial proofs never in training",
"0 accepts required",
"0/61 (P1); red-team 0 confirmed",
"654c20c",
"High",
"Running"
]
]
} |