| { |
| "claims": { |
| "claim1": { |
| "implementation": { |
| "basic_lean_sha256": "a908c060d1031506ac5844b838b819391a001474eee88a5df184fa9bdac97dc4", |
| "solving_server_sha256": "76a8760c1651b63d33d26c1d0097896f61881210efbfc576811ba79f38bd474b" |
| }, |
| "mdp": { |
| "actions": 8, |
| "deterministic_violations": 0, |
| "hidden_history_mutant_detected": true, |
| "repeated_transition_comparisons": 8192, |
| "states": 256, |
| "terminal_reward_hits": 16 |
| } |
| }, |
| "claim2": { |
| "backward_sound_pairs": 3280, |
| "by_domain": [ |
| { |
| "backward_sound": 1, |
| "forward_complete": 1, |
| "iff": 1, |
| "n": 0, |
| "pairs": 1 |
| }, |
| { |
| "backward_sound": 3, |
| "forward_complete": 3, |
| "iff": 2, |
| "n": 1, |
| "pairs": 4 |
| }, |
| { |
| "backward_sound": 9, |
| "forward_complete": 9, |
| "iff": 4, |
| "n": 2, |
| "pairs": 16 |
| }, |
| { |
| "backward_sound": 27, |
| "forward_complete": 27, |
| "iff": 8, |
| "n": 3, |
| "pairs": 64 |
| }, |
| { |
| "backward_sound": 81, |
| "forward_complete": 81, |
| "iff": 16, |
| "n": 4, |
| "pairs": 256 |
| }, |
| { |
| "backward_sound": 243, |
| "forward_complete": 243, |
| "iff": 32, |
| "n": 5, |
| "pairs": 1024 |
| }, |
| { |
| "backward_sound": 729, |
| "forward_complete": 729, |
| "iff": 64, |
| "n": 6, |
| "pairs": 4096 |
| }, |
| { |
| "backward_sound": 2187, |
| "forward_complete": 2187, |
| "iff": 128, |
| "n": 7, |
| "pairs": 16384 |
| } |
| ], |
| "complete_and_sound_pairs": 255, |
| "domains": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7 |
| ], |
| "expected_closed_forms": { |
| "iff": 255, |
| "one_direction": 3280, |
| "pairs": 21845 |
| }, |
| "forward_complete_pairs": 3280, |
| "forward_only_complete_but_unsound": 3025, |
| "predicate_pairs": 21845 |
| }, |
| "claim3": { |
| "accepted_true_memberships": 4097, |
| "answer_predicate_cases": 8194, |
| "by_domain": [ |
| { |
| "accepted": 1, |
| "answer_predicate_cases": 2, |
| "n": 1, |
| "rejected": 1 |
| }, |
| { |
| "accepted": 4, |
| "answer_predicate_cases": 8, |
| "n": 2, |
| "rejected": 4 |
| }, |
| { |
| "accepted": 12, |
| "answer_predicate_cases": 24, |
| "n": 3, |
| "rejected": 12 |
| }, |
| { |
| "accepted": 32, |
| "answer_predicate_cases": 64, |
| "n": 4, |
| "rejected": 32 |
| }, |
| { |
| "accepted": 80, |
| "answer_predicate_cases": 160, |
| "n": 5, |
| "rejected": 80 |
| }, |
| { |
| "accepted": 192, |
| "answer_predicate_cases": 384, |
| "n": 6, |
| "rejected": 192 |
| }, |
| { |
| "accepted": 448, |
| "answer_predicate_cases": 896, |
| "n": 7, |
| "rejected": 448 |
| }, |
| { |
| "accepted": 1024, |
| "answer_predicate_cases": 2048, |
| "n": 8, |
| "rejected": 1024 |
| }, |
| { |
| "accepted": 2304, |
| "answer_predicate_cases": 4608, |
| "n": 9, |
| "rejected": 2304 |
| } |
| ], |
| "domains": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9 |
| ], |
| "proof_field_omission_mutant_false_accepts": 4097, |
| "rejected_false_memberships": 4097, |
| "soundness_violations": 0 |
| }, |
| "claim4": { |
| "complete_and_sound_pairs": 255, |
| "complete_pairs": 3280, |
| "one_direction_only_invalid_pairs": 3025, |
| "sound_pairs": 3280 |
| }, |
| "claim5": { |
| "benchmark_counts": { |
| "formal_math500": 387, |
| "minif2f_solving": 375, |
| "putnam_solving": 324 |
| }, |
| "details": [ |
| { |
| "benchmark": "formal_math500", |
| "byte_mismatches": [], |
| "records": 387 |
| }, |
| { |
| "benchmark": "minif2f_solving", |
| "byte_mismatches": [], |
| "records": 375 |
| }, |
| { |
| "benchmark": "putnam_solving", |
| "byte_mismatches": [], |
| "records": 324 |
| } |
| ], |
| "export_mutant_detected": true, |
| "lean_files_expected": 1086, |
| "lean_files_regenerated_byte_exact": 1086, |
| "records_total": 1086, |
| "records_with_required_schema": 1086, |
| "unique_global_fingerprints": 1086, |
| "unique_within_benchmarks": 1086 |
| }, |
| "claim6": { |
| "expected_maxima": { |
| "formal_math500": { |
| "denominator": 387, |
| "percent": 23.77, |
| "solved": 92 |
| }, |
| "minif2f_solving": { |
| "denominator": 375, |
| "percent": 27.47, |
| "solved": 103 |
| }, |
| "putnam_solving": { |
| "denominator": 324, |
| "percent": 0.31, |
| "solved": 1 |
| } |
| }, |
| "files": 18, |
| "maxima": { |
| "formal_math500": { |
| "denominator": 387, |
| "method": "proof_search/internlm2.5-step-prover", |
| "percent": 23.77, |
| "solved": 92 |
| }, |
| "minif2f_solving": { |
| "denominator": 375, |
| "method": "proof_search/internlm2.5-step-prover", |
| "percent": 27.47, |
| "solved": 103 |
| }, |
| "putnam_solving": { |
| "denominator": 324, |
| "method": "whole_proof_generation/deepseek-prover-v1.5", |
| "percent": 0.31, |
| "solved": 1 |
| } |
| }, |
| "method_problem_records_present": 5779, |
| "method_problem_slots": 6516, |
| "methods": [ |
| { |
| "benchmark": "formal_math500", |
| "denominator": 387, |
| "method": "prompting_methods/hybrid_cot", |
| "records_present": 387, |
| "solved": 60, |
| "solved_percent": 15.503875968992, |
| "submitted_without_equivalence": 4 |
| }, |
| { |
| "benchmark": "minif2f_solving", |
| "denominator": 375, |
| "method": "prompting_methods/hybrid_cot", |
| "records_present": 375, |
| "solved": 81, |
| "solved_percent": 21.6, |
| "submitted_without_equivalence": 0 |
| }, |
| { |
| "benchmark": "putnam_solving", |
| "denominator": 324, |
| "method": "prompting_methods/hybrid_cot", |
| "records_present": 324, |
| "solved": 0, |
| "solved_percent": 0.0, |
| "submitted_without_equivalence": 1 |
| }, |
| { |
| "benchmark": "formal_math500", |
| "denominator": 387, |
| "method": "prompting_methods/icl", |
| "records_present": 387, |
| "solved": 53, |
| "solved_percent": 13.695090439276, |
| "submitted_without_equivalence": 0 |
| }, |
| { |
| "benchmark": "minif2f_solving", |
| "denominator": 375, |
| "method": "prompting_methods/icl", |
| "records_present": 375, |
| "solved": 82, |
| "solved_percent": 21.866666666667, |
| "submitted_without_equivalence": 0 |
| }, |
| { |
| "benchmark": "putnam_solving", |
| "denominator": 324, |
| "method": "prompting_methods/icl", |
| "records_present": 324, |
| "solved": 0, |
| "solved_percent": 0.0, |
| "submitted_without_equivalence": 0 |
| }, |
| { |
| "benchmark": "formal_math500", |
| "denominator": 387, |
| "method": "proof_search/internlm2.5-step-prover", |
| "records_present": 387, |
| "solved": 92, |
| "solved_percent": 23.772609819121, |
| "submitted_without_equivalence": 75 |
| }, |
| { |
| "benchmark": "minif2f_solving", |
| "denominator": 375, |
| "method": "proof_search/internlm2.5-step-prover", |
| "records_present": 375, |
| "solved": 103, |
| "solved_percent": 27.466666666667, |
| "submitted_without_equivalence": 51 |
| }, |
| { |
| "benchmark": "putnam_solving", |
| "denominator": 324, |
| "method": "proof_search/internlm2.5-step-prover", |
| "records_present": 324, |
| "solved": 0, |
| "solved_percent": 0.0, |
| "submitted_without_equivalence": 91 |
| }, |
| { |
| "benchmark": "formal_math500", |
| "denominator": 387, |
| "method": "proof_search/leanstar", |
| "records_present": 378, |
| "solved": 91, |
| "solved_percent": 23.514211886305, |
| "submitted_without_equivalence": 81 |
| }, |
| { |
| "benchmark": "minif2f_solving", |
| "denominator": 375, |
| "method": "proof_search/leanstar", |
| "records_present": 370, |
| "solved": 91, |
| "solved_percent": 24.266666666667, |
| "submitted_without_equivalence": 54 |
| }, |
| { |
| "benchmark": "putnam_solving", |
| "denominator": 324, |
| "method": "proof_search/leanstar", |
| "records_present": 324, |
| "solved": 0, |
| "solved_percent": 0.0, |
| "submitted_without_equivalence": 133 |
| }, |
| { |
| "benchmark": "formal_math500", |
| "denominator": 387, |
| "method": "whole_proof_generation/deepseek-prover-v1.5", |
| "records_present": 147, |
| "solved": 86, |
| "solved_percent": 22.222222222222, |
| "submitted_without_equivalence": 56 |
| }, |
| { |
| "benchmark": "minif2f_solving", |
| "denominator": 375, |
| "method": "whole_proof_generation/deepseek-prover-v1.5", |
| "records_present": 128, |
| "solved": 84, |
| "solved_percent": 22.4, |
| "submitted_without_equivalence": 41 |
| }, |
| { |
| "benchmark": "putnam_solving", |
| "denominator": 324, |
| "method": "whole_proof_generation/deepseek-prover-v1.5", |
| "records_present": 102, |
| "solved": 1, |
| "solved_percent": 0.308641975309, |
| "submitted_without_equivalence": 72 |
| }, |
| { |
| "benchmark": "formal_math500", |
| "denominator": 387, |
| "method": "whole_proof_generation/theoremllama", |
| "records_present": 378, |
| "solved": 62, |
| "solved_percent": 16.020671834625, |
| "submitted_without_equivalence": 60 |
| }, |
| { |
| "benchmark": "minif2f_solving", |
| "denominator": 375, |
| "method": "whole_proof_generation/theoremllama", |
| "records_present": 370, |
| "solved": 49, |
| "solved_percent": 13.066666666667, |
| "submitted_without_equivalence": 33 |
| }, |
| { |
| "benchmark": "putnam_solving", |
| "denominator": 324, |
| "method": "whole_proof_generation/theoremllama", |
| "records_present": 324, |
| "solved": 0, |
| "solved_percent": 0.0, |
| "submitted_without_equivalence": 54 |
| } |
| ], |
| "missing_records_counted_unsolved": 737, |
| "solving_rule_mutant_detected": true, |
| "submission_only_mutant_maxima": { |
| "formal_math500": 172, |
| "minif2f_solving": 154, |
| "putnam_solving": 133 |
| } |
| } |
| }, |
| "gates": [ |
| { |
| "name": "pinned_input_hashes", |
| "passed": true |
| }, |
| { |
| "name": "paper_and_code_tokens", |
| "passed": true |
| }, |
| { |
| "name": "mdp_exhaustive_determinism", |
| "passed": true |
| }, |
| { |
| "name": "mdp_hidden_history_control", |
| "passed": true |
| }, |
| { |
| "name": "dfps_all_predicate_pairs", |
| "passed": true |
| }, |
| { |
| "name": "dfps_forward_completeness", |
| "passed": true |
| }, |
| { |
| "name": "dfps_optional_backward_boundary", |
| "passed": true |
| }, |
| { |
| "name": "dfps_full_iff", |
| "passed": true |
| }, |
| { |
| "name": "dfps_closed_form_oracle", |
| "passed": true |
| }, |
| { |
| "name": "fps_exhaustive_cases", |
| "passed": true |
| }, |
| { |
| "name": "fps_soundness", |
| "passed": true |
| }, |
| { |
| "name": "fps_proof_field_control", |
| "passed": true |
| }, |
| { |
| "name": "benchmark_counts", |
| "passed": true |
| }, |
| { |
| "name": "benchmark_schema", |
| "passed": true |
| }, |
| { |
| "name": "benchmark_uniqueness", |
| "passed": true |
| }, |
| { |
| "name": "official_export_byte_identity", |
| "passed": true |
| }, |
| { |
| "name": "export_mutation_control", |
| "passed": true |
| }, |
| { |
| "name": "initial_baseline_files", |
| "passed": true |
| }, |
| { |
| "name": "initial_baseline_records", |
| "passed": true |
| }, |
| { |
| "name": "table_1_exact_maxima", |
| "passed": true |
| }, |
| { |
| "name": "table_1_solved_rule_control", |
| "passed": true |
| } |
| ], |
| "input_sha256": { |
| "current_archive": "b144a1051b840f4b7f59428151ba8bf6f27e7665a3b9f950d165c39055ef2315", |
| "initial_archive": "cef6b9340b797355ab6e3fb425ee02a1e6c2cc7b76eff6fc77d79bf6e99753fe", |
| "paper_pdf": "bba3d0293dda50476bd07d7dab987d4ed5ded4fa21132837f35c4feff72403e3", |
| "paper_source": "b08c84c35b90d0128fc6ff512b60b07ead29ef6de847c8834948df660b00034a" |
| }, |
| "paper_id": "hgMZraPlSv", |
| "scope": { |
| "current_official_commit": "3e8bafd18a080fe01263ff93b9c3db78d993e1a1", |
| "initial_release_commit": "39489d1f0c32b521845429e1cb26c48639d8f823", |
| "lean_compilation_executed": false, |
| "lean_scope_note": "Lean 4 was not installed in this environment. The exact Lean/Pantograph implementation and proof source are pinned and structurally audited; finite semantics, all benchmark exports, and all archived Table-1 records are executed independently." |
| }, |
| "source_audits": [ |
| { |
| "artifact": "paper/method.tex", |
| "missing": [], |
| "required_tokens": 6 |
| }, |
| { |
| "artifact": "paper/appendix.proofs.tex", |
| "missing": [], |
| "required_tokens": 4 |
| }, |
| { |
| "artifact": "official/Basic.lean", |
| "missing": [], |
| "required_tokens": 7 |
| }, |
| { |
| "artifact": "official/solving_server.py", |
| "missing": [], |
| "required_tokens": 4 |
| }, |
| { |
| "artifact": "paper/evaluation.tex", |
| "missing": [], |
| "required_tokens": 4 |
| }, |
| { |
| "artifact": "paper/experiment.tex", |
| "missing": [], |
| "required_tokens": 3 |
| }, |
| { |
| "artifact": "paper/main.tex", |
| "missing": [], |
| "required_tokens": 4 |
| } |
| ], |
| "summary": { |
| "all_passed": true, |
| "passed": 21, |
| "total": 21 |
| } |
| } |
|
|
