Spaces:
Running
Running
File size: 3,575 Bytes
a6a5d8e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | {
"schemaVersion": 1,
"generatedBy": "manual-benchmark-evolution-v1",
"observedAt": "2026-05-30",
"canonicalRule": "A benchmark claim is active only when corpus digest, scoring rubric, judge panel, receipts, and CI validation are present. competition-math benchmark claims must be raw-score claims.",
"publication": {
"canonicalSource": "https://github.com/szl-holdings/a11oy",
"huggingFaceResultsRepo": "SZLHOLDINGS/a11oy-test-results",
"huggingFaceRepoType": "dataset",
"publishMode": "mirror-not-canonical"
},
"entries": [
{
"id": "mathcomp-raw-score-v1",
"family": "mathematics",
"status": "staged-no-corpus-digest",
"sourceEcosystem": [
"agi-forecast",
"lutar-lean",
"a11oy"
],
"corpus": {
"corpusId": "mathcomp-public-years",
"corpusVersion": "v1",
"immutability": "content-addressed",
"sourceUri": "external-official-or-licensed-source",
"license": "do-not-redistribute-unless-permitted",
"canonicalization": "utf8-nfc-jsonl-sorted-keys",
"digestStatus": "external-digest-required-before-scoring",
"problemCount": 0,
"sealed": false
},
"scoring": {
"scoreType": "raw_points",
"maxRawScore": 120,
"perProblemMax": 10,
"partialCreditAllowed": true,
"reportFields": [
"year",
"problem",
"earned",
"possible",
"judgeIds",
"disagreement"
]
},
"honesty": {
"allowedClaim": "staged competition-math raw-score harness design; no live score claimed",
"disallowedClaims": [
"solved the benchmark",
"beat the benchmark",
"AGI proven"
],
"headlineThresholdRequiresUnanimousJudges": true
},
"formulaRoutes": [
{
"formula": "FalsePosition",
"theoremRuntimeManifestId": "RUNTIME-FALSE-POSITION",
"allowedUse": "routing-or-verification-when-problem-matches"
},
{
"formula": "MadhavaBound",
"theoremRuntimeManifestId": "RUNTIME-MADHAVA-BOUND",
"allowedUse": "numeric-bound-check-only"
},
{
"formula": "LiuHuiPi",
"theoremRuntimeManifestId": "RUNTIME-LIU-HUI-PI",
"allowedUse": "numeric-approximation-check-only"
},
{
"formula": "SummationInvariant",
"theoremRuntimeManifestId": "RUNTIME-SUMMATION-INVARIANT",
"allowedUse": "additive-invariant-check-only"
},
{
"formula": "QECLineage",
"theoremRuntimeManifestId": "RUNTIME-QEC-LINEAGE",
"allowedUse": "receipt-integrity-not-math-credit"
}
],
"judges": [
{
"id": "raw_grader",
"role": "rubric_score",
"required": true
},
{
"id": "proof_judge",
"role": "proof_validity",
"required": true
},
{
"id": "provenance_judge",
"role": "receipt_and_claim_guard",
"required": true
}
],
"receipts": {
"required": true,
"ledgerPathPattern": "benchmarks/results/{runId}/receipts.jsonl",
"chain": "hash_chain",
"quorum": "2-of-3"
},
"ciGates": [
"validate-benchmark-map",
"verify-corpus-digests",
"verify-receipt-chain",
"verify-formula-routes",
"reject-unsupported-benchmark-claims"
]
}
]
}
|