File size: 3,391 Bytes
13a1073 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | #!/usr/bin/env python3
"""Validate the frozen paper evaluation files without requiring source sets."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
EXPECTED = {
"data/evaluation/metric_qa/requests_paper_um2000.jsonl": {
"rows": 2000,
"sha256": "5c6fa80d160c587cc0a2f7da5c92fd8aa1472aaac40ac45be487fcb10f1093df",
},
"data/evaluation/caption/requests_paper_um2000.jsonl": {
"rows": 2000,
"sha256": "c6e27dae5461da60e74962e4ec45597534687303530eb82ea60c999895019e2d",
},
"data/evaluation/caption/hidden_gt_paper_um2000.jsonl": {
"rows": 2000,
"sha256": "1fc8ffa3966a7d8d14284cb5b79c286ab8469db3527a6fff94578d45045c8647",
},
"data/evaluation/tsqa/series_overlap_audit.json": {
"sha256": "e568e2b01799ec83f48f499ca36cf748ab44b7baba65c4c01e085c653ec18ccb",
},
"data/evaluation/tsqa/requests_clean3264.jsonl": {
"rows": 3264,
"sha256": "fc48b3447fb8d52b1f6d90a7770ba9d0f8e57ae61225f99a8c43cf9795f259b7",
},
}
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def read_ids(path: Path) -> list[str]:
return [
str(json.loads(line)["request_id"])
for line in path.read_text(encoding="utf-8").splitlines()
if line.strip()
]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo-root", type=Path, required=True)
args = parser.parse_args()
repo = args.repo_root.resolve()
for relative, expected in EXPECTED.items():
path = repo / relative
if not path.is_file():
raise FileNotFoundError(path)
actual_hash = sha256(path)
if actual_hash != expected["sha256"]:
raise RuntimeError(
f"{relative}: expected SHA256 {expected['sha256']}, found {actual_hash}"
)
if "rows" in expected:
with path.open("rb") as handle:
rows = sum(1 for line in handle if line.strip())
if rows != expected["rows"]:
raise RuntimeError(
f"{relative}: expected {expected['rows']} rows, found {rows}"
)
caption_requests = read_ids(
repo / "data/evaluation/caption/requests_paper_um2000.jsonl"
)
caption_hidden = read_ids(
repo / "data/evaluation/caption/hidden_gt_paper_um2000.jsonl"
)
if caption_requests != caption_hidden:
raise RuntimeError("Caption request and hidden-GT ID order differs")
audit = json.loads(
(repo / "data/evaluation/tsqa/series_overlap_audit.json").read_text(
encoding="utf-8"
)
)
excluded = {str(item) for item in audit["affected_test_request_ids"]}
clean_ids = read_ids(repo / "data/evaluation/tsqa/requests_clean3264.jsonl")
if len(excluded) != 255 or len(clean_ids) != len(set(clean_ids)):
raise RuntimeError("Invalid TSQA exclusion audit or clean request IDs")
if set(clean_ids) & excluded:
raise RuntimeError("TSQA clean requests contain an excluded request ID")
print("Frozen paper evaluation data verified.")
if __name__ == "__main__":
main()
|