File size: 7,046 Bytes
ed3aeeb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | #!/usr/bin/env python3
"""Independently validate the compact 21-model accuracy CSV."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import os
import tempfile
from collections import Counter
from pathlib import Path
from typing import Any
FIELDS = ["model_id", "model_name", "metric", "fp32", "quantized", "delta", "published_comparison"]
EXPECTED: dict[str, tuple[str, str, str, str, str]] = {
"AD01": ("AUC / pAUC (max_fpr=0.1)", "0.876001 / 0.764121", "0.840250 / 0.720049", "-0.035750 / -0.044071", "공개 FP32 AUC/pAUC 0.876001/0.764121; 측정값 동일"),
"LM04": ("AUROC / TPR@FPR 5% / 1% (RAID-extra OOD)", "0.667078 / 0.318983 / 0.233873", "0.668862 / 0.322615 / 0.235853", "+0.001785 / +0.003632 / +0.001981", ""),
"OD06": ("COCO bbox mAP (AP@[0.50:0.95])", "24.8751%", "24.2822%", "-0.5929 pp", ""),
"OD07": ("COCO bbox mAP (AP@[0.50:0.95])", "31.8594%", "31.4191%", "-0.4403 pp", ""),
"SG06": ("mIoU", "75.6398%", "74.1290%", "-1.5108 pp", "공개 FP32 mIoU 75.32%; 측정 75.6398% (+0.3198 pp)"),
"SG07": ("mIoU", "70.6473%", "69.6191%", "-1.0282 pp", "공개 FP32 mIoU 70.19%; 측정 70.6473% (+0.4573 pp)"),
"SG08": ("mIoU (CamVid cross-dataset)", "50.6498%", "51.1600%", "+0.5102 pp", ""),
"SP01": ("Top-1 accuracy", "91.86%", "91.66%", "-0.2045 pp", "공개 시험 정확도 약 92%; 측정 FP32 91.86%, 양자화 91.66% (근접)"),
"SP02": ("FP / FN (1 s)", "5 / 6", "4 / 6", "-1 / +0", ""),
"SP08": ("Top-1 accuracy (yes/no subset)", "94.05%", "94.05%", "+0.0000 pp", ""),
"SP09": ("Top-1 accuracy", "95.06%", "94.70%", "-0.3590 pp", ""),
"VC01": ("Top-1 accuracy", "85.10%", "85.60%", "+0.5000 pp", ""),
"VC02": ("Top-1 accuracy", "87.00%", "87.00%", "+0.0000 pp", ""),
"VC03": ("Top-1 / Top-5 accuracy", "49.80% / 74.20%", "48.00% / 72.80%", "-1.8000 pp / -1.4000 pp", ""),
"VC04": ("Top-1 / Top-5 accuracy", "63.30% / 84.90%", "60.70% / 83.20%", "-2.6000 pp / -1.7000 pp", ""),
"VC05": ("Top-1 accuracy", "58.13%", "56.77%", "-1.3600 pp", ""),
"VC06": ("Top-1 accuracy", "66.20%", "65.31%", "-0.8900 pp", ""),
"VC09": ("Top-1 / Top-5 accuracy", "56.85% / 79.87%", "56.48% / 79.76%", "-0.3700 pp / -0.1100 pp", ""),
"VC11": ("Top-1 accuracy", "75.10%", "74.40%", "-0.7000 pp", ""),
"VC12": ("Top-1 / Top-5 accuracy", "69.48% / 89.26%", "68.30% / 88.44%", "-1.1800 pp / -0.8200 pp", ""),
"VC13": ("Top-1 / Top-5 error", "33.65% / 13.43%", "33.85% / 13.66%", "+0.2000 pp / +0.2300 pp", ""),
}
NO_PUBLISHED_VALUE = "공개 수치 없음."
EXPECTED = {
model_id: (*values[:4], values[4] or NO_PUBLISHED_VALUE)
for model_id, values in EXPECTED.items()
}
class Checks:
def __init__(self) -> None:
self.total = 0
self.failures: list[dict[str, str]] = []
self.categories: Counter[str] = Counter()
def check(self, condition: bool, category: str, detail: str) -> None:
self.total += 1
self.categories[category] += 1
if not condition:
self.failures.append({"category": category, "detail": detail})
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def atomic_json(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=path.parent)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
json.dump(value, handle, indent=2, sort_keys=True, ensure_ascii=False)
handle.write("\n")
os.replace(temporary, path)
except BaseException:
Path(temporary).unlink(missing_ok=True)
raise
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo-root", type=Path, required=True)
parser.add_argument("--csv", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
root = args.repo_root.resolve()
csv_path = args.csv if args.csv.is_absolute() else root / args.csv
output = args.output if args.output.is_absolute() else root / args.output
checks = Checks()
with csv_path.open(newline="", encoding="utf-8") as handle:
reader = csv.DictReader(handle)
checks.check(reader.fieldnames == FIELDS, "schema", str(reader.fieldnames))
rows = list(reader)
checks.check(len(rows) == 21, "coverage", f"rows={len(rows)}")
by_id = {row["model_id"]: row for row in rows}
checks.check(len(by_id) == len(rows), "coverage", "duplicate model ID")
checks.check(set(by_id) == set(EXPECTED), "coverage", "model ID set differs")
registry = {
row["model_id"]: row for row in csv.DictReader(
(root / "model_registry.csv").open(newline="", encoding="utf-8")
) if row["eligibility"] == "ELIGIBLE"
}
checks.check(set(registry) == set(EXPECTED), "registry", "active registry set differs")
for model_id, expected in EXPECTED.items():
row = by_id.get(model_id, {})
checks.check(row.get("model_name") == registry[model_id]["model_name"], "model_name", model_id)
for field, value in zip(FIELDS[2:], expected):
checks.check(row.get(field) == value, "metric_value", f"{model_id} {field}")
checks.check(by_id.get("VC09", {}).get("fp32") == "56.85% / 79.87%", "known_correction", "VC09")
checks.check(by_id.get("VC12", {}).get("quantized") == "68.30% / 88.44%", "known_correction", "VC12")
checks.check("error" in by_id.get("VC13", {}).get("metric", "").lower(), "known_correction", "VC13")
checks.check(
{
model_id for model_id, row in by_id.items()
if row.get("published_comparison") != NO_PUBLISHED_VALUE
} == {"AD01", "SG06", "SG07", "SP01"},
"published_scope", "comparison column scope",
)
vc11 = json.loads((root / "research/evidence/vision/vc11_compute_graph_equivalence.json").read_text(encoding="utf-8"))
checks.check(vc11.get("status") == "PASS", "vc11_equivalence", "status")
for variant, facts in vc11.get("variants", {}).items():
checks.check(facts.get("official_compute_graph_sha256") == facts.get("acquired_compute_graph_sha256"), "vc11_equivalence", variant)
status = "PASS" if not checks.failures else "FAIL"
atomic_json(output, {
"schema_version": "1.0", "status": status,
"validated_csv": str(csv_path), "validated_csv_sha256": sha256(csv_path),
"row_count": len(rows), "check_count": checks.total,
"failure_count": len(checks.failures), "failures": checks.failures,
"categories": dict(sorted(checks.categories.items())),
"policy": {"model_runtime_executed": False, "dataset_evaluation_executed": False},
})
print(json.dumps({"status": status, "checks": checks.total, "failures": len(checks.failures)}, sort_keys=True))
return 0 if status == "PASS" else 1
if __name__ == "__main__":
raise SystemExit(main())
|