| |
| """Independently validate the compact 21-model accuracy CSV.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import csv |
| import hashlib |
| import json |
| import os |
| import tempfile |
| from collections import Counter |
| from pathlib import Path |
| from typing import Any |
|
|
|
|
| FIELDS = ["model_id", "model_name", "metric", "fp32", "quantized", "delta", "published_comparison"] |
| EXPECTED: dict[str, tuple[str, str, str, str, str]] = { |
| "AD01": ("AUC / pAUC (max_fpr=0.1)", "0.876001 / 0.764121", "0.840250 / 0.720049", "-0.035750 / -0.044071", "공개 FP32 AUC/pAUC 0.876001/0.764121; 측정값 동일"), |
| "LM04": ("AUROC / TPR@FPR 5% / 1% (RAID-extra OOD)", "0.667078 / 0.318983 / 0.233873", "0.668862 / 0.322615 / 0.235853", "+0.001785 / +0.003632 / +0.001981", ""), |
| "OD06": ("COCO bbox mAP (AP@[0.50:0.95])", "24.8751%", "24.2822%", "-0.5929 pp", ""), |
| "OD07": ("COCO bbox mAP (AP@[0.50:0.95])", "31.8594%", "31.4191%", "-0.4403 pp", ""), |
| "SG06": ("mIoU", "75.6398%", "74.1290%", "-1.5108 pp", "공개 FP32 mIoU 75.32%; 측정 75.6398% (+0.3198 pp)"), |
| "SG07": ("mIoU", "70.6473%", "69.6191%", "-1.0282 pp", "공개 FP32 mIoU 70.19%; 측정 70.6473% (+0.4573 pp)"), |
| "SG08": ("mIoU (CamVid cross-dataset)", "50.6498%", "51.1600%", "+0.5102 pp", ""), |
| "SP01": ("Top-1 accuracy", "91.86%", "91.66%", "-0.2045 pp", "공개 시험 정확도 약 92%; 측정 FP32 91.86%, 양자화 91.66% (근접)"), |
| "SP02": ("FP / FN (1 s)", "5 / 6", "4 / 6", "-1 / +0", ""), |
| "SP08": ("Top-1 accuracy (yes/no subset)", "94.05%", "94.05%", "+0.0000 pp", ""), |
| "SP09": ("Top-1 accuracy", "95.06%", "94.70%", "-0.3590 pp", ""), |
| "VC01": ("Top-1 accuracy", "85.10%", "85.60%", "+0.5000 pp", ""), |
| "VC02": ("Top-1 accuracy", "87.00%", "87.00%", "+0.0000 pp", ""), |
| "VC03": ("Top-1 / Top-5 accuracy", "49.80% / 74.20%", "48.00% / 72.80%", "-1.8000 pp / -1.4000 pp", ""), |
| "VC04": ("Top-1 / Top-5 accuracy", "63.30% / 84.90%", "60.70% / 83.20%", "-2.6000 pp / -1.7000 pp", ""), |
| "VC05": ("Top-1 accuracy", "58.13%", "56.77%", "-1.3600 pp", ""), |
| "VC06": ("Top-1 accuracy", "66.20%", "65.31%", "-0.8900 pp", ""), |
| "VC09": ("Top-1 / Top-5 accuracy", "56.85% / 79.87%", "56.48% / 79.76%", "-0.3700 pp / -0.1100 pp", ""), |
| "VC11": ("Top-1 accuracy", "75.10%", "74.40%", "-0.7000 pp", ""), |
| "VC12": ("Top-1 / Top-5 accuracy", "69.48% / 89.26%", "68.30% / 88.44%", "-1.1800 pp / -0.8200 pp", ""), |
| "VC13": ("Top-1 / Top-5 error", "33.65% / 13.43%", "33.85% / 13.66%", "+0.2000 pp / +0.2300 pp", ""), |
| } |
|
|
| NO_PUBLISHED_VALUE = "공개 수치 없음." |
| EXPECTED = { |
| model_id: (*values[:4], values[4] or NO_PUBLISHED_VALUE) |
| for model_id, values in EXPECTED.items() |
| } |
|
|
|
|
| class Checks: |
| def __init__(self) -> None: |
| self.total = 0 |
| self.failures: list[dict[str, str]] = [] |
| self.categories: Counter[str] = Counter() |
|
|
| def check(self, condition: bool, category: str, detail: str) -> None: |
| self.total += 1 |
| self.categories[category] += 1 |
| if not condition: |
| self.failures.append({"category": category, "detail": detail}) |
|
|
|
|
| def sha256(path: Path) -> str: |
| return hashlib.sha256(path.read_bytes()).hexdigest() |
|
|
|
|
| def atomic_json(path: Path, value: dict[str, Any]) -> None: |
| path.parent.mkdir(parents=True, exist_ok=True) |
| fd, temporary = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=path.parent) |
| try: |
| with os.fdopen(fd, "w", encoding="utf-8") as handle: |
| json.dump(value, handle, indent=2, sort_keys=True, ensure_ascii=False) |
| handle.write("\n") |
| os.replace(temporary, path) |
| except BaseException: |
| Path(temporary).unlink(missing_ok=True) |
| raise |
|
|
|
|
| def main() -> int: |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--repo-root", type=Path, required=True) |
| parser.add_argument("--csv", type=Path, required=True) |
| parser.add_argument("--output", type=Path, required=True) |
| args = parser.parse_args() |
| root = args.repo_root.resolve() |
| csv_path = args.csv if args.csv.is_absolute() else root / args.csv |
| output = args.output if args.output.is_absolute() else root / args.output |
| checks = Checks() |
| with csv_path.open(newline="", encoding="utf-8") as handle: |
| reader = csv.DictReader(handle) |
| checks.check(reader.fieldnames == FIELDS, "schema", str(reader.fieldnames)) |
| rows = list(reader) |
| checks.check(len(rows) == 21, "coverage", f"rows={len(rows)}") |
| by_id = {row["model_id"]: row for row in rows} |
| checks.check(len(by_id) == len(rows), "coverage", "duplicate model ID") |
| checks.check(set(by_id) == set(EXPECTED), "coverage", "model ID set differs") |
| registry = { |
| row["model_id"]: row for row in csv.DictReader( |
| (root / "model_registry.csv").open(newline="", encoding="utf-8") |
| ) if row["eligibility"] == "ELIGIBLE" |
| } |
| checks.check(set(registry) == set(EXPECTED), "registry", "active registry set differs") |
| for model_id, expected in EXPECTED.items(): |
| row = by_id.get(model_id, {}) |
| checks.check(row.get("model_name") == registry[model_id]["model_name"], "model_name", model_id) |
| for field, value in zip(FIELDS[2:], expected): |
| checks.check(row.get(field) == value, "metric_value", f"{model_id} {field}") |
| checks.check(by_id.get("VC09", {}).get("fp32") == "56.85% / 79.87%", "known_correction", "VC09") |
| checks.check(by_id.get("VC12", {}).get("quantized") == "68.30% / 88.44%", "known_correction", "VC12") |
| checks.check("error" in by_id.get("VC13", {}).get("metric", "").lower(), "known_correction", "VC13") |
| checks.check( |
| { |
| model_id for model_id, row in by_id.items() |
| if row.get("published_comparison") != NO_PUBLISHED_VALUE |
| } == {"AD01", "SG06", "SG07", "SP01"}, |
| "published_scope", "comparison column scope", |
| ) |
| vc11 = json.loads((root / "research/evidence/vision/vc11_compute_graph_equivalence.json").read_text(encoding="utf-8")) |
| checks.check(vc11.get("status") == "PASS", "vc11_equivalence", "status") |
| for variant, facts in vc11.get("variants", {}).items(): |
| checks.check(facts.get("official_compute_graph_sha256") == facts.get("acquired_compute_graph_sha256"), "vc11_equivalence", variant) |
| status = "PASS" if not checks.failures else "FAIL" |
| atomic_json(output, { |
| "schema_version": "1.0", "status": status, |
| "validated_csv": str(csv_path), "validated_csv_sha256": sha256(csv_path), |
| "row_count": len(rows), "check_count": checks.total, |
| "failure_count": len(checks.failures), "failures": checks.failures, |
| "categories": dict(sorted(checks.categories.items())), |
| "policy": {"model_runtime_executed": False, "dataset_evaluation_executed": False}, |
| }) |
| print(json.dumps({"status": status, "checks": checks.total, "failures": len(checks.failures)}, sort_keys=True)) |
| return 0 if status == "PASS" else 1 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
