scripts/build_ad01_compiled_numerical_report.py

#!/usr/bin/env python3
"""Build the concise AD01 compiled-MLIR numerical and task-quality report."""

from __future__ import annotations

import argparse
import csv
import hashlib
import json
import os
import tempfile
from pathlib import Path
from typing import Any


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--repo-root", required=True, type=Path)
    parser.add_argument("--result-dir", required=True, type=Path)
    parser.add_argument("--report-csv", required=True, type=Path)
    parser.add_argument("--report-md", required=True, type=Path)
    return parser.parse_args()


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for block in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def atomic_text(path: Path, value: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle:
        handle.write(value)
        temporary = Path(handle.name)
    os.replace(temporary, path)


def main() -> int:
    args = parse_args()
    root = args.repo_root.resolve()
    results = args.result_dir.resolve()
    fixed = json.loads((results / "compiled_output_comparison.json").read_text())
    validation = json.loads((results / "validation.json").read_text())
    q1_summaries = {
        variant: json.loads((results / f"q1/{variant}/quality_summary.json").read_text())
        for variant in ("fp32", "public_quantized")
    }
    metrics_path = results / "compiled_quality_metrics.csv"
    with metrics_path.open(newline="") as handle:
        metrics = list(csv.DictReader(handle))
    lookup = {(row["variant"], row["machine_id"]): row for row in metrics}
    rows: list[dict[str, Any]] = []
    for variant, label in (
        ("fp32", "FP32"),
        ("public_quantized", "PUBLIC_INT8"),
    ):
        compiler_key = (
            "fp32_compiled_vs_onnxruntime"
            if variant == "fp32"
            else "public_quantized_compiled_vs_onnxruntime"
        )
        comparison = fixed["comparisons"][compiler_key]
        q1_summary = q1_summaries[variant]
        fidelity = q1_summary["row_level_fidelity"]
        runtime_prefix = "fp32" if variant == "fp32" else "public_quantized"
        ort_metric = lookup[(f"{runtime_prefix}_onnxruntime", "Average")]
        compiled_metric = lookup[(f"{runtime_prefix}_compiled", "Average")]
        rows.append(
            {
                "model_id": "AD01",
                "variant": label,
                "compiled_invoke": fixed["abi_runtime_checks"][variant]["status"],
                "onnxruntime_vs_compiled": comparison["status"],
                "official_dataset_fidelity": q1_summary["fidelity_status"],
                "comparison_rule": "allclose(atol=1e-5,rtol=1e-5)" if variant == "fp32" else "raw_int8_exact",
                "max_abs_error": comparison["max_abs_error"],
                "mismatch_elements": comparison["mismatch_element_count"],
                "official_matching_vectors": fidelity["matching_rows"],
                "official_mismatching_vectors": fidelity["mismatching_rows"],
                "official_max_abs_error": fidelity["max_abs_error"],
                "official_mean_abs_error": fidelity["mean_abs_error"],
                "official_files": 2459,
                "official_vectors": 481964,
                "onnxruntime_auc": ort_metric["auc"],
                "compiled_auc": compiled_metric["auc"],
                "compiled_minus_onnxruntime_auc": float(compiled_metric["auc"]) - float(ort_metric["auc"]),
                "onnxruntime_pauc": ort_metric["pauc"],
                "compiled_pauc": compiled_metric["pauc"],
                "compiled_minus_onnxruntime_pauc": float(compiled_metric["pauc"]) - float(ort_metric["pauc"]),
                "q1_acceptance": "THRESHOLD_UNDEFINED",
            }
        )
    fp32_compiled_auc = float(next(row for row in rows if row["variant"] == "FP32")["compiled_auc"])
    fp32_compiled_pauc = float(next(row for row in rows if row["variant"] == "FP32")["compiled_pauc"])
    for row in rows:
        row["compiled_minus_fp32_auc"] = float(row["compiled_auc"]) - fp32_compiled_auc
        row["compiled_minus_fp32_pauc"] = float(row["compiled_pauc"]) - fp32_compiled_pauc
    fieldnames = list(rows[0])
    import io

    csv_buffer = io.StringIO(newline="")
    writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames)
    writer.writeheader()
    writer.writerows(rows)
    atomic_text(args.report_csv.resolve(), csv_buffer.getvalue())

    lines = [
        "# AD01 ONNX-MLIR compiled accuracy validation",
        "",
        "## 결과",
        "",
        "| Variant | invoke | fixed ORT↔compiled | official vectors match/mismatch | official max abs | compiled AUC / pAUC |",
        "|---|---|---|---:|---:|---:|",
    ]
    for row in rows:
        lines.append(
            f"| {row['variant']} | {row['compiled_invoke']} | {row['onnxruntime_vs_compiled']} "
            f"| {row['official_matching_vectors']}/{row['official_mismatching_vectors']} "
            f"| {float(row['official_max_abs_error']):.10g} "
            f"| {float(row['compiled_auc']):.10f} / {float(row['compiled_pauc']):.10f} |"
        )
    fp32_row = next(row for row in rows if row["variant"] == "FP32")
    quantized_row = next(row for row in rows if row["variant"] == "PUBLIC_INT8")
    quantized_minus_fp32_auc = float(quantized_row["compiled_auc"]) - float(fp32_row["compiled_auc"])
    quantized_minus_fp32_pauc = float(quantized_row["compiled_pauc"]) - float(fp32_row["compiled_pauc"])
    lines.extend(
        [
            "",
            "공식 DCASE ToyCar test 2,459개 파일(481,964 feature vector)을 사용했다.",
            "",
            f"Compiled FP32 대비 PUBLIC_INT8의 task accuracy 변화는 AUC `{quantized_minus_fp32_auc:.10f}`, pAUC `{quantized_minus_fp32_pauc:.10f}`이다.",
            "",
            "고정 fixture ORT 비교는 FP32·PUBLIC_INT8 모두 통과했다. 공식 전체 입력에서는 FP32 47,620개, PUBLIC_INT8 4,080개 vector가 엄격 비교 기준을 벗어났다.",
            "",
            "## 무결성",
            "",
            f"- Independent validation: `{validation['status']}` ({validation['check_summary']['passed']}/{validation['check_summary']['total']})",
            f"- Fixed comparison SHA-256: `{sha256_file(results / 'compiled_output_comparison.json')}`",
            f"- Quality scores SHA-256: `{sha256_file(results / 'compiled_file_scores.csv')}`",
            f"- Quality metrics SHA-256: `{sha256_file(metrics_path)}`",
            "",
        ]
    )
    atomic_text(args.report_md.resolve(), "\n".join(lines))
    print(json.dumps({"status": "PASS", "rows": len(rows), "report": str(args.report_md.resolve())}, sort_keys=True))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())