#!/usr/bin/env python3 """Build the concise AD01 compiled-MLIR numerical and task-quality report.""" from __future__ import annotations import argparse import csv import hashlib import json import os import tempfile from pathlib import Path from typing import Any def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo-root", required=True, type=Path) parser.add_argument("--result-dir", required=True, type=Path) parser.add_argument("--report-csv", required=True, type=Path) parser.add_argument("--report-md", required=True, type=Path) return parser.parse_args() def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def atomic_text(path: Path, value: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: handle.write(value) temporary = Path(handle.name) os.replace(temporary, path) def main() -> int: args = parse_args() root = args.repo_root.resolve() results = args.result_dir.resolve() fixed = json.loads((results / "compiled_output_comparison.json").read_text()) validation = json.loads((results / "validation.json").read_text()) q1_summaries = { variant: json.loads((results / f"q1/{variant}/quality_summary.json").read_text()) for variant in ("fp32", "public_quantized") } metrics_path = results / "compiled_quality_metrics.csv" with metrics_path.open(newline="") as handle: metrics = list(csv.DictReader(handle)) lookup = {(row["variant"], row["machine_id"]): row for row in metrics} rows: list[dict[str, Any]] = [] for variant, label in ( ("fp32", "FP32"), ("public_quantized", "PUBLIC_INT8"), ): compiler_key = ( "fp32_compiled_vs_onnxruntime" if variant == "fp32" else "public_quantized_compiled_vs_onnxruntime" ) comparison = fixed["comparisons"][compiler_key] q1_summary = q1_summaries[variant] fidelity = q1_summary["row_level_fidelity"] runtime_prefix = "fp32" if variant == "fp32" else "public_quantized" ort_metric = lookup[(f"{runtime_prefix}_onnxruntime", "Average")] compiled_metric = lookup[(f"{runtime_prefix}_compiled", "Average")] rows.append( { "model_id": "AD01", "variant": label, "compiled_invoke": fixed["abi_runtime_checks"][variant]["status"], "onnxruntime_vs_compiled": comparison["status"], "official_dataset_fidelity": q1_summary["fidelity_status"], "comparison_rule": "allclose(atol=1e-5,rtol=1e-5)" if variant == "fp32" else "raw_int8_exact", "max_abs_error": comparison["max_abs_error"], "mismatch_elements": comparison["mismatch_element_count"], "official_matching_vectors": fidelity["matching_rows"], "official_mismatching_vectors": fidelity["mismatching_rows"], "official_max_abs_error": fidelity["max_abs_error"], "official_mean_abs_error": fidelity["mean_abs_error"], "official_files": 2459, "official_vectors": 481964, "onnxruntime_auc": ort_metric["auc"], "compiled_auc": compiled_metric["auc"], "compiled_minus_onnxruntime_auc": float(compiled_metric["auc"]) - float(ort_metric["auc"]), "onnxruntime_pauc": ort_metric["pauc"], "compiled_pauc": compiled_metric["pauc"], "compiled_minus_onnxruntime_pauc": float(compiled_metric["pauc"]) - float(ort_metric["pauc"]), "q1_acceptance": "THRESHOLD_UNDEFINED", } ) fp32_compiled_auc = float(next(row for row in rows if row["variant"] == "FP32")["compiled_auc"]) fp32_compiled_pauc = float(next(row for row in rows if row["variant"] == "FP32")["compiled_pauc"]) for row in rows: row["compiled_minus_fp32_auc"] = float(row["compiled_auc"]) - fp32_compiled_auc row["compiled_minus_fp32_pauc"] = float(row["compiled_pauc"]) - fp32_compiled_pauc fieldnames = list(rows[0]) import io csv_buffer = io.StringIO(newline="") writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) atomic_text(args.report_csv.resolve(), csv_buffer.getvalue()) lines = [ "# AD01 ONNX-MLIR compiled accuracy validation", "", "## 결과", "", "| Variant | invoke | fixed ORT↔compiled | official vectors match/mismatch | official max abs | compiled AUC / pAUC |", "|---|---|---|---:|---:|---:|", ] for row in rows: lines.append( f"| {row['variant']} | {row['compiled_invoke']} | {row['onnxruntime_vs_compiled']} " f"| {row['official_matching_vectors']}/{row['official_mismatching_vectors']} " f"| {float(row['official_max_abs_error']):.10g} " f"| {float(row['compiled_auc']):.10f} / {float(row['compiled_pauc']):.10f} |" ) fp32_row = next(row for row in rows if row["variant"] == "FP32") quantized_row = next(row for row in rows if row["variant"] == "PUBLIC_INT8") quantized_minus_fp32_auc = float(quantized_row["compiled_auc"]) - float(fp32_row["compiled_auc"]) quantized_minus_fp32_pauc = float(quantized_row["compiled_pauc"]) - float(fp32_row["compiled_pauc"]) lines.extend( [ "", "공식 DCASE ToyCar test 2,459개 파일(481,964 feature vector)을 사용했다.", "", f"Compiled FP32 대비 PUBLIC_INT8의 task accuracy 변화는 AUC `{quantized_minus_fp32_auc:.10f}`, pAUC `{quantized_minus_fp32_pauc:.10f}`이다.", "", "고정 fixture ORT 비교는 FP32·PUBLIC_INT8 모두 통과했다. 공식 전체 입력에서는 FP32 47,620개, PUBLIC_INT8 4,080개 vector가 엄격 비교 기준을 벗어났다.", "", "## 무결성", "", f"- Independent validation: `{validation['status']}` ({validation['check_summary']['passed']}/{validation['check_summary']['total']})", f"- Fixed comparison SHA-256: `{sha256_file(results / 'compiled_output_comparison.json')}`", f"- Quality scores SHA-256: `{sha256_file(results / 'compiled_file_scores.csv')}`", f"- Quality metrics SHA-256: `{sha256_file(metrics_path)}`", "", ] ) atomic_text(args.report_md.resolve(), "\n".join(lines)) print(json.dumps({"status": "PASS", "rows": len(rows), "report": str(args.report_md.resolve())}, sort_keys=True)) return 0 if __name__ == "__main__": raise SystemExit(main())