scripts/summarize_math_ink_06_formula_adapter_seeds.py

"""Formula adapter seed 17·31·47 결과를 동일 gate로 요약한다."""

from __future__ import annotations

import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
from statistics import mean, pstdev


def _metric06(reports: list[dict], section: str, name: str) -> dict:
    """필요 변수: seed report·section·metric. 작동 원리: 값·평균·표준편차·최솟값을 반환한다."""

    values = [float(report[section][name]) for report in reports]
    return {
        "values": values,
        "mean": mean(values),
        "std": pstdev(values),
        "minimum": min(values),
    }


def main() -> None:
    """필요 변수: seed report 세 개. 작동 원리: validation/test 92% gate와 일반화 gap을 고정한다."""

    parser = argparse.ArgumentParser(description="Summarize Math Ink 0.6 formula adapters")
    parser.add_argument("--report", type=Path, action="append", required=True)
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()
    reports = [
        json.loads(path.read_text(encoding="utf-8")) for path in args.report
    ]
    if len(reports) != 3:
        raise ValueError("formula adapter 요약에는 seed report 세 개가 필요합니다.")
    validation_visual = _metric06(
        reports, "selected_validation", "visual_family_top1",
    )
    test_visual = _metric06(reports, "official_test", "visual_family_top1")
    summary = {
        "experiment": "R-MATH-INK-06-FORMULA-ADAPTER-3SEED-001",
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "reports": [str(path) for path in args.report],
        "metrics": {
            "validation_exact_top1": _metric06(
                reports, "selected_validation", "exact_top1",
            ),
            "validation_family_head_top1": _metric06(
                reports, "selected_validation", "family_head_top1",
            ),
            "validation_visual_family_top1": validation_visual,
            "official_test_exact_top1": _metric06(
                reports, "official_test", "exact_top1",
            ),
            "official_test_family_head_top1": _metric06(
                reports, "official_test", "family_head_top1",
            ),
            "official_test_visual_family_top1": test_visual,
            "visual_family_generalization_gap_pp": {
                "values": [
                    (
                        float(report["selected_validation"]["visual_family_top1"])
                        - float(report["official_test"]["visual_family_top1"])
                    ) * 100.0
                    for report in reports
                ],
            },
        },
        "decision": {
            "all_seed_validation_92_passed": validation_visual["minimum"] >= 0.92,
            "all_seed_official_test_92_passed": test_visual["minimum"] >= 0.92,
            "architecture_recoverable": validation_visual["minimum"] >= 0.92,
            "writer_domain_generalization_gate_passed": test_visual["minimum"] >= 0.92,
            "product_validation": False,
            "distillation_allowed": False,
            "next_gate": "P 연속식 writer/device-disjoint formula adapter 학습·평가",
        },
        "track": "R_noncommercial_only",
        "product_validation": False,
    }
    gaps = summary["metrics"]["visual_family_generalization_gap_pp"]["values"]
    summary["metrics"]["visual_family_generalization_gap_pp"].update({
        "mean": mean(gaps),
        "std": pstdev(gaps),
        "maximum": max(gaps),
    })
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(
        json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )
    print(json.dumps(summary, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()