research/evals/src/slm_evals/utils/reporter.py

"""
utils/reporter.py
──────────────────
Persist benchmark results to JSON, Markdown, and CSV.
"""

from __future__ import annotations
import csv
import json
import datetime
from pathlib import Path
from typing import Any


class Reporter:
    def __init__(
        self,
        output_dir: str,
        experiment_name: str,
        model_path: str,
    ):
        self.output_dir     = Path(output_dir)
        self.experiment_name = experiment_name
        self.model_path     = model_path
        self.timestamp      = datetime.datetime.now().isoformat(timespec="seconds")

        self.run_dir = self.output_dir / experiment_name
        self.run_dir.mkdir(parents=True, exist_ok=True)

    # ── Public ────────────────────────────────────────────────────────────────

    def save(self, all_results: dict[str, Any]) -> dict[str, str]:
        """Write all three output formats. Returns dict of format → filepath."""
        payload = self._build_payload(all_results)
        paths = {
            "json":     self._write_json(payload),
            "csv":      self._write_csv(payload),
            "markdown": self._write_markdown(payload),
        }
        return paths

    # ── Private ───────────────────────────────────────────────────────────────

    def _build_payload(self, all_results: dict[str, Any]) -> dict[str, Any]:
        aggregate_score = (
            sum(r["score"] for r in all_results.values()) / len(all_results)
            if all_results else 0.0
        )
        return {
            "experiment_name":  self.experiment_name,
            "timestamp":        self.timestamp,
            "model_path":       self.model_path,
            "aggregate_score":  round(aggregate_score, 4),
            "benchmarks":       all_results,
        }

    def _write_json(self, payload: dict) -> str:
        path = self.run_dir / "results.json"
        with open(path, "w") as f:
            json.dump(payload, f, indent=2, default=str)
        return str(path)

    def _write_csv(self, payload: dict) -> str:
        path = self.run_dir / "results.csv"
        rows = []
        for bench_name, result in payload["benchmarks"].items():
            base_row = {
                "experiment":      payload["experiment_name"],
                "timestamp":       payload["timestamp"],
                "model_path":      payload["model_path"],
                "benchmark":       bench_name,
                "score":           round(result["score"], 4),
                "passed":          result["passed"],
                "total":           result["total"],
                "error_count":     result.get("error_count", 0),
                "avg_latency_s":   result.get("avg_latency_s", ""),
            }
            rows.append(base_row)

        if rows:
            with open(path, "w", newline="") as f:
                writer = csv.DictWriter(f, fieldnames=rows[0].keys())
                writer.writeheader()
                writer.writerows(rows)
        return str(path)

    def _write_markdown(self, payload: dict) -> str:
        path = self.run_dir / "report.md"
        lines = [
            f"# Benchmark Report — {payload['experiment_name']}",
            "",
            f"| Field | Value |",
            f"|---|---|",
            f"| **Timestamp** | {payload['timestamp']} |",
            f"| **Model** | `{payload['model_path']}` |",
            f"| **Aggregate Score** | **{payload['aggregate_score']:.2%}** |",
            "",
            "## Results by Benchmark",
            "",
            "| Benchmark | Score | Passed | Total | Errors | Avg Latency |",
            "|---|---|---|---|---|---|",
        ]

        for bench_name, result in payload["benchmarks"].items():
            lines.append(
                f"| {bench_name} "
                f"| {result['score']:.2%} "
                f"| {result['passed']} "
                f"| {result['total']} "
                f"| {result.get('error_count', 0)} "
                f"| {result.get('avg_latency_s', 'n/a')}s |"
            )

        lines += ["", "## Per-Sample Details", ""]

        for bench_name, result in payload["benchmarks"].items():
            lines += [
                f"### {bench_name.upper()}",
                "",
                "| # | Sample ID | Status | Score | Note |",
                "|---|---|---|---|---|",
            ]
            for i, sample in enumerate(result.get("samples", []), 1):
                status = "✅" if sample.get("passed") else "❌"
                lines.append(
                    f"| {i} "
                    f"| {sample.get('id', '—')} "
                    f"| {status} "
                    f"| {sample.get('score', ''):.2f} "
                    f"| {sample.get('note', '')} |"
                )
            lines.append("")

        with open(path, "w") as f:
            f.write("\n".join(lines))
        return str(path)