File size: 2,081 Bytes
c87881a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | """Generate paper-consumable summaries from locked evaluation records."""
from __future__ import annotations
from pathlib import Path
from typing import Any
import pandas as pd
from .artifacts import write_json_immutable
from .statistics import aggregate_draws_by_group, hierarchical_bootstrap_ci
def summarize_results(
results: pd.DataFrame,
metrics: list[str],
bootstrap_samples: int,
confidence: float,
seed: int,
) -> tuple[pd.DataFrame, pd.DataFrame]:
aggregated = aggregate_draws_by_group(results, metrics)
rows: list[dict[str, Any]] = []
for method, method_rows in aggregated.groupby("method"):
for metric in metrics:
lower, upper = hierarchical_bootstrap_ci(
method_rows[metric], bootstrap_samples, confidence, seed
)
rows.append(
{
"method": method,
"metric": metric,
"mean": float(method_rows[metric].mean()),
"ci_lower": lower,
"ci_upper": upper,
"groups": int(method_rows["group_id"].nunique()),
}
)
return aggregated, pd.DataFrame(rows)
def write_paper_outputs(
output_dir: str | Path,
results: pd.DataFrame,
metrics: list[str],
bootstrap_samples: int,
confidence: float,
seed: int,
metadata: dict[str, Any],
) -> None:
output = Path(output_dir)
output.mkdir(parents=True, exist_ok=True)
for name in ("query_results.csv", "group_results.csv", "summary.csv", "metadata.json"):
if (output / name).exists():
raise FileExistsError(f"Refusing to overwrite paper output: {output / name}")
aggregated, summary = summarize_results(
results, metrics, bootstrap_samples, confidence, seed
)
results.to_csv(output / "query_results.csv", index=False)
aggregated.to_csv(output / "group_results.csv", index=False)
summary.to_csv(output / "summary.csv", index=False)
write_json_immutable(output / "metadata.json", metadata)
|