File size: 8,222 Bytes
d82db76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 | """Paper-oriented exports from benchmark summary artifacts."""
from __future__ import annotations
import json
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .errors import BenchmarkError
DEFAULT_PAPER_METRICS = ["cer", "table_cell_f1", "reading_order_pair_accuracy"]
@dataclass(slots=True)
class PaperTableRow:
system: str
display_name: str
sample_count: int | None
metrics: dict[str, float]
sources: list[str]
def to_dict(self) -> dict[str, object]:
return {
"system": self.system,
"display_name": self.display_name,
"sample_count": self.sample_count,
"metrics": self.metrics,
"sources": self.sources,
}
@dataclass(slots=True)
class PaperBenchmarkTable:
title: str
slice_name: str
metrics: list[str]
rows: list[PaperTableRow]
source_summaries: list[str]
def to_dict(self) -> dict[str, object]:
return {
"title": self.title,
"slice": self.slice_name,
"metrics": self.metrics,
"rows": [row.to_dict() for row in self.rows],
"source_summaries": self.source_summaries,
}
def export_paper_benchmark_table(
summary_paths: list[str | Path],
output_dir: str | Path,
*,
slice_name: str = "all",
metrics: list[str] | None = None,
systems: list[str] | None = None,
display_names: dict[str, str] | None = None,
source_preferences: dict[str, str] | None = None,
title: str | None = None,
) -> PaperBenchmarkTable:
if not summary_paths:
raise BenchmarkError("paper benchmark table requires at least one summary artifact")
selected_metrics = metrics or list(DEFAULT_PAPER_METRICS)
ordered_systems = list(dict.fromkeys(systems or []))
filter_systems = set(ordered_systems)
labels = dict(display_names or {})
preferred_sources = {
system: str(Path(path))
for system, path in dict(source_preferences or {}).items()
}
rows_by_system: dict[str, PaperTableRow] = {}
seen_order: list[str] = []
source_summaries: list[str] = []
for summary_path in summary_paths:
payload = _load_summary_json(summary_path)
normalized_summary_path = str(Path(summary_path))
source_summaries.append(normalized_summary_path)
aggregate_metrics = payload.get("aggregate_metrics")
if not isinstance(aggregate_metrics, dict):
raise BenchmarkError(f"summary {summary_path} missing aggregate_metrics object")
for system_name, slice_map in aggregate_metrics.items():
system = str(system_name)
if filter_systems and system not in filter_systems:
continue
preferred_source = preferred_sources.get(system)
if preferred_source and preferred_source != normalized_summary_path:
continue
if not isinstance(slice_map, dict):
continue
slice_metrics = slice_map.get(slice_name)
if not isinstance(slice_metrics, dict):
continue
row = rows_by_system.get(system)
sample_count = _optional_int(slice_metrics.get("sample_count"), field_name=f"{system}.{slice_name}.sample_count")
if row is None:
row = PaperTableRow(
system=system,
display_name=labels.get(system, system),
sample_count=sample_count,
metrics={},
sources=[],
)
rows_by_system[system] = row
seen_order.append(system)
elif sample_count is not None:
_merge_sample_count(row, sample_count, summary_path, slice_name)
for metric in selected_metrics:
value = slice_metrics.get(metric)
if value is None:
continue
if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(float(value)):
raise BenchmarkError(
f"summary {summary_path} has non-finite {metric!r} for system {system!r} slice {slice_name!r}"
)
_merge_metric(row, metric, float(value), summary_path, slice_name)
row.sources.append(str(Path(summary_path)))
if not rows_by_system:
raise BenchmarkError(f"no systems found for slice {slice_name!r} in the provided summaries")
if ordered_systems:
missing = [system for system in ordered_systems if system not in rows_by_system]
if missing:
raise BenchmarkError(f"requested systems missing from provided summaries: {', '.join(missing)}")
final_systems = ordered_systems
else:
final_systems = seen_order
table = PaperBenchmarkTable(
title=title or f"Benchmark Comparison ({slice_name})",
slice_name=slice_name,
metrics=selected_metrics,
rows=[rows_by_system[system] for system in final_systems],
source_summaries=source_summaries,
)
_write_paper_benchmark_table(table, Path(output_dir))
return table
def _load_summary_json(path: str | Path) -> dict[str, Any]:
raw_path = Path(path)
try:
payload = json.loads(raw_path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise BenchmarkError(f"summary artifact not found: {raw_path}") from exc
except json.JSONDecodeError as exc:
raise BenchmarkError(f"invalid summary JSON {raw_path}: {exc}") from exc
if not isinstance(payload, dict):
raise BenchmarkError(f"summary artifact must be a JSON object: {raw_path}")
return payload
def _optional_int(value: Any, *, field_name: str) -> int | None:
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, int | float):
raise BenchmarkError(f"{field_name} must be numeric, got {type(value).__name__}")
return int(value)
def _merge_sample_count(row: PaperTableRow, sample_count: int, source_path: str | Path, slice_name: str) -> None:
if row.sample_count is None:
row.sample_count = sample_count
return
if row.sample_count != sample_count:
raise BenchmarkError(
f"conflicting sample_count for system {row.system!r} slice {slice_name!r}: "
f"{row.sample_count} vs {sample_count} in {source_path}"
)
def _merge_metric(row: PaperTableRow, metric: str, value: float, source_path: str | Path, slice_name: str) -> None:
previous = row.metrics.get(metric)
if previous is None:
row.metrics[metric] = value
return
if not math.isclose(previous, value, rel_tol=1e-9, abs_tol=1e-9):
raise BenchmarkError(
f"conflicting aggregate value for system {row.system!r} slice {slice_name!r} metric {metric!r}: "
f"{previous} vs {value} in {source_path}"
)
def _write_paper_benchmark_table(table: PaperBenchmarkTable, output_dir: Path) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / "paper-table.json").write_text(
json.dumps(table.to_dict(), ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
columns = ["system", "sample_count", *table.metrics]
lines = [
"# Paper Benchmark Table",
"",
f"Title: {table.title}",
f"Slice: `{table.slice_name}`",
"",
"| " + " | ".join(columns) + " |",
"| " + " | ".join("---:" if index > 0 else "---" for index, _ in enumerate(columns)) + " |",
]
for row in table.rows:
values = [row.display_name, "" if row.sample_count is None else str(row.sample_count)]
for metric in table.metrics:
metric_value = row.metrics.get(metric)
values.append("" if metric_value is None else f"{metric_value:.6f}")
lines.append("| " + " | ".join(values) + " |")
lines.extend(["", "## Source Summaries", ""])
for source in table.source_summaries:
lines.append(f"- `{source}`")
(output_dir / "paper-table.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|