File size: 8,222 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
"""Paper-oriented exports from benchmark summary artifacts."""

from __future__ import annotations

import json
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from .errors import BenchmarkError


DEFAULT_PAPER_METRICS = ["cer", "table_cell_f1", "reading_order_pair_accuracy"]


@dataclass(slots=True)
class PaperTableRow:
    system: str
    display_name: str
    sample_count: int | None
    metrics: dict[str, float]
    sources: list[str]

    def to_dict(self) -> dict[str, object]:
        return {
            "system": self.system,
            "display_name": self.display_name,
            "sample_count": self.sample_count,
            "metrics": self.metrics,
            "sources": self.sources,
        }


@dataclass(slots=True)
class PaperBenchmarkTable:
    title: str
    slice_name: str
    metrics: list[str]
    rows: list[PaperTableRow]
    source_summaries: list[str]

    def to_dict(self) -> dict[str, object]:
        return {
            "title": self.title,
            "slice": self.slice_name,
            "metrics": self.metrics,
            "rows": [row.to_dict() for row in self.rows],
            "source_summaries": self.source_summaries,
        }


def export_paper_benchmark_table(
    summary_paths: list[str | Path],
    output_dir: str | Path,
    *,
    slice_name: str = "all",
    metrics: list[str] | None = None,
    systems: list[str] | None = None,
    display_names: dict[str, str] | None = None,
    source_preferences: dict[str, str] | None = None,
    title: str | None = None,
) -> PaperBenchmarkTable:
    if not summary_paths:
        raise BenchmarkError("paper benchmark table requires at least one summary artifact")
    selected_metrics = metrics or list(DEFAULT_PAPER_METRICS)
    ordered_systems = list(dict.fromkeys(systems or []))
    filter_systems = set(ordered_systems)
    labels = dict(display_names or {})
    preferred_sources = {
        system: str(Path(path))
        for system, path in dict(source_preferences or {}).items()
    }

    rows_by_system: dict[str, PaperTableRow] = {}
    seen_order: list[str] = []
    source_summaries: list[str] = []
    for summary_path in summary_paths:
        payload = _load_summary_json(summary_path)
        normalized_summary_path = str(Path(summary_path))
        source_summaries.append(normalized_summary_path)
        aggregate_metrics = payload.get("aggregate_metrics")
        if not isinstance(aggregate_metrics, dict):
            raise BenchmarkError(f"summary {summary_path} missing aggregate_metrics object")
        for system_name, slice_map in aggregate_metrics.items():
            system = str(system_name)
            if filter_systems and system not in filter_systems:
                continue
            preferred_source = preferred_sources.get(system)
            if preferred_source and preferred_source != normalized_summary_path:
                continue
            if not isinstance(slice_map, dict):
                continue
            slice_metrics = slice_map.get(slice_name)
            if not isinstance(slice_metrics, dict):
                continue
            row = rows_by_system.get(system)
            sample_count = _optional_int(slice_metrics.get("sample_count"), field_name=f"{system}.{slice_name}.sample_count")
            if row is None:
                row = PaperTableRow(
                    system=system,
                    display_name=labels.get(system, system),
                    sample_count=sample_count,
                    metrics={},
                    sources=[],
                )
                rows_by_system[system] = row
                seen_order.append(system)
            elif sample_count is not None:
                _merge_sample_count(row, sample_count, summary_path, slice_name)
            for metric in selected_metrics:
                value = slice_metrics.get(metric)
                if value is None:
                    continue
                if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(float(value)):
                    raise BenchmarkError(
                        f"summary {summary_path} has non-finite {metric!r} for system {system!r} slice {slice_name!r}"
                    )
                _merge_metric(row, metric, float(value), summary_path, slice_name)
            row.sources.append(str(Path(summary_path)))

    if not rows_by_system:
        raise BenchmarkError(f"no systems found for slice {slice_name!r} in the provided summaries")
    if ordered_systems:
        missing = [system for system in ordered_systems if system not in rows_by_system]
        if missing:
            raise BenchmarkError(f"requested systems missing from provided summaries: {', '.join(missing)}")
        final_systems = ordered_systems
    else:
        final_systems = seen_order
    table = PaperBenchmarkTable(
        title=title or f"Benchmark Comparison ({slice_name})",
        slice_name=slice_name,
        metrics=selected_metrics,
        rows=[rows_by_system[system] for system in final_systems],
        source_summaries=source_summaries,
    )
    _write_paper_benchmark_table(table, Path(output_dir))
    return table


def _load_summary_json(path: str | Path) -> dict[str, Any]:
    raw_path = Path(path)
    try:
        payload = json.loads(raw_path.read_text(encoding="utf-8"))
    except FileNotFoundError as exc:
        raise BenchmarkError(f"summary artifact not found: {raw_path}") from exc
    except json.JSONDecodeError as exc:
        raise BenchmarkError(f"invalid summary JSON {raw_path}: {exc}") from exc
    if not isinstance(payload, dict):
        raise BenchmarkError(f"summary artifact must be a JSON object: {raw_path}")
    return payload


def _optional_int(value: Any, *, field_name: str) -> int | None:
    if value is None:
        return None
    if isinstance(value, bool) or not isinstance(value, int | float):
        raise BenchmarkError(f"{field_name} must be numeric, got {type(value).__name__}")
    return int(value)


def _merge_sample_count(row: PaperTableRow, sample_count: int, source_path: str | Path, slice_name: str) -> None:
    if row.sample_count is None:
        row.sample_count = sample_count
        return
    if row.sample_count != sample_count:
        raise BenchmarkError(
            f"conflicting sample_count for system {row.system!r} slice {slice_name!r}: "
            f"{row.sample_count} vs {sample_count} in {source_path}"
        )


def _merge_metric(row: PaperTableRow, metric: str, value: float, source_path: str | Path, slice_name: str) -> None:
    previous = row.metrics.get(metric)
    if previous is None:
        row.metrics[metric] = value
        return
    if not math.isclose(previous, value, rel_tol=1e-9, abs_tol=1e-9):
        raise BenchmarkError(
            f"conflicting aggregate value for system {row.system!r} slice {slice_name!r} metric {metric!r}: "
            f"{previous} vs {value} in {source_path}"
        )


def _write_paper_benchmark_table(table: PaperBenchmarkTable, output_dir: Path) -> None:
    output_dir.mkdir(parents=True, exist_ok=True)
    (output_dir / "paper-table.json").write_text(
        json.dumps(table.to_dict(), ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )

    columns = ["system", "sample_count", *table.metrics]
    lines = [
        "# Paper Benchmark Table",
        "",
        f"Title: {table.title}",
        f"Slice: `{table.slice_name}`",
        "",
        "| " + " | ".join(columns) + " |",
        "| " + " | ".join("---:" if index > 0 else "---" for index, _ in enumerate(columns)) + " |",
    ]
    for row in table.rows:
        values = [row.display_name, "" if row.sample_count is None else str(row.sample_count)]
        for metric in table.metrics:
            metric_value = row.metrics.get(metric)
            values.append("" if metric_value is None else f"{metric_value:.6f}")
        lines.append("| " + " | ".join(values) + " |")
    lines.extend(["", "## Source Summaries", ""])
    for source in table.source_summaries:
        lines.append(f"- `{source}`")
    (output_dir / "paper-table.md").write_text("\n".join(lines) + "\n", encoding="utf-8")