prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
19.1 kB
"""Error analysis utilities for benchmark reports."""
from __future__ import annotations
import json
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .benchmark import _load_benchmark_rows
from .errors import BenchmarkError
from .manifest import ManifestEntry, load_manifest, write_manifest
from .validation import metric_direction, sample_key
@dataclass(slots=True)
class BenchmarkErrorAnalysis:
benchmark_report: str
system: str
metric: str
direction: str
baselines: list[str]
sample_count: int
top_failures_path: str
analysis_path: str
def to_dict(self) -> dict[str, object]:
return {
"benchmark_report": self.benchmark_report,
"system": self.system,
"metric": self.metric,
"direction": self.direction,
"baselines": self.baselines,
"sample_count": self.sample_count,
"top_failures_path": self.top_failures_path,
"analysis_path": self.analysis_path,
}
@dataclass(slots=True)
class FailureManifestExport:
source_manifest: str
top_failures_path: str
output_manifest: str
selected_count: int
split: str | None
summary_path: str
def to_dict(self) -> dict[str, object]:
return {
"source_manifest": self.source_manifest,
"top_failures_path": self.top_failures_path,
"output_manifest": self.output_manifest,
"selected_count": self.selected_count,
"split": self.split,
"summary_path": self.summary_path,
}
def analyze_benchmark_errors(
report_path: str | Path,
output_dir: str | Path,
*,
system: str,
baselines: list[str] | None = None,
metric: str = "cer",
direction: str | None = None,
top_n: int = 20,
) -> BenchmarkErrorAnalysis:
if top_n < 1:
raise BenchmarkError("top_n must be at least 1")
rows = _load_benchmark_rows(report_path)
if not rows:
raise BenchmarkError("benchmark report has no rows")
target_direction = direction or metric_direction(metric)
available_systems = sorted({str(row.get("baseline") or "") for row in rows if str(row.get("baseline") or "")})
if system not in available_systems:
raise BenchmarkError(f"system {system!r} is not present in benchmark report")
compare_baselines = baselines or []
missing = [name for name in compare_baselines if name not in available_systems]
if missing:
raise BenchmarkError(f"requested baselines missing from benchmark report: {', '.join(missing)}")
rows_by_system = _metric_rows_by_system(rows, metric)
system_rows = rows_by_system.get(system, {})
if not system_rows:
raise BenchmarkError(f"system {system!r} has no ok rows with metric {metric!r}")
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
top_failures = _top_failures(system_rows, rows_by_system, compare_baselines, metric, target_direction, top_n)
slice_summary = _slice_summary(system_rows, rows_by_system, compare_baselines, metric, target_direction)
baseline_summary = _baseline_summary(system_rows, rows_by_system, compare_baselines, metric, target_direction)
top_failure_slice_counts = _top_failure_slice_counts(top_failures)
top_failures_path = out / "top-failures.json"
top_failures_path.write_text(json.dumps(top_failures, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
payload = {
"benchmark_report": str(Path(report_path)),
"system": system,
"metric": metric,
"direction": target_direction,
"baselines": compare_baselines,
"sample_count": len(system_rows),
"baseline_summary": baseline_summary,
"slice_summary": slice_summary,
"top_failure_slice_counts": top_failure_slice_counts,
"top_failures_path": str(top_failures_path),
}
analysis_path = out / "error-analysis.json"
analysis_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_markdown(out / "error-analysis.md", payload, top_failures)
return BenchmarkErrorAnalysis(
benchmark_report=str(Path(report_path)),
system=system,
metric=metric,
direction=target_direction,
baselines=compare_baselines,
sample_count=len(system_rows),
top_failures_path=str(top_failures_path),
analysis_path=str(analysis_path),
)
def export_failure_manifest(
source_manifest: str | Path,
top_failures_path: str | Path,
output_manifest: str | Path,
*,
split: str | None = None,
) -> FailureManifestExport:
entries = load_manifest(source_manifest)
entries_by_sample = {entry.sample_id: entry for entry in entries}
try:
failure_rows = json.loads(Path(top_failures_path).read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise BenchmarkError(f"invalid top failures JSON {top_failures_path}: {exc}") from exc
if not isinstance(failure_rows, list):
raise BenchmarkError("top failures file must contain a JSON list")
selected: list[ManifestEntry] = []
missing: list[str] = []
seen: set[str] = set()
for rank, item in enumerate(failure_rows, start=1):
if not isinstance(item, dict):
raise BenchmarkError(f"top failure row {rank} must be an object")
sample_id = str(item.get("sample_id") or item.get("sample_key") or "")
if not sample_id:
raise BenchmarkError(f"top failure row {rank} missing sample_id/sample_key")
if sample_id in seen:
continue
seen.add(sample_id)
source = entries_by_sample.get(sample_id)
if source is None:
missing.append(sample_id)
continue
selected.append(_failure_entry(source, failure=item, rank=rank, split=split))
if missing:
raise BenchmarkError(f"top failure samples missing from source manifest: {', '.join(missing)}")
if not selected:
raise BenchmarkError("failure manifest export selected zero samples")
out_path = Path(output_manifest)
write_manifest(selected, out_path)
summary_path = out_path.with_name(f"{out_path.stem}-failure-manifest.json")
export = FailureManifestExport(
source_manifest=str(Path(source_manifest)),
top_failures_path=str(Path(top_failures_path)),
output_manifest=str(out_path),
selected_count=len(selected),
split=split,
summary_path=str(summary_path),
)
summary_path.write_text(json.dumps(export.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_failure_manifest_markdown(out_path.with_name(f"{out_path.stem}-failure-manifest.md"), export, selected)
return export
def _metric_rows_by_system(rows: list[dict[str, Any]], metric: str) -> dict[str, dict[str, dict[str, Any]]]:
by_system: dict[str, dict[str, dict[str, Any]]] = {}
for row in rows:
if row.get("status") != "ok":
continue
baseline = str(row.get("baseline") or "")
if not baseline:
continue
metrics = row.get("metrics")
if not isinstance(metrics, dict):
continue
value = metrics.get(metric)
if not isinstance(value, int | float) or not math.isfinite(float(value)):
continue
by_system.setdefault(baseline, {})[sample_key(row)] = row
return by_system
def _failure_entry(source: ManifestEntry, *, failure: dict[str, Any], rank: int, split: str | None) -> ManifestEntry:
metadata = dict(source.metadata or {})
if split:
existing_slices = metadata.get("slices")
slice_values = list(existing_slices) if isinstance(existing_slices, list) else []
if split not in {str(item) for item in slice_values}:
slice_values.append(split)
metadata["slices"] = slice_values
metric_name, metric_value = _failure_metric(failure)
metadata["failure_analysis"] = {
"rank": rank,
"metric": metric_name,
"metric_value": metric_value,
"worst_gap": failure.get("worst_gap"),
"comparisons": failure.get("comparisons") if isinstance(failure.get("comparisons"), dict) else {},
}
return ManifestEntry(
sample_id=source.sample_id,
dataset=source.dataset,
split=split or source.split,
image_path=source.image_path,
text=source.text,
sha256=source.sha256,
metadata=metadata,
)
def _failure_metric(failure: dict[str, Any]) -> tuple[str, float]:
for key, value in failure.items():
if key in {"badness", "worst_gap"}:
continue
if isinstance(value, int | float) and math.isfinite(float(value)):
return key, float(value)
raise BenchmarkError("top failure row has no finite metric value")
def _write_failure_manifest_markdown(path: Path, export: FailureManifestExport, selected: list[ManifestEntry]) -> None:
lines = [
"# Failure Manifest Export",
"",
f"Source manifest: `{export.source_manifest}`",
f"Top failures: `{export.top_failures_path}`",
f"Output manifest: `{export.output_manifest}`",
f"Selected samples: `{export.selected_count}`",
f"Split override: `{export.split or ''}`",
"",
"| rank | sample_id | metric | value | image_path |",
"| ---: | --- | --- | ---: | --- |",
]
for entry in selected:
failure = (entry.metadata or {}).get("failure_analysis") or {}
lines.append(
"| {rank} | {sample_id} | {metric} | {value:.6f} | {image_path} |".format(
rank=int(failure.get("rank") or 0),
sample_id=entry.sample_id,
metric=str(failure.get("metric") or ""),
value=float(failure.get("metric_value") or 0.0),
image_path=entry.image_path,
)
)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _top_failures(
system_rows: dict[str, dict[str, Any]],
rows_by_system: dict[str, dict[str, dict[str, Any]]],
baselines: list[str],
metric: str,
direction: str,
top_n: int,
) -> list[dict[str, Any]]:
ranked = [_failure_record(sample, row, rows_by_system, baselines, metric, direction) for sample, row in system_rows.items()]
ranked.sort(key=_failure_sort_key, reverse=True)
return ranked[:top_n]
def _top_failure_slice_counts(top_failures: list[dict[str, Any]]) -> list[dict[str, object]]:
counts: dict[str, int] = {}
for item in top_failures:
slices = item.get("slices")
if not isinstance(slices, list):
continue
for slice_name in slices:
text = str(slice_name)
if text:
counts[text] = counts.get(text, 0) + 1
return [{"slice": slice_name, "count": count} for slice_name, count in sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))]
def _failure_sort_key(item: dict[str, Any]) -> tuple[float, float, float]:
worst_gap = item.get("worst_gap")
badness = float(item["badness"])
if isinstance(worst_gap, int | float) and math.isfinite(float(worst_gap)):
gap = float(worst_gap)
return (1.0 if gap > 0 else 0.0, gap if gap > 0 else badness, badness)
return (0.0, badness, badness)
def _failure_record(
sample: str,
row: dict[str, Any],
rows_by_system: dict[str, dict[str, dict[str, Any]]],
baselines: list[str],
metric: str,
direction: str,
) -> dict[str, Any]:
metrics = row.get("metrics") or {}
value = float(metrics[metric])
comparisons: dict[str, dict[str, float]] = {}
gaps: list[float] = []
for baseline in baselines:
baseline_row = rows_by_system.get(baseline, {}).get(sample)
if baseline_row is None:
continue
baseline_value = float((baseline_row.get("metrics") or {})[metric])
gap = _worse_than_baseline(value, baseline_value, direction)
gaps.append(gap)
comparisons[baseline] = {
"baseline_value": baseline_value,
"worse_than_baseline": gap,
}
return {
"sample_key": sample,
"sample_id": row.get("sample_id"),
"input_path": row.get("input_path"),
"slices": sorted({str(item) for item in row.get("slices") or []}),
metric: value,
"badness": _badness(value, direction),
"worst_gap": max(gaps) if gaps else None,
"comparisons": comparisons,
}
def _slice_summary(
system_rows: dict[str, dict[str, Any]],
rows_by_system: dict[str, dict[str, dict[str, Any]]],
baselines: list[str],
metric: str,
direction: str,
) -> list[dict[str, Any]]:
slices = sorted({slice_name for row in system_rows.values() for slice_name in _row_slices(row)})
summaries = []
for slice_name in slices:
keys = sorted(sample for sample, row in system_rows.items() if slice_name in _row_slices(row))
system_values = [float((system_rows[key].get("metrics") or {})[metric]) for key in keys]
item: dict[str, Any] = {
"slice": slice_name,
"sample_count": len(system_values),
"system_mean": _mean(system_values),
"system_max": max(system_values) if system_values else None,
}
baseline_items = {}
for baseline in baselines:
shared = [key for key in keys if key in rows_by_system.get(baseline, {})]
if not shared:
continue
baseline_values = [float((rows_by_system[baseline][key].get("metrics") or {})[metric]) for key in shared]
shared_system_values = [float((system_rows[key].get("metrics") or {})[metric]) for key in shared]
baseline_items[baseline] = {
"pairs": len(shared),
"baseline_mean": _mean(baseline_values),
"mean_worse_than_baseline": _worse_than_baseline(_mean(shared_system_values), _mean(baseline_values), direction),
}
item["baselines"] = baseline_items
summaries.append(item)
summaries.sort(key=lambda item: _badness(float(item["system_mean"]), direction), reverse=True)
return summaries
def _baseline_summary(
system_rows: dict[str, dict[str, Any]],
rows_by_system: dict[str, dict[str, dict[str, Any]]],
baselines: list[str],
metric: str,
direction: str,
) -> dict[str, dict[str, float | int]]:
summary: dict[str, dict[str, float | int]] = {}
for baseline in baselines:
shared = sorted(set(system_rows) & set(rows_by_system.get(baseline, {})))
if not shared:
continue
system_values = [float((system_rows[key].get("metrics") or {})[metric]) for key in shared]
baseline_values = [float((rows_by_system[baseline][key].get("metrics") or {})[metric]) for key in shared]
gaps = [_worse_than_baseline(system_value, baseline_value, direction) for system_value, baseline_value in zip(system_values, baseline_values, strict=True)]
summary[baseline] = {
"pairs": len(shared),
"system_mean": _mean(system_values),
"baseline_mean": _mean(baseline_values),
"mean_worse_than_baseline": _mean(gaps),
"system_worse_count": sum(1 for gap in gaps if gap > 0),
"system_better_count": sum(1 for gap in gaps if gap < 0),
"tie_count": sum(1 for gap in gaps if gap == 0),
}
return summary
def _row_slices(row: dict[str, Any]) -> set[str]:
values = {str(item) for item in row.get("slices") or [] if str(item)}
values.add("all")
return values
def _badness(value: float, direction: str) -> float:
if direction == "lower":
return value
return -value
def _worse_than_baseline(system_value: float, baseline_value: float, direction: str) -> float:
if direction == "lower":
return system_value - baseline_value
return baseline_value - system_value
def _mean(values: list[float]) -> float:
if not values:
raise BenchmarkError("cannot average an empty metric series")
return sum(values) / len(values)
def _write_markdown(path: Path, payload: dict[str, Any], top_failures: list[dict[str, Any]]) -> None:
metric = str(payload["metric"])
lines = [
"# Benchmark Error Analysis",
"",
f"Report: `{payload['benchmark_report']}`",
f"System: `{payload['system']}`",
f"Metric: `{metric}`",
f"Direction: `{payload['direction']}`",
f"Samples: `{payload['sample_count']}`",
"",
"## Baseline Summary",
"",
"| baseline | pairs | system_mean | baseline_mean | mean_worse_than_baseline | system_worse | system_better | ties |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
]
baseline_summary = payload.get("baseline_summary")
if isinstance(baseline_summary, dict) and baseline_summary:
for baseline, item in sorted(baseline_summary.items()):
lines.append(
"| {baseline} | {pairs} | {system_mean:.6f} | {baseline_mean:.6f} | {gap:.6f} | {worse} | {better} | {ties} |".format(
baseline=baseline,
pairs=int(item["pairs"]),
system_mean=float(item["system_mean"]),
baseline_mean=float(item["baseline_mean"]),
gap=float(item["mean_worse_than_baseline"]),
worse=int(item["system_worse_count"]),
better=int(item["system_better_count"]),
ties=int(item["tie_count"]),
)
)
else:
lines.append("| none | 0 | | | | | | |")
lines.extend(
[
"",
"## Hardest Samples",
"",
f"| sample | slices | {metric} | worst_gap | input |",
"| --- | --- | ---: | ---: | --- |",
]
)
for item in top_failures:
gap = item["worst_gap"]
lines.append(
"| {sample} | {slices} | {value:.6f} | {gap} | {input_path} |".format(
sample=item.get("sample_id") or item["sample_key"],
slices=",".join(item.get("slices") or []),
value=float(item[metric]),
gap=f"{float(gap):.6f}" if gap is not None else "",
input_path=item.get("input_path") or "",
)
)
top_failure_counts = payload.get("top_failure_slice_counts")
lines.extend(["", "## Top Failure Slice Counts", "", "| slice | count |", "| --- | ---: |"])
if isinstance(top_failure_counts, list) and top_failure_counts:
for item in top_failure_counts:
if isinstance(item, dict):
lines.append(f"| {item.get('slice', '')} | {int(item.get('count') or 0)} |")
else:
lines.append("| none | 0 |")
lines.extend(["", f"Top failures JSON: `{payload['top_failures_path']}`"])
path.write_text("\n".join(lines) + "\n", encoding="utf-8")