from __future__ import annotations import json from pathlib import Path import pytest from ocrtech.errors import BenchmarkError from ocrtech.paper import export_paper_benchmark_table def test_export_paper_benchmark_table_merges_multiple_summaries(tmp_path: Path) -> None: first = tmp_path / "first-summary.json" second = tmp_path / "second-summary.json" _write_summary( first, { "candidate": {"all": {"sample_count": 5, "cer": 0.22, "table_cell_f1": 1.0, "latency_seconds": 4.2}}, "tesseract": {"all": {"sample_count": 5, "cer": 0.73, "table_cell_f1": 0.0, "latency_seconds": 0.2}}, }, ) _write_summary( second, { "candidate": {"all": {"sample_count": 5, "cer": 0.22, "reading_order_pair_accuracy": 0.78, "latency_seconds": 4.2}}, "surya": {"all": {"sample_count": 5, "cer": 0.19, "reading_order_pair_accuracy": 0.75, "latency_seconds": 3.8}}, }, ) table = export_paper_benchmark_table( [first, second], tmp_path / "paper-table", systems=["candidate", "surya", "tesseract"], display_names={"candidate": "Candidate", "surya": "Surya", "tesseract": "Tesseract"}, metrics=["cer", "table_cell_f1", "reading_order_pair_accuracy", "latency_seconds"], ) assert [row.system for row in table.rows] == ["candidate", "surya", "tesseract"] assert table.rows[0].display_name == "Candidate" assert table.rows[0].metrics["cer"] == 0.22 assert table.rows[0].metrics["table_cell_f1"] == 1.0 assert table.rows[0].metrics["reading_order_pair_accuracy"] == 0.78 assert table.rows[0].metrics["latency_seconds"] == 4.2 assert (tmp_path / "paper-table" / "paper-table.json").exists() assert (tmp_path / "paper-table" / "paper-table.md").exists() def test_export_paper_benchmark_table_rejects_conflicting_metric_values(tmp_path: Path) -> None: first = tmp_path / "first-summary.json" second = tmp_path / "second-summary.json" _write_summary(first, {"candidate": {"all": {"sample_count": 5, "cer": 0.22}}}) _write_summary(second, {"candidate": {"all": {"sample_count": 5, "cer": 0.25}}}) with pytest.raises(BenchmarkError, match="conflicting aggregate value"): export_paper_benchmark_table([first, second], tmp_path / "paper-table") def test_export_paper_benchmark_table_allows_source_preference_for_conflicting_system_metrics(tmp_path: Path) -> None: first = tmp_path / "first-summary.json" second = tmp_path / "second-summary.json" _write_summary( first, { "candidate": {"all": {"sample_count": 5, "cer": 0.22, "latency_seconds": 4.2}}, "tesseract": {"all": {"sample_count": 5, "cer": 0.73, "latency_seconds": 0.2}}, }, ) _write_summary( second, { "candidate": {"all": {"sample_count": 5, "cer": 0.22, "latency_seconds": 5.1}}, "surya": {"all": {"sample_count": 5, "cer": 0.19, "latency_seconds": 3.8}}, }, ) table = export_paper_benchmark_table( [first, second], tmp_path / "paper-table", systems=["candidate", "surya", "tesseract"], metrics=["cer", "latency_seconds"], source_preferences={"candidate": str(first)}, ) candidate = next(row for row in table.rows if row.system == "candidate") assert candidate.metrics["latency_seconds"] == 4.2 def _write_summary(path: Path, aggregate_metrics: dict[str, dict[str, dict[str, float | int]]]) -> None: payload = { "report_path": "outputs/example/report.json", "candidate": "candidate", "baselines": ["tesseract"], "metrics": ["cer"], "aggregate_metrics": aggregate_metrics, "paired_metrics": [], } path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")