File size: 3,893 Bytes
1ef1de7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech.errors import BenchmarkError
from ocrtech.paper import export_paper_benchmark_table
def test_export_paper_benchmark_table_merges_multiple_summaries(tmp_path: Path) -> None:
first = tmp_path / "first-summary.json"
second = tmp_path / "second-summary.json"
_write_summary(
first,
{
"candidate": {"all": {"sample_count": 5, "cer": 0.22, "table_cell_f1": 1.0, "latency_seconds": 4.2}},
"tesseract": {"all": {"sample_count": 5, "cer": 0.73, "table_cell_f1": 0.0, "latency_seconds": 0.2}},
},
)
_write_summary(
second,
{
"candidate": {"all": {"sample_count": 5, "cer": 0.22, "reading_order_pair_accuracy": 0.78, "latency_seconds": 4.2}},
"surya": {"all": {"sample_count": 5, "cer": 0.19, "reading_order_pair_accuracy": 0.75, "latency_seconds": 3.8}},
},
)
table = export_paper_benchmark_table(
[first, second],
tmp_path / "paper-table",
systems=["candidate", "surya", "tesseract"],
display_names={"candidate": "Candidate", "surya": "Surya", "tesseract": "Tesseract"},
metrics=["cer", "table_cell_f1", "reading_order_pair_accuracy", "latency_seconds"],
)
assert [row.system for row in table.rows] == ["candidate", "surya", "tesseract"]
assert table.rows[0].display_name == "Candidate"
assert table.rows[0].metrics["cer"] == 0.22
assert table.rows[0].metrics["table_cell_f1"] == 1.0
assert table.rows[0].metrics["reading_order_pair_accuracy"] == 0.78
assert table.rows[0].metrics["latency_seconds"] == 4.2
assert (tmp_path / "paper-table" / "paper-table.json").exists()
assert (tmp_path / "paper-table" / "paper-table.md").exists()
def test_export_paper_benchmark_table_rejects_conflicting_metric_values(tmp_path: Path) -> None:
first = tmp_path / "first-summary.json"
second = tmp_path / "second-summary.json"
_write_summary(first, {"candidate": {"all": {"sample_count": 5, "cer": 0.22}}})
_write_summary(second, {"candidate": {"all": {"sample_count": 5, "cer": 0.25}}})
with pytest.raises(BenchmarkError, match="conflicting aggregate value"):
export_paper_benchmark_table([first, second], tmp_path / "paper-table")
def test_export_paper_benchmark_table_allows_source_preference_for_conflicting_system_metrics(tmp_path: Path) -> None:
first = tmp_path / "first-summary.json"
second = tmp_path / "second-summary.json"
_write_summary(
first,
{
"candidate": {"all": {"sample_count": 5, "cer": 0.22, "latency_seconds": 4.2}},
"tesseract": {"all": {"sample_count": 5, "cer": 0.73, "latency_seconds": 0.2}},
},
)
_write_summary(
second,
{
"candidate": {"all": {"sample_count": 5, "cer": 0.22, "latency_seconds": 5.1}},
"surya": {"all": {"sample_count": 5, "cer": 0.19, "latency_seconds": 3.8}},
},
)
table = export_paper_benchmark_table(
[first, second],
tmp_path / "paper-table",
systems=["candidate", "surya", "tesseract"],
metrics=["cer", "latency_seconds"],
source_preferences={"candidate": str(first)},
)
candidate = next(row for row in table.rows if row.system == "candidate")
assert candidate.metrics["latency_seconds"] == 4.2
def _write_summary(path: Path, aggregate_metrics: dict[str, dict[str, dict[str, float | int]]]) -> None:
payload = {
"report_path": "outputs/example/report.json",
"candidate": "candidate",
"baselines": ["tesseract"],
"metrics": ["cer"],
"aggregate_metrics": aggregate_metrics,
"paired_metrics": [],
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|