File size: 3,893 Bytes
1ef1de7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
from __future__ import annotations

import json
from pathlib import Path

import pytest

from ocrtech.errors import BenchmarkError
from ocrtech.paper import export_paper_benchmark_table


def test_export_paper_benchmark_table_merges_multiple_summaries(tmp_path: Path) -> None:
    first = tmp_path / "first-summary.json"
    second = tmp_path / "second-summary.json"
    _write_summary(
        first,
        {
            "candidate": {"all": {"sample_count": 5, "cer": 0.22, "table_cell_f1": 1.0, "latency_seconds": 4.2}},
            "tesseract": {"all": {"sample_count": 5, "cer": 0.73, "table_cell_f1": 0.0, "latency_seconds": 0.2}},
        },
    )
    _write_summary(
        second,
        {
            "candidate": {"all": {"sample_count": 5, "cer": 0.22, "reading_order_pair_accuracy": 0.78, "latency_seconds": 4.2}},
            "surya": {"all": {"sample_count": 5, "cer": 0.19, "reading_order_pair_accuracy": 0.75, "latency_seconds": 3.8}},
        },
    )

    table = export_paper_benchmark_table(
        [first, second],
        tmp_path / "paper-table",
        systems=["candidate", "surya", "tesseract"],
        display_names={"candidate": "Candidate", "surya": "Surya", "tesseract": "Tesseract"},
        metrics=["cer", "table_cell_f1", "reading_order_pair_accuracy", "latency_seconds"],
    )

    assert [row.system for row in table.rows] == ["candidate", "surya", "tesseract"]
    assert table.rows[0].display_name == "Candidate"
    assert table.rows[0].metrics["cer"] == 0.22
    assert table.rows[0].metrics["table_cell_f1"] == 1.0
    assert table.rows[0].metrics["reading_order_pair_accuracy"] == 0.78
    assert table.rows[0].metrics["latency_seconds"] == 4.2
    assert (tmp_path / "paper-table" / "paper-table.json").exists()
    assert (tmp_path / "paper-table" / "paper-table.md").exists()


def test_export_paper_benchmark_table_rejects_conflicting_metric_values(tmp_path: Path) -> None:
    first = tmp_path / "first-summary.json"
    second = tmp_path / "second-summary.json"
    _write_summary(first, {"candidate": {"all": {"sample_count": 5, "cer": 0.22}}})
    _write_summary(second, {"candidate": {"all": {"sample_count": 5, "cer": 0.25}}})

    with pytest.raises(BenchmarkError, match="conflicting aggregate value"):
        export_paper_benchmark_table([first, second], tmp_path / "paper-table")


def test_export_paper_benchmark_table_allows_source_preference_for_conflicting_system_metrics(tmp_path: Path) -> None:
    first = tmp_path / "first-summary.json"
    second = tmp_path / "second-summary.json"
    _write_summary(
        first,
        {
            "candidate": {"all": {"sample_count": 5, "cer": 0.22, "latency_seconds": 4.2}},
            "tesseract": {"all": {"sample_count": 5, "cer": 0.73, "latency_seconds": 0.2}},
        },
    )
    _write_summary(
        second,
        {
            "candidate": {"all": {"sample_count": 5, "cer": 0.22, "latency_seconds": 5.1}},
            "surya": {"all": {"sample_count": 5, "cer": 0.19, "latency_seconds": 3.8}},
        },
    )

    table = export_paper_benchmark_table(
        [first, second],
        tmp_path / "paper-table",
        systems=["candidate", "surya", "tesseract"],
        metrics=["cer", "latency_seconds"],
        source_preferences={"candidate": str(first)},
    )

    candidate = next(row for row in table.rows if row.system == "candidate")
    assert candidate.metrics["latency_seconds"] == 4.2


def _write_summary(path: Path, aggregate_metrics: dict[str, dict[str, dict[str, float | int]]]) -> None:
    payload = {
        "report_path": "outputs/example/report.json",
        "candidate": "candidate",
        "baselines": ["tesseract"],
        "metrics": ["cer"],
        "aggregate_metrics": aggregate_metrics,
        "paired_metrics": [],
    }
    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")