File size: 4,357 Bytes
f9bc0c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
from __future__ import annotations

import hashlib
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path


def _load_tool():
    path = Path(__file__).resolve().parents[1] / "tools" / "admit_page_ocr_line_pruning_threshold.py"
    spec = importlib.util.spec_from_file_location("admit_page_ocr_line_pruning_threshold", path)
    assert spec is not None
    assert spec.loader is not None
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


def _sha256(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


def _write_json(path: Path, payload: object) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")


def _candidate(threshold: float, cp: float, gr: float) -> dict[str, object]:
    return {
        "threshold": threshold,
        "threshold_id": f"conf{int(round(threshold * 100)):03d}",
        "documents_audit_passed": True,
        "comparison_claim_safe": False,
        "codepoint_absolute_improvement": cp,
        "grapheme_absolute_improvement": gr,
        "text_cer_codepoint_micro": 0.5 - cp,
        "text_cer_grapheme_micro": 0.6 - gr,
        "kept_line_count": 10,
        "removed_line_count": 2,
        "documents_dir": f"outputs/documents/{threshold}",
        "score_dir": f"outputs/score/{threshold}",
        "comparison_dir": f"outputs/comparison/{threshold}",
        "documents_filter_run_sha256": "filter-run",
        "documents_line_report_sha256": "line-report",
        "documents_audit_sha256": "audit",
        "page_eval_run_sha256": "page-run",
        "metrics_sha256": "metrics",
        "comparison_sha256": "comparison",
        "comparison_warnings": ["not claim-safe"],
    }


def _sweep(path: Path, label: str, candidates: list[dict[str, object]]) -> Path:
    payload = {
        "source_documents_dir": f"outputs/{label}",
        "claim_safe": False,
        "candidates": candidates,
    }
    _write_json(path, payload)
    return path


def test_admit_threshold_selects_common_non_regressing_candidate(tmp_path: Path) -> None:
    tool = _load_tool()
    first = _sweep(
        tmp_path / "first" / "ocr-line-pruning-sweep.json",
        "first-page",
        [_candidate(0.7, 0.02, 0.03), _candidate(0.85, 0.04, 0.04)],
    )
    second = _sweep(
        tmp_path / "second" / "ocr-line-pruning-sweep.json",
        "second-page",
        [_candidate(0.7, 0.01, 0.02), _candidate(0.85, -0.20, -0.10)],
    )

    report = tool.admit_threshold(
        Namespace(
            sweep=[first, second],
            out=tmp_path / "out",
            min_sweeps=2,
            require_common_thresholds=True,
            min_codepoint_improvement=0.0,
            max_codepoint_regression=0.0,
            max_grapheme_regression=0.0,
        )
    )

    assert report["selected_threshold"] == 0.7
    assert report["admitted_candidate_count"] == 1
    assert report["selected_candidate"]["per_page"][0]["documents_dir"] == "outputs/documents/0.7"
    assert report["selected_candidate"]["per_page"][0]["page_eval_run_sha256"] == "page-run"
    rejected = [row for row in report["candidates"] if row["threshold"] == 0.85][0]
    assert rejected["admitted_operational"] is False
    assert any("regresses" in issue for issue in rejected["issues"])
    assert report["sweeps"][0]["sha256"] == _sha256(first)


def test_admit_threshold_can_require_positive_improvement_on_every_page(tmp_path: Path) -> None:
    tool = _load_tool()
    first = _sweep(tmp_path / "first.json", "first-page", [_candidate(0.5, 0.02, 0.01)])
    second = _sweep(tmp_path / "second.json", "second-page", [_candidate(0.5, 0.0, 0.01)])

    report = tool.admit_threshold(
        Namespace(
            sweep=[first, second],
            out=tmp_path / "out",
            min_sweeps=2,
            require_common_thresholds=True,
            min_codepoint_improvement=0.001,
            max_codepoint_regression=0.0,
            max_grapheme_regression=0.0,
        )
    )

    assert report["selected_threshold"] is None
    assert report["admitted_candidate_count"] == 0
    assert any("minimum codepoint improvement" in issue for issue in report["candidates"][0]["issues"])