from __future__ import annotations import hashlib import importlib.util import json import sys from argparse import Namespace from pathlib import Path def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "admit_page_ocr_line_pruning_threshold.py" spec = importlib.util.spec_from_file_location("admit_page_ocr_line_pruning_threshold", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def _write_json(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") def _candidate(threshold: float, cp: float, gr: float) -> dict[str, object]: return { "threshold": threshold, "threshold_id": f"conf{int(round(threshold * 100)):03d}", "documents_audit_passed": True, "comparison_claim_safe": False, "codepoint_absolute_improvement": cp, "grapheme_absolute_improvement": gr, "text_cer_codepoint_micro": 0.5 - cp, "text_cer_grapheme_micro": 0.6 - gr, "kept_line_count": 10, "removed_line_count": 2, "documents_dir": f"outputs/documents/{threshold}", "score_dir": f"outputs/score/{threshold}", "comparison_dir": f"outputs/comparison/{threshold}", "documents_filter_run_sha256": "filter-run", "documents_line_report_sha256": "line-report", "documents_audit_sha256": "audit", "page_eval_run_sha256": "page-run", "metrics_sha256": "metrics", "comparison_sha256": "comparison", "comparison_warnings": ["not claim-safe"], } def _sweep(path: Path, label: str, candidates: list[dict[str, object]]) -> Path: payload = { "source_documents_dir": f"outputs/{label}", "claim_safe": False, "candidates": candidates, } _write_json(path, payload) return path def test_admit_threshold_selects_common_non_regressing_candidate(tmp_path: Path) -> None: tool = _load_tool() first = _sweep( tmp_path / "first" / "ocr-line-pruning-sweep.json", "first-page", [_candidate(0.7, 0.02, 0.03), _candidate(0.85, 0.04, 0.04)], ) second = _sweep( tmp_path / "second" / "ocr-line-pruning-sweep.json", "second-page", [_candidate(0.7, 0.01, 0.02), _candidate(0.85, -0.20, -0.10)], ) report = tool.admit_threshold( Namespace( sweep=[first, second], out=tmp_path / "out", min_sweeps=2, require_common_thresholds=True, min_codepoint_improvement=0.0, max_codepoint_regression=0.0, max_grapheme_regression=0.0, ) ) assert report["selected_threshold"] == 0.7 assert report["admitted_candidate_count"] == 1 assert report["selected_candidate"]["per_page"][0]["documents_dir"] == "outputs/documents/0.7" assert report["selected_candidate"]["per_page"][0]["page_eval_run_sha256"] == "page-run" rejected = [row for row in report["candidates"] if row["threshold"] == 0.85][0] assert rejected["admitted_operational"] is False assert any("regresses" in issue for issue in rejected["issues"]) assert report["sweeps"][0]["sha256"] == _sha256(first) def test_admit_threshold_can_require_positive_improvement_on_every_page(tmp_path: Path) -> None: tool = _load_tool() first = _sweep(tmp_path / "first.json", "first-page", [_candidate(0.5, 0.02, 0.01)]) second = _sweep(tmp_path / "second.json", "second-page", [_candidate(0.5, 0.0, 0.01)]) report = tool.admit_threshold( Namespace( sweep=[first, second], out=tmp_path / "out", min_sweeps=2, require_common_thresholds=True, min_codepoint_improvement=0.001, max_codepoint_regression=0.0, max_grapheme_regression=0.0, ) ) assert report["selected_threshold"] is None assert report["admitted_candidate_count"] == 0 assert any("minimum codepoint improvement" in issue for issue in report["candidates"][0]["issues"])