from __future__ import annotations import csv import importlib.util import json import sys from argparse import Namespace from pathlib import Path def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "audit_candidate_page_line_application_effect.py" spec = importlib.util.spec_from_file_location("audit_candidate_page_line_application_effect", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _write_json(path: Path, payload: dict[str, object]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _metrics(**overrides: object) -> dict[str, object]: metrics: dict[str, object] = { "text_cer_codepoint_micro": 0.50, "text_cer_grapheme_micro": 0.60, "line_detection_f1_micro": 0.80, "line_detection_precision_micro": 0.90, "line_detection_recall_micro": 0.72, "line_unmatched_reference_count": 10, "line_unmatched_prediction_count": 2, "line_blank_prediction_count": 0, "matched_line_reading_order_pair_accuracy": 0.70, "matched_line_text_cer_codepoint_micro": 0.10, } metrics.update(overrides) return metrics def _write_application(path: Path, rows: list[dict[str, object]]) -> None: _write_json( path, { "tool": "tools/apply_candidate_page_line_reviews.py", "applied_count": len(rows), "applied_rows": rows, }, ) def _write_alignment(path: Path, rows: list[dict[str, object]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) fieldnames = [ "sample_id", "kind", "reference_line_id", "predicted_line_id", "reference_bbox", "predicted_bbox", "page_index", "iou", "reference_text", "predicted_text", "text_codepoint_edits", "text_codepoint_ref_len", "text_cer_codepoint", "text_grapheme_edits", "text_grapheme_ref_len", "text_cer_grapheme", ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() for row in rows: writer.writerow({field: row.get(field, "") for field in fieldnames}) def test_audit_candidate_page_line_application_effect_rejects_extra_line_regression(tmp_path: Path) -> None: tool = _load_tool() baseline = tmp_path / "baseline" candidate = tmp_path / "candidate" _write_json(baseline / "metrics.json", _metrics()) _write_json( candidate / "metrics.json", _metrics( text_cer_codepoint_micro=0.49, text_cer_grapheme_micro=0.59, line_detection_f1_micro=0.78, line_unmatched_prediction_count=3, matched_line_reading_order_pair_accuracy=0.65, ), ) application = tmp_path / "application.json" _write_application( application, [ { "row_id": "row-1", "sample_id": "sample-1", "candidate_line_id": "cand-1", "inserted_line_id": "insert-1", "inserted_text": "text", "selection_policy": "probe", } ], ) _write_alignment( candidate / "line-alignment.csv", [ { "sample_id": "sample-1", "kind": "prediction_unmatched", "predicted_line_id": "insert-1", "predicted_text": "text", "text_codepoint_edits": 4, "text_codepoint_ref_len": 0, "text_cer_codepoint": 1.0, } ], ) report = tool.audit_application_effect( Namespace( application_report=application, baseline_score_dir=baseline, candidate_score_dir=candidate, out=tmp_path / "audit", ) ) assert report["book_ocr_objective_safe"] is False assert report["inserted_alignment_counts"] == {"prediction_unmatched": 1} assert any("line_detection_f1_micro regresses" in issue for issue in report["issues"]) assert any("line_unmatched_prediction_count increases" in issue for issue in report["issues"]) tsv = (tmp_path / "audit" / "candidate-page-line-application-inserted-lines.tsv").read_text(encoding="utf-8") assert "reference_text" not in tsv assert "prediction_unmatched" in tsv def test_audit_candidate_page_line_application_effect_accepts_objective_safe_candidate(tmp_path: Path) -> None: tool = _load_tool() baseline = tmp_path / "baseline" candidate = tmp_path / "candidate" _write_json(baseline / "metrics.json", _metrics()) _write_json( candidate / "metrics.json", _metrics( text_cer_codepoint_micro=0.48, text_cer_grapheme_micro=0.58, line_detection_f1_micro=0.82, line_unmatched_reference_count=9, line_unmatched_prediction_count=2, matched_line_reading_order_pair_accuracy=0.72, ), ) application = tmp_path / "application.json" _write_application( application, [ { "row_id": "row-1", "sample_id": "sample-1", "candidate_line_id": "cand-1", "inserted_line_id": "insert-1", "inserted_text": "text", "selection_policy": "probe", } ], ) _write_alignment( candidate / "line-alignment.csv", [ { "sample_id": "sample-1", "kind": "matched", "reference_line_id": "ref-1", "predicted_line_id": "insert-1", "iou": 0.9, "reference_text": "hidden reference", "predicted_text": "text", "text_codepoint_edits": 0, "text_codepoint_ref_len": 4, "text_cer_codepoint": 0.0, } ], ) report = tool.audit_application_effect( Namespace( application_report=application, baseline_score_dir=baseline, candidate_score_dir=candidate, out=tmp_path / "audit", ) ) assert report["book_ocr_objective_safe"] is True assert report["issues"] == [] assert report["inserted_alignment_counts"] == {"matched": 1} tsv = (tmp_path / "audit" / "candidate-page-line-application-inserted-lines.tsv").read_text(encoding="utf-8") assert "hidden reference" not in tsv