ocr-projects-archive / ocr-tech /tests /test_audit_admitted_page_ocr_candidate.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
7.4 kB
from __future__ import annotations
import csv
import hashlib
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
def _load_tool():
path = Path(__file__).resolve().parents[1] / "tools" / "audit_admitted_page_ocr_candidate.py"
spec = importlib.util.spec_from_file_location("audit_admitted_page_ocr_candidate", path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _write_json(path: Path, payload: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
def _write_alignment(path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
rows = [
{
"sample_id": "page-1",
"kind": "matched",
"reference_line_id": "ref-1",
"predicted_line_id": "pred-1",
"reference_bbox": "[0,0,10,10]",
"predicted_bbox": "[0,0,10,10]",
"page_index": "0",
"iou": "1.0",
"reference_text": "abc",
"predicted_text": "abc",
"text_codepoint_edits": "0",
"text_codepoint_ref_len": "3",
"text_cer_codepoint": "0.0",
"text_grapheme_edits": "0",
"text_grapheme_ref_len": "3",
"text_cer_grapheme": "0.0",
},
{
"sample_id": "page-1",
"kind": "reference_unmatched",
"reference_line_id": "ref-2",
"predicted_line_id": "",
"reference_bbox": "[0,20,10,10]",
"predicted_bbox": "",
"page_index": "0",
"iou": "0.6",
"reference_text": "missing",
"predicted_text": "",
"text_codepoint_edits": "7",
"text_codepoint_ref_len": "7",
"text_cer_codepoint": "1.0",
"text_grapheme_edits": "7",
"text_grapheme_ref_len": "7",
"text_cer_grapheme": "1.0",
},
{
"sample_id": "page-1",
"kind": "prediction_unmatched",
"reference_line_id": "",
"predicted_line_id": "pred-2",
"reference_bbox": "",
"predicted_bbox": "[50,50,10,10]",
"page_index": "0",
"iou": "0.0",
"reference_text": "",
"predicted_text": "extra",
"text_codepoint_edits": "5",
"text_codepoint_ref_len": "0",
"text_cer_codepoint": "0.0",
"text_grapheme_edits": "5",
"text_grapheme_ref_len": "0",
"text_cer_grapheme": "0.0",
},
]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys()))
writer.writeheader()
writer.writerows(rows)
def test_audit_admitted_candidate_summarizes_residuals(tmp_path: Path) -> None:
tool = _load_tool()
score_dir = tmp_path / "score"
_write_alignment(score_dir / "line-alignment.csv")
_write_json(score_dir / "page-eval-run.json", {"claim_safe": False})
_write_json(score_dir / "metrics.json", {"text_cer_codepoint_micro": 0.5})
admission = {
"selected_threshold": 0.7,
"selected_threshold_id": "conf070",
"selected_candidate": {
"per_page": [
{
"label": "fixture-page",
"score_dir": str(score_dir),
"page_eval_run_sha256": _sha256(score_dir / "page-eval-run.json"),
"metrics_sha256": _sha256(score_dir / "metrics.json"),
}
]
},
}
admission_path = tmp_path / "admission.json"
_write_json(admission_path, admission)
report = tool.audit_candidate(
Namespace(
admission_report=admission_path,
out=tmp_path / "out",
high_iou_unmatched_threshold=0.5,
near_miss_iou_threshold=0.3,
bad_codepoint_cer=0.25,
bad_grapheme_cer=0.35,
top_n=5,
)
)
page = report["pages"][0]
assert report["selected_threshold"] == 0.7
assert report["totals"]["matched"] == 1
assert report["totals"]["reference_unmatched"] == 1
assert report["totals"]["prediction_unmatched"] == 1
assert report["totals"]["high_iou_unmatched_references"] == 1
assert report["artifact_integrity_passed"] is True
assert report["artifact_integrity_issues"] == []
assert page["rates"]["reference_recall"] == 0.5
assert page["line_alignment_sha256"] == _sha256(score_dir / "line-alignment.csv")
assert page["artifact_integrity"]["verified"] is True
assert [check["label"] for check in page["artifact_integrity"]["checks"]] == ["page_eval_run", "metrics"]
assert page["page_extent"] == {
"min_x": 0.0,
"min_y": 0.0,
"max_x": 60.0,
"max_y": 60.0,
"width": 60.0,
"height": 60.0,
}
assert page["geometry_summary"]["unmatched_references"]["by_vertical_zone"] == {"middle": 1}
assert page["geometry_summary"]["unmatched_references"]["by_width_bin"] == {"short": 1}
assert page["geometry_summary"]["unmatched_predictions"]["by_horizontal_zone"] == {"right": 1}
assert page["top_high_iou_unmatched_references"][0]["reference_line_id"] == "ref-2"
assert page["top_high_iou_unmatched_references"][0]["reference_geometry"]["vertical_zone"] == "middle"
assert page["top_unmatched_predictions"][0]["predicted_geometry"]["horizontal_zone"] == "right"
def test_audit_admitted_candidate_rejects_stale_score_hashes(tmp_path: Path) -> None:
tool = _load_tool()
score_dir = tmp_path / "score"
_write_alignment(score_dir / "line-alignment.csv")
page_run = score_dir / "page-eval-run.json"
metrics = score_dir / "metrics.json"
_write_json(page_run, {"claim_safe": False})
_write_json(metrics, {"text_cer_codepoint_micro": 0.5})
recorded_page_hash = _sha256(page_run)
_write_json(page_run, {"claim_safe": True})
admission = {
"selected_threshold": 0.7,
"selected_threshold_id": "conf070",
"selected_candidate": {
"per_page": [
{
"label": "fixture-page",
"score_dir": str(score_dir),
"page_eval_run_sha256": recorded_page_hash,
"metrics_sha256": _sha256(metrics),
}
]
},
}
admission_path = tmp_path / "admission.json"
_write_json(admission_path, admission)
report = tool.audit_candidate(
Namespace(
admission_report=admission_path,
out=tmp_path / "out",
high_iou_unmatched_threshold=0.5,
near_miss_iou_threshold=0.3,
bad_codepoint_cer=0.25,
bad_grapheme_cer=0.35,
top_n=5,
)
)
assert report["artifact_integrity_passed"] is False
assert any("page_eval_run sha256 mismatch" in issue for issue in report["artifact_integrity_issues"])
assert report["pages"][0]["artifact_integrity"]["verified"] is False