ocr-projects-archive / ocr-tech /tests /test_audit_candidate_page_line_application_effect.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
6.85 kB
from __future__ import annotations
import csv
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
def _load_tool():
path = Path(__file__).resolve().parents[1] / "tools" / "audit_candidate_page_line_application_effect.py"
spec = importlib.util.spec_from_file_location("audit_candidate_page_line_application_effect", path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _write_json(path: Path, payload: dict[str, object]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _metrics(**overrides: object) -> dict[str, object]:
metrics: dict[str, object] = {
"text_cer_codepoint_micro": 0.50,
"text_cer_grapheme_micro": 0.60,
"line_detection_f1_micro": 0.80,
"line_detection_precision_micro": 0.90,
"line_detection_recall_micro": 0.72,
"line_unmatched_reference_count": 10,
"line_unmatched_prediction_count": 2,
"line_blank_prediction_count": 0,
"matched_line_reading_order_pair_accuracy": 0.70,
"matched_line_text_cer_codepoint_micro": 0.10,
}
metrics.update(overrides)
return metrics
def _write_application(path: Path, rows: list[dict[str, object]]) -> None:
_write_json(
path,
{
"tool": "tools/apply_candidate_page_line_reviews.py",
"applied_count": len(rows),
"applied_rows": rows,
},
)
def _write_alignment(path: Path, rows: list[dict[str, object]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fieldnames = [
"sample_id",
"kind",
"reference_line_id",
"predicted_line_id",
"reference_bbox",
"predicted_bbox",
"page_index",
"iou",
"reference_text",
"predicted_text",
"text_codepoint_edits",
"text_codepoint_ref_len",
"text_cer_codepoint",
"text_grapheme_edits",
"text_grapheme_ref_len",
"text_cer_grapheme",
]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({field: row.get(field, "") for field in fieldnames})
def test_audit_candidate_page_line_application_effect_rejects_extra_line_regression(tmp_path: Path) -> None:
tool = _load_tool()
baseline = tmp_path / "baseline"
candidate = tmp_path / "candidate"
_write_json(baseline / "metrics.json", _metrics())
_write_json(
candidate / "metrics.json",
_metrics(
text_cer_codepoint_micro=0.49,
text_cer_grapheme_micro=0.59,
line_detection_f1_micro=0.78,
line_unmatched_prediction_count=3,
matched_line_reading_order_pair_accuracy=0.65,
),
)
application = tmp_path / "application.json"
_write_application(
application,
[
{
"row_id": "row-1",
"sample_id": "sample-1",
"candidate_line_id": "cand-1",
"inserted_line_id": "insert-1",
"inserted_text": "text",
"selection_policy": "probe",
}
],
)
_write_alignment(
candidate / "line-alignment.csv",
[
{
"sample_id": "sample-1",
"kind": "prediction_unmatched",
"predicted_line_id": "insert-1",
"predicted_text": "text",
"text_codepoint_edits": 4,
"text_codepoint_ref_len": 0,
"text_cer_codepoint": 1.0,
}
],
)
report = tool.audit_application_effect(
Namespace(
application_report=application,
baseline_score_dir=baseline,
candidate_score_dir=candidate,
out=tmp_path / "audit",
)
)
assert report["book_ocr_objective_safe"] is False
assert report["inserted_alignment_counts"] == {"prediction_unmatched": 1}
assert any("line_detection_f1_micro regresses" in issue for issue in report["issues"])
assert any("line_unmatched_prediction_count increases" in issue for issue in report["issues"])
tsv = (tmp_path / "audit" / "candidate-page-line-application-inserted-lines.tsv").read_text(encoding="utf-8")
assert "reference_text" not in tsv
assert "prediction_unmatched" in tsv
def test_audit_candidate_page_line_application_effect_accepts_objective_safe_candidate(tmp_path: Path) -> None:
tool = _load_tool()
baseline = tmp_path / "baseline"
candidate = tmp_path / "candidate"
_write_json(baseline / "metrics.json", _metrics())
_write_json(
candidate / "metrics.json",
_metrics(
text_cer_codepoint_micro=0.48,
text_cer_grapheme_micro=0.58,
line_detection_f1_micro=0.82,
line_unmatched_reference_count=9,
line_unmatched_prediction_count=2,
matched_line_reading_order_pair_accuracy=0.72,
),
)
application = tmp_path / "application.json"
_write_application(
application,
[
{
"row_id": "row-1",
"sample_id": "sample-1",
"candidate_line_id": "cand-1",
"inserted_line_id": "insert-1",
"inserted_text": "text",
"selection_policy": "probe",
}
],
)
_write_alignment(
candidate / "line-alignment.csv",
[
{
"sample_id": "sample-1",
"kind": "matched",
"reference_line_id": "ref-1",
"predicted_line_id": "insert-1",
"iou": 0.9,
"reference_text": "hidden reference",
"predicted_text": "text",
"text_codepoint_edits": 0,
"text_codepoint_ref_len": 4,
"text_cer_codepoint": 0.0,
}
],
)
report = tool.audit_application_effect(
Namespace(
application_report=application,
baseline_score_dir=baseline,
candidate_score_dir=candidate,
out=tmp_path / "audit",
)
)
assert report["book_ocr_objective_safe"] is True
assert report["issues"] == []
assert report["inserted_alignment_counts"] == {"matched": 1}
tsv = (tmp_path / "audit" / "candidate-page-line-application-inserted-lines.tsv").read_text(encoding="utf-8")
assert "hidden reference" not in tsv