Download ocr-tech/tests/test_audit_candidate_page_line_application_effect.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 6.85 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_audit_candidate_page_line_application_effect.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tests/test_audit_candidate_page_line_application_effect.py
-
curl -L -o test_audit_candidate_page_line_application_effect.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tests/test_audit_candidate_page_line_application_effect.py
6.85 kB
| from __future__ import annotations | |
| import csv | |
| import importlib.util | |
| import json | |
| import sys | |
| from argparse import Namespace | |
| from pathlib import Path | |
| def _load_tool(): | |
| path = Path(__file__).resolve().parents[1] / "tools" / "audit_candidate_page_line_application_effect.py" | |
| spec = importlib.util.spec_from_file_location("audit_candidate_page_line_application_effect", path) | |
| assert spec is not None | |
| assert spec.loader is not None | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[spec.name] = module | |
| spec.loader.exec_module(module) | |
| return module | |
| def _write_json(path: Path, payload: dict[str, object]) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| def _metrics(**overrides: object) -> dict[str, object]: | |
| metrics: dict[str, object] = { | |
| "text_cer_codepoint_micro": 0.50, | |
| "text_cer_grapheme_micro": 0.60, | |
| "line_detection_f1_micro": 0.80, | |
| "line_detection_precision_micro": 0.90, | |
| "line_detection_recall_micro": 0.72, | |
| "line_unmatched_reference_count": 10, | |
| "line_unmatched_prediction_count": 2, | |
| "line_blank_prediction_count": 0, | |
| "matched_line_reading_order_pair_accuracy": 0.70, | |
| "matched_line_text_cer_codepoint_micro": 0.10, | |
| } | |
| metrics.update(overrides) | |
| return metrics | |
| def _write_application(path: Path, rows: list[dict[str, object]]) -> None: | |
| _write_json( | |
| path, | |
| { | |
| "tool": "tools/apply_candidate_page_line_reviews.py", | |
| "applied_count": len(rows), | |
| "applied_rows": rows, | |
| }, | |
| ) | |
| def _write_alignment(path: Path, rows: list[dict[str, object]]) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| fieldnames = [ | |
| "sample_id", | |
| "kind", | |
| "reference_line_id", | |
| "predicted_line_id", | |
| "reference_bbox", | |
| "predicted_bbox", | |
| "page_index", | |
| "iou", | |
| "reference_text", | |
| "predicted_text", | |
| "text_codepoint_edits", | |
| "text_codepoint_ref_len", | |
| "text_cer_codepoint", | |
| "text_grapheme_edits", | |
| "text_grapheme_ref_len", | |
| "text_cer_grapheme", | |
| ] | |
| with path.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fieldnames) | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({field: row.get(field, "") for field in fieldnames}) | |
| def test_audit_candidate_page_line_application_effect_rejects_extra_line_regression(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| baseline = tmp_path / "baseline" | |
| candidate = tmp_path / "candidate" | |
| _write_json(baseline / "metrics.json", _metrics()) | |
| _write_json( | |
| candidate / "metrics.json", | |
| _metrics( | |
| text_cer_codepoint_micro=0.49, | |
| text_cer_grapheme_micro=0.59, | |
| line_detection_f1_micro=0.78, | |
| line_unmatched_prediction_count=3, | |
| matched_line_reading_order_pair_accuracy=0.65, | |
| ), | |
| ) | |
| application = tmp_path / "application.json" | |
| _write_application( | |
| application, | |
| [ | |
| { | |
| "row_id": "row-1", | |
| "sample_id": "sample-1", | |
| "candidate_line_id": "cand-1", | |
| "inserted_line_id": "insert-1", | |
| "inserted_text": "text", | |
| "selection_policy": "probe", | |
| } | |
| ], | |
| ) | |
| _write_alignment( | |
| candidate / "line-alignment.csv", | |
| [ | |
| { | |
| "sample_id": "sample-1", | |
| "kind": "prediction_unmatched", | |
| "predicted_line_id": "insert-1", | |
| "predicted_text": "text", | |
| "text_codepoint_edits": 4, | |
| "text_codepoint_ref_len": 0, | |
| "text_cer_codepoint": 1.0, | |
| } | |
| ], | |
| ) | |
| report = tool.audit_application_effect( | |
| Namespace( | |
| application_report=application, | |
| baseline_score_dir=baseline, | |
| candidate_score_dir=candidate, | |
| out=tmp_path / "audit", | |
| ) | |
| ) | |
| assert report["book_ocr_objective_safe"] is False | |
| assert report["inserted_alignment_counts"] == {"prediction_unmatched": 1} | |
| assert any("line_detection_f1_micro regresses" in issue for issue in report["issues"]) | |
| assert any("line_unmatched_prediction_count increases" in issue for issue in report["issues"]) | |
| tsv = (tmp_path / "audit" / "candidate-page-line-application-inserted-lines.tsv").read_text(encoding="utf-8") | |
| assert "reference_text" not in tsv | |
| assert "prediction_unmatched" in tsv | |
| def test_audit_candidate_page_line_application_effect_accepts_objective_safe_candidate(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| baseline = tmp_path / "baseline" | |
| candidate = tmp_path / "candidate" | |
| _write_json(baseline / "metrics.json", _metrics()) | |
| _write_json( | |
| candidate / "metrics.json", | |
| _metrics( | |
| text_cer_codepoint_micro=0.48, | |
| text_cer_grapheme_micro=0.58, | |
| line_detection_f1_micro=0.82, | |
| line_unmatched_reference_count=9, | |
| line_unmatched_prediction_count=2, | |
| matched_line_reading_order_pair_accuracy=0.72, | |
| ), | |
| ) | |
| application = tmp_path / "application.json" | |
| _write_application( | |
| application, | |
| [ | |
| { | |
| "row_id": "row-1", | |
| "sample_id": "sample-1", | |
| "candidate_line_id": "cand-1", | |
| "inserted_line_id": "insert-1", | |
| "inserted_text": "text", | |
| "selection_policy": "probe", | |
| } | |
| ], | |
| ) | |
| _write_alignment( | |
| candidate / "line-alignment.csv", | |
| [ | |
| { | |
| "sample_id": "sample-1", | |
| "kind": "matched", | |
| "reference_line_id": "ref-1", | |
| "predicted_line_id": "insert-1", | |
| "iou": 0.9, | |
| "reference_text": "hidden reference", | |
| "predicted_text": "text", | |
| "text_codepoint_edits": 0, | |
| "text_codepoint_ref_len": 4, | |
| "text_cer_codepoint": 0.0, | |
| } | |
| ], | |
| ) | |
| report = tool.audit_application_effect( | |
| Namespace( | |
| application_report=application, | |
| baseline_score_dir=baseline, | |
| candidate_score_dir=candidate, | |
| out=tmp_path / "audit", | |
| ) | |
| ) | |
| assert report["book_ocr_objective_safe"] is True | |
| assert report["issues"] == [] | |
| assert report["inserted_alignment_counts"] == {"matched": 1} | |
| tsv = (tmp_path / "audit" / "candidate-page-line-application-inserted-lines.tsv").read_text(encoding="utf-8") | |
| assert "hidden reference" not in tsv | |