Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/audit_candidate_line_review_application.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 13.1 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_candidate_line_review_application.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/audit_candidate_line_review_application.py
-
curl -L -o audit_candidate_line_review_application.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_candidate_line_review_application.py
13.1 kB
| #!/usr/bin/env python3 | |
| """Audit a candidate-line review application output.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import re | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import sha256_file | |
| from ocrtech.normalization import normalize_ocr_text | |
| from ocrtech.schemas import Document | |
| def _read_json_object(path: Path, issues: list[str], *, label: str) -> dict[str, Any] | None: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError: | |
| issues.append(f"{label} is missing: {path}") | |
| return None | |
| except json.JSONDecodeError as exc: | |
| issues.append(f"{label} is invalid JSON: {path} ({exc})") | |
| return None | |
| if not isinstance(payload, dict): | |
| issues.append(f"{label} must be a JSON object: {path}") | |
| return None | |
| return payload | |
| def _read_jsonl(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]: | |
| if not path.is_file(): | |
| issues.append(f"{label} is missing: {path}") | |
| return [] | |
| rows: list[dict[str, Any]] = [] | |
| for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): | |
| if not line.strip(): | |
| continue | |
| try: | |
| payload = json.loads(line) | |
| except json.JSONDecodeError as exc: | |
| issues.append(f"{label} invalid JSON at line {line_number}: {exc}") | |
| continue | |
| if not isinstance(payload, dict): | |
| issues.append(f"{label} row {line_number} must be a JSON object") | |
| continue | |
| rows.append(payload) | |
| return rows | |
| def _read_tsv(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]: | |
| if not path.is_file(): | |
| issues.append(f"{label} is missing: {path}") | |
| return [] | |
| rows: list[dict[str, Any]] = [] | |
| with path.open("r", encoding="utf-8", newline="") as handle: | |
| reader = csv.DictReader(handle, delimiter="\t") | |
| if reader.fieldnames is None: | |
| issues.append(f"{label} has no header: {path}") | |
| return [] | |
| for row_number, row in enumerate(reader, start=2): | |
| payload = {key: value for key, value in row.items() if key is not None} | |
| payload["_row_number"] = row_number | |
| rows.append(payload) | |
| return rows | |
| def _read_review_rows(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]: | |
| if path.suffix.lower() == ".tsv": | |
| return _read_tsv(path, issues, label=label) | |
| return _read_jsonl(path, issues, label=label) | |
| def _check_file_sha(path: Path, expected: object, issues: list[str], *, label: str) -> None: | |
| if not path.is_file(): | |
| issues.append(f"{label} is missing: {path}") | |
| return | |
| if not isinstance(expected, str) or not expected: | |
| issues.append(f"{label} missing recorded SHA-256") | |
| return | |
| actual = sha256_file(path) | |
| if actual != expected: | |
| issues.append(f"{label} SHA-256 mismatch: recorded={expected} actual={actual} path={path}") | |
| def _candidate_inserted_lines(document: Document) -> list[Any]: | |
| lines: list[Any] = [] | |
| for page in document.pages: | |
| for line in page.text_lines: | |
| metadata = line.metadata if isinstance(line.metadata, dict) else {} | |
| if isinstance(metadata.get("candidate_line_review_insertion"), dict): | |
| lines.append(line) | |
| return lines | |
| def _candidate_inserted_lines_from_review(document: Document, review_path: Path) -> list[Any]: | |
| expected = str(review_path) | |
| lines: list[Any] = [] | |
| for line in _candidate_inserted_lines(document): | |
| metadata = line.metadata if isinstance(line.metadata, dict) else {} | |
| insertion = metadata.get("candidate_line_review_insertion") | |
| if not isinstance(insertion, dict): | |
| continue | |
| if str(insertion.get("source_review_jsonl") or "") == expected: | |
| lines.append(line) | |
| return lines | |
| def _validate_current_inserted_lines(lines: list[Any], issues: list[str], *, page_id: str) -> None: | |
| for line in lines: | |
| metadata = line.metadata if isinstance(line.metadata, dict) else {} | |
| insertion = metadata.get("candidate_line_review_insertion") | |
| if not isinstance(insertion, dict): | |
| continue | |
| line_id = str(getattr(line, "line_id", "") or "") | |
| corrected_text = normalize_ocr_text(str(insertion.get("corrected_text") or "")) | |
| inserted_text = normalize_ocr_text(str(insertion.get("inserted_text") or "")) | |
| line_text = normalize_ocr_text(str(getattr(line, "text", "") or "")) | |
| if not corrected_text.strip(): | |
| issues.append(f"{page_id} inserted candidate line {line_id!r} missing corrected_text") | |
| if not inserted_text.strip(): | |
| issues.append(f"{page_id} inserted candidate line {line_id!r} missing inserted_text") | |
| if inserted_text.strip() and line_text != inserted_text: | |
| issues.append( | |
| f"{page_id} inserted candidate line {line_id!r} text does not match inserted_text: " | |
| f"line={line_text!r} inserted={inserted_text!r}" | |
| ) | |
| if corrected_text.strip() and line_text != corrected_text: | |
| issues.append( | |
| f"{page_id} inserted candidate line {line_id!r} text does not match corrected_text: " | |
| f"line={line_text!r} corrected={corrected_text!r}" | |
| ) | |
| def _oracle_or_reference_assisted_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: | |
| flagged: list[dict[str, Any]] = [] | |
| for index, row in enumerate(rows, start=1): | |
| values = [ | |
| row.get("selection_policy"), | |
| row.get("claim_boundary"), | |
| row.get("reviewer"), | |
| row.get("notes"), | |
| ] | |
| keys = set(row) | |
| text = " ".join(str(value or "") for value in values).lower() | |
| text = re.sub(r"\bnon[-_ ]oracle\b", " ", text) | |
| has_oracle_marker = bool( | |
| re.search(r"\boracle(?:_|-|$|\s)", text) | |
| or "reference_overlap" in text | |
| or any(str(key).startswith("oracle_") for key in keys) | |
| ) | |
| if has_oracle_marker: | |
| flagged.append( | |
| { | |
| "row_number": index, | |
| "row_id": row.get("row_id") or "", | |
| "reviewer": row.get("reviewer") or "", | |
| "selection_policy": row.get("selection_policy") or "", | |
| } | |
| ) | |
| return flagged | |
| def audit_application(args: argparse.Namespace) -> dict[str, Any]: | |
| output_dir = Path(args.output_dir) | |
| issues: list[str] = [] | |
| warnings: list[str] = [] | |
| application_audit_path = output_dir / "candidate-line-insertion-audit.json" | |
| application = _read_json_object(application_audit_path, issues, label="candidate-line-insertion-audit.json") | |
| inserted_line_count = 0 | |
| total_inserted_line_count = 0 | |
| pair_rows: list[dict[str, Any]] = [] | |
| review_rows: list[dict[str, Any]] = [] | |
| reference_oracle_rows: list[dict[str, Any]] = [] | |
| if application is not None: | |
| source_book_run = Path(str(application.get("source_book_run") or "")) | |
| _check_file_sha(source_book_run, application.get("source_book_run_sha256"), issues, label="source book-run.json") | |
| review_jsonl = Path(str(application.get("candidate_review_jsonl") or "")) | |
| _check_file_sha(review_jsonl, application.get("candidate_review_jsonl_sha256"), issues, label="candidate review JSONL") | |
| review_rows = _read_review_rows(review_jsonl, issues, label="candidate review file") | |
| reference_oracle_rows = _oracle_or_reference_assisted_rows(review_rows) | |
| if reference_oracle_rows: | |
| warnings.append( | |
| "candidate review JSONL contains oracle/reference-assisted rows; " | |
| "application is diagnostic-only and must not pass operational admission" | |
| ) | |
| _check_file_sha(Path(str(application.get("book_md") or output_dir / "book.md")), application.get("book_md_sha256"), issues, label="book.md") | |
| _check_file_sha( | |
| Path(str(application.get("book_body_md") or output_dir / "book.body.md")), | |
| application.get("book_body_md_sha256"), | |
| issues, | |
| label="book.body.md", | |
| ) | |
| pairs_path = Path(str(application.get("insertion_pairs") or output_dir / "candidate-line-insertion-pairs.jsonl")) | |
| _check_file_sha(pairs_path, application.get("insertion_pairs_sha256"), issues, label="candidate-line-insertion-pairs.jsonl") | |
| pair_rows = _read_jsonl(pairs_path, issues, label="candidate-line-insertion-pairs.jsonl") | |
| pages = application.get("pages") | |
| if not isinstance(pages, list) or not all(isinstance(page, dict) for page in pages): | |
| issues.append("candidate-line-insertion-audit.json pages must be a list of objects") | |
| pages = [] | |
| for index, page in enumerate(pages): | |
| page_id = str(page.get("page_id") or f"page-{index}") | |
| source_document = Path(str(page.get("source_document") or "")) | |
| _check_file_sha(source_document, page.get("source_document_sha256"), issues, label=f"{page_id} source document") | |
| document_json = Path(str(page.get("document_json") or "")) | |
| _check_file_sha(document_json, page.get("document_json_sha256"), issues, label=f"{page_id} document.json") | |
| document_dir = Path(str(page.get("document_dir") or "")) | |
| _check_file_sha(document_dir / "document.md", page.get("document_md_sha256"), issues, label=f"{page_id} document.md") | |
| _check_file_sha( | |
| document_dir / "document.body.md", | |
| page.get("document_body_md_sha256"), | |
| issues, | |
| label=f"{page_id} document.body.md", | |
| ) | |
| score_path = Path(str(page.get("score_document_path") or "")) | |
| _check_file_sha(score_path, page.get("score_document_sha256"), issues, label=f"{page_id} score document") | |
| document_payload = _read_json_object(document_json, issues, label=f"{page_id} document.json") | |
| if document_payload is not None: | |
| document = Document.from_dict(document_payload) | |
| total_page_inserted = _candidate_inserted_lines(document) | |
| page_inserted = _candidate_inserted_lines_from_review(document, review_jsonl) | |
| _validate_current_inserted_lines(page_inserted, issues, page_id=page_id) | |
| total_inserted_line_count += len(total_page_inserted) | |
| inserted_line_count += len(page_inserted) | |
| expected_page_count = int(page.get("applied_count") or 0) | |
| if len(page_inserted) != expected_page_count: | |
| issues.append(f"{page_id} inserted line count mismatch: recorded={expected_page_count} actual={len(page_inserted)}") | |
| applied_count = int(application.get("applied_count") or 0) | |
| if inserted_line_count != applied_count: | |
| issues.append(f"applied_count mismatch: recorded={applied_count} inserted_lines={inserted_line_count}") | |
| if len(pair_rows) != applied_count: | |
| issues.append(f"insertion pair count mismatch: applied_count={applied_count} pairs={len(pair_rows)}") | |
| report = { | |
| "created_at_utc": datetime.now(UTC).isoformat(), | |
| "tool": "tools/audit_candidate_line_review_application.py", | |
| "output_dir": str(output_dir), | |
| "application_audit": str(application_audit_path), | |
| "audit_passed": not issues, | |
| "issues": issues, | |
| "warnings": warnings, | |
| "applied_count": application.get("applied_count") if isinstance(application, dict) else None, | |
| "inserted_line_count": inserted_line_count, | |
| "total_inserted_line_count": total_inserted_line_count, | |
| "inherited_inserted_line_count": total_inserted_line_count - inserted_line_count, | |
| "insertion_pair_count": len(pair_rows), | |
| "review_row_count": len(review_rows), | |
| "reference_oracle_review_count": len(reference_oracle_rows), | |
| "reference_oracle_review_rows": reference_oracle_rows, | |
| "operational_review_safe": not reference_oracle_rows, | |
| "claim_boundary": "artifact consistency audit for candidate-line review application; it does not certify OCR quality", | |
| } | |
| out = Path(args.out) if getattr(args, "out", None) else output_dir / "candidate-line-application-audit.json" | |
| out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("output_dir", type=Path) | |
| parser.add_argument("--out", type=Path) | |
| args = parser.parse_args() | |
| report = audit_application(args) | |
| print(f"candidate-line-application audit {'passed' if report['audit_passed'] else 'failed'} issues={len(report['issues'])}") | |
| raise SystemExit(0 if report["audit_passed"] else 1) | |
| if __name__ == "__main__": | |
| main() | |