ocr-projects-archive / ocr-tech /tools /audit_candidate_line_review_application.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
13.1 kB
#!/usr/bin/env python3
"""Audit a candidate-line review application output."""
from __future__ import annotations
import argparse
import csv
import json
import re
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
from ocrtech.normalization import normalize_ocr_text
from ocrtech.schemas import Document
def _read_json_object(path: Path, issues: list[str], *, label: str) -> dict[str, Any] | None:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError:
issues.append(f"{label} is missing: {path}")
return None
except json.JSONDecodeError as exc:
issues.append(f"{label} is invalid JSON: {path} ({exc})")
return None
if not isinstance(payload, dict):
issues.append(f"{label} must be a JSON object: {path}")
return None
return payload
def _read_jsonl(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]:
if not path.is_file():
issues.append(f"{label} is missing: {path}")
return []
rows: list[dict[str, Any]] = []
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
payload = json.loads(line)
except json.JSONDecodeError as exc:
issues.append(f"{label} invalid JSON at line {line_number}: {exc}")
continue
if not isinstance(payload, dict):
issues.append(f"{label} row {line_number} must be a JSON object")
continue
rows.append(payload)
return rows
def _read_tsv(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]:
if not path.is_file():
issues.append(f"{label} is missing: {path}")
return []
rows: list[dict[str, Any]] = []
with path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle, delimiter="\t")
if reader.fieldnames is None:
issues.append(f"{label} has no header: {path}")
return []
for row_number, row in enumerate(reader, start=2):
payload = {key: value for key, value in row.items() if key is not None}
payload["_row_number"] = row_number
rows.append(payload)
return rows
def _read_review_rows(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]:
if path.suffix.lower() == ".tsv":
return _read_tsv(path, issues, label=label)
return _read_jsonl(path, issues, label=label)
def _check_file_sha(path: Path, expected: object, issues: list[str], *, label: str) -> None:
if not path.is_file():
issues.append(f"{label} is missing: {path}")
return
if not isinstance(expected, str) or not expected:
issues.append(f"{label} missing recorded SHA-256")
return
actual = sha256_file(path)
if actual != expected:
issues.append(f"{label} SHA-256 mismatch: recorded={expected} actual={actual} path={path}")
def _candidate_inserted_lines(document: Document) -> list[Any]:
lines: list[Any] = []
for page in document.pages:
for line in page.text_lines:
metadata = line.metadata if isinstance(line.metadata, dict) else {}
if isinstance(metadata.get("candidate_line_review_insertion"), dict):
lines.append(line)
return lines
def _candidate_inserted_lines_from_review(document: Document, review_path: Path) -> list[Any]:
expected = str(review_path)
lines: list[Any] = []
for line in _candidate_inserted_lines(document):
metadata = line.metadata if isinstance(line.metadata, dict) else {}
insertion = metadata.get("candidate_line_review_insertion")
if not isinstance(insertion, dict):
continue
if str(insertion.get("source_review_jsonl") or "") == expected:
lines.append(line)
return lines
def _validate_current_inserted_lines(lines: list[Any], issues: list[str], *, page_id: str) -> None:
for line in lines:
metadata = line.metadata if isinstance(line.metadata, dict) else {}
insertion = metadata.get("candidate_line_review_insertion")
if not isinstance(insertion, dict):
continue
line_id = str(getattr(line, "line_id", "") or "")
corrected_text = normalize_ocr_text(str(insertion.get("corrected_text") or ""))
inserted_text = normalize_ocr_text(str(insertion.get("inserted_text") or ""))
line_text = normalize_ocr_text(str(getattr(line, "text", "") or ""))
if not corrected_text.strip():
issues.append(f"{page_id} inserted candidate line {line_id!r} missing corrected_text")
if not inserted_text.strip():
issues.append(f"{page_id} inserted candidate line {line_id!r} missing inserted_text")
if inserted_text.strip() and line_text != inserted_text:
issues.append(
f"{page_id} inserted candidate line {line_id!r} text does not match inserted_text: "
f"line={line_text!r} inserted={inserted_text!r}"
)
if corrected_text.strip() and line_text != corrected_text:
issues.append(
f"{page_id} inserted candidate line {line_id!r} text does not match corrected_text: "
f"line={line_text!r} corrected={corrected_text!r}"
)
def _oracle_or_reference_assisted_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
flagged: list[dict[str, Any]] = []
for index, row in enumerate(rows, start=1):
values = [
row.get("selection_policy"),
row.get("claim_boundary"),
row.get("reviewer"),
row.get("notes"),
]
keys = set(row)
text = " ".join(str(value or "") for value in values).lower()
text = re.sub(r"\bnon[-_ ]oracle\b", " ", text)
has_oracle_marker = bool(
re.search(r"\boracle(?:_|-|$|\s)", text)
or "reference_overlap" in text
or any(str(key).startswith("oracle_") for key in keys)
)
if has_oracle_marker:
flagged.append(
{
"row_number": index,
"row_id": row.get("row_id") or "",
"reviewer": row.get("reviewer") or "",
"selection_policy": row.get("selection_policy") or "",
}
)
return flagged
def audit_application(args: argparse.Namespace) -> dict[str, Any]:
output_dir = Path(args.output_dir)
issues: list[str] = []
warnings: list[str] = []
application_audit_path = output_dir / "candidate-line-insertion-audit.json"
application = _read_json_object(application_audit_path, issues, label="candidate-line-insertion-audit.json")
inserted_line_count = 0
total_inserted_line_count = 0
pair_rows: list[dict[str, Any]] = []
review_rows: list[dict[str, Any]] = []
reference_oracle_rows: list[dict[str, Any]] = []
if application is not None:
source_book_run = Path(str(application.get("source_book_run") or ""))
_check_file_sha(source_book_run, application.get("source_book_run_sha256"), issues, label="source book-run.json")
review_jsonl = Path(str(application.get("candidate_review_jsonl") or ""))
_check_file_sha(review_jsonl, application.get("candidate_review_jsonl_sha256"), issues, label="candidate review JSONL")
review_rows = _read_review_rows(review_jsonl, issues, label="candidate review file")
reference_oracle_rows = _oracle_or_reference_assisted_rows(review_rows)
if reference_oracle_rows:
warnings.append(
"candidate review JSONL contains oracle/reference-assisted rows; "
"application is diagnostic-only and must not pass operational admission"
)
_check_file_sha(Path(str(application.get("book_md") or output_dir / "book.md")), application.get("book_md_sha256"), issues, label="book.md")
_check_file_sha(
Path(str(application.get("book_body_md") or output_dir / "book.body.md")),
application.get("book_body_md_sha256"),
issues,
label="book.body.md",
)
pairs_path = Path(str(application.get("insertion_pairs") or output_dir / "candidate-line-insertion-pairs.jsonl"))
_check_file_sha(pairs_path, application.get("insertion_pairs_sha256"), issues, label="candidate-line-insertion-pairs.jsonl")
pair_rows = _read_jsonl(pairs_path, issues, label="candidate-line-insertion-pairs.jsonl")
pages = application.get("pages")
if not isinstance(pages, list) or not all(isinstance(page, dict) for page in pages):
issues.append("candidate-line-insertion-audit.json pages must be a list of objects")
pages = []
for index, page in enumerate(pages):
page_id = str(page.get("page_id") or f"page-{index}")
source_document = Path(str(page.get("source_document") or ""))
_check_file_sha(source_document, page.get("source_document_sha256"), issues, label=f"{page_id} source document")
document_json = Path(str(page.get("document_json") or ""))
_check_file_sha(document_json, page.get("document_json_sha256"), issues, label=f"{page_id} document.json")
document_dir = Path(str(page.get("document_dir") or ""))
_check_file_sha(document_dir / "document.md", page.get("document_md_sha256"), issues, label=f"{page_id} document.md")
_check_file_sha(
document_dir / "document.body.md",
page.get("document_body_md_sha256"),
issues,
label=f"{page_id} document.body.md",
)
score_path = Path(str(page.get("score_document_path") or ""))
_check_file_sha(score_path, page.get("score_document_sha256"), issues, label=f"{page_id} score document")
document_payload = _read_json_object(document_json, issues, label=f"{page_id} document.json")
if document_payload is not None:
document = Document.from_dict(document_payload)
total_page_inserted = _candidate_inserted_lines(document)
page_inserted = _candidate_inserted_lines_from_review(document, review_jsonl)
_validate_current_inserted_lines(page_inserted, issues, page_id=page_id)
total_inserted_line_count += len(total_page_inserted)
inserted_line_count += len(page_inserted)
expected_page_count = int(page.get("applied_count") or 0)
if len(page_inserted) != expected_page_count:
issues.append(f"{page_id} inserted line count mismatch: recorded={expected_page_count} actual={len(page_inserted)}")
applied_count = int(application.get("applied_count") or 0)
if inserted_line_count != applied_count:
issues.append(f"applied_count mismatch: recorded={applied_count} inserted_lines={inserted_line_count}")
if len(pair_rows) != applied_count:
issues.append(f"insertion pair count mismatch: applied_count={applied_count} pairs={len(pair_rows)}")
report = {
"created_at_utc": datetime.now(UTC).isoformat(),
"tool": "tools/audit_candidate_line_review_application.py",
"output_dir": str(output_dir),
"application_audit": str(application_audit_path),
"audit_passed": not issues,
"issues": issues,
"warnings": warnings,
"applied_count": application.get("applied_count") if isinstance(application, dict) else None,
"inserted_line_count": inserted_line_count,
"total_inserted_line_count": total_inserted_line_count,
"inherited_inserted_line_count": total_inserted_line_count - inserted_line_count,
"insertion_pair_count": len(pair_rows),
"review_row_count": len(review_rows),
"reference_oracle_review_count": len(reference_oracle_rows),
"reference_oracle_review_rows": reference_oracle_rows,
"operational_review_safe": not reference_oracle_rows,
"claim_boundary": "artifact consistency audit for candidate-line review application; it does not certify OCR quality",
}
out = Path(args.out) if getattr(args, "out", None) else output_dir / "candidate-line-application-audit.json"
out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return report
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("output_dir", type=Path)
parser.add_argument("--out", type=Path)
args = parser.parse_args()
report = audit_application(args)
print(f"candidate-line-application audit {'passed' if report['audit_passed'] else 'failed'} issues={len(report['issues'])}")
raise SystemExit(0 if report["audit_passed"] else 1)
if __name__ == "__main__":
main()