#!/usr/bin/env python3 """Audit a candidate-line review application output.""" from __future__ import annotations import argparse import csv import json import re from datetime import UTC, datetime from pathlib import Path from typing import Any from ocrtech.manifest import sha256_file from ocrtech.normalization import normalize_ocr_text from ocrtech.schemas import Document def _read_json_object(path: Path, issues: list[str], *, label: str) -> dict[str, Any] | None: try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError: issues.append(f"{label} is missing: {path}") return None except json.JSONDecodeError as exc: issues.append(f"{label} is invalid JSON: {path} ({exc})") return None if not isinstance(payload, dict): issues.append(f"{label} must be a JSON object: {path}") return None return payload def _read_jsonl(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]: if not path.is_file(): issues.append(f"{label} is missing: {path}") return [] rows: list[dict[str, Any]] = [] for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): if not line.strip(): continue try: payload = json.loads(line) except json.JSONDecodeError as exc: issues.append(f"{label} invalid JSON at line {line_number}: {exc}") continue if not isinstance(payload, dict): issues.append(f"{label} row {line_number} must be a JSON object") continue rows.append(payload) return rows def _read_tsv(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]: if not path.is_file(): issues.append(f"{label} is missing: {path}") return [] rows: list[dict[str, Any]] = [] with path.open("r", encoding="utf-8", newline="") as handle: reader = csv.DictReader(handle, delimiter="\t") if reader.fieldnames is None: issues.append(f"{label} has no header: {path}") return [] for row_number, row in enumerate(reader, start=2): payload = {key: value for key, value in row.items() if key is not None} payload["_row_number"] = row_number rows.append(payload) return rows def _read_review_rows(path: Path, issues: list[str], *, label: str) -> list[dict[str, Any]]: if path.suffix.lower() == ".tsv": return _read_tsv(path, issues, label=label) return _read_jsonl(path, issues, label=label) def _check_file_sha(path: Path, expected: object, issues: list[str], *, label: str) -> None: if not path.is_file(): issues.append(f"{label} is missing: {path}") return if not isinstance(expected, str) or not expected: issues.append(f"{label} missing recorded SHA-256") return actual = sha256_file(path) if actual != expected: issues.append(f"{label} SHA-256 mismatch: recorded={expected} actual={actual} path={path}") def _candidate_inserted_lines(document: Document) -> list[Any]: lines: list[Any] = [] for page in document.pages: for line in page.text_lines: metadata = line.metadata if isinstance(line.metadata, dict) else {} if isinstance(metadata.get("candidate_line_review_insertion"), dict): lines.append(line) return lines def _candidate_inserted_lines_from_review(document: Document, review_path: Path) -> list[Any]: expected = str(review_path) lines: list[Any] = [] for line in _candidate_inserted_lines(document): metadata = line.metadata if isinstance(line.metadata, dict) else {} insertion = metadata.get("candidate_line_review_insertion") if not isinstance(insertion, dict): continue if str(insertion.get("source_review_jsonl") or "") == expected: lines.append(line) return lines def _validate_current_inserted_lines(lines: list[Any], issues: list[str], *, page_id: str) -> None: for line in lines: metadata = line.metadata if isinstance(line.metadata, dict) else {} insertion = metadata.get("candidate_line_review_insertion") if not isinstance(insertion, dict): continue line_id = str(getattr(line, "line_id", "") or "") corrected_text = normalize_ocr_text(str(insertion.get("corrected_text") or "")) inserted_text = normalize_ocr_text(str(insertion.get("inserted_text") or "")) line_text = normalize_ocr_text(str(getattr(line, "text", "") or "")) if not corrected_text.strip(): issues.append(f"{page_id} inserted candidate line {line_id!r} missing corrected_text") if not inserted_text.strip(): issues.append(f"{page_id} inserted candidate line {line_id!r} missing inserted_text") if inserted_text.strip() and line_text != inserted_text: issues.append( f"{page_id} inserted candidate line {line_id!r} text does not match inserted_text: " f"line={line_text!r} inserted={inserted_text!r}" ) if corrected_text.strip() and line_text != corrected_text: issues.append( f"{page_id} inserted candidate line {line_id!r} text does not match corrected_text: " f"line={line_text!r} corrected={corrected_text!r}" ) def _oracle_or_reference_assisted_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: flagged: list[dict[str, Any]] = [] for index, row in enumerate(rows, start=1): values = [ row.get("selection_policy"), row.get("claim_boundary"), row.get("reviewer"), row.get("notes"), ] keys = set(row) text = " ".join(str(value or "") for value in values).lower() text = re.sub(r"\bnon[-_ ]oracle\b", " ", text) has_oracle_marker = bool( re.search(r"\boracle(?:_|-|$|\s)", text) or "reference_overlap" in text or any(str(key).startswith("oracle_") for key in keys) ) if has_oracle_marker: flagged.append( { "row_number": index, "row_id": row.get("row_id") or "", "reviewer": row.get("reviewer") or "", "selection_policy": row.get("selection_policy") or "", } ) return flagged def audit_application(args: argparse.Namespace) -> dict[str, Any]: output_dir = Path(args.output_dir) issues: list[str] = [] warnings: list[str] = [] application_audit_path = output_dir / "candidate-line-insertion-audit.json" application = _read_json_object(application_audit_path, issues, label="candidate-line-insertion-audit.json") inserted_line_count = 0 total_inserted_line_count = 0 pair_rows: list[dict[str, Any]] = [] review_rows: list[dict[str, Any]] = [] reference_oracle_rows: list[dict[str, Any]] = [] if application is not None: source_book_run = Path(str(application.get("source_book_run") or "")) _check_file_sha(source_book_run, application.get("source_book_run_sha256"), issues, label="source book-run.json") review_jsonl = Path(str(application.get("candidate_review_jsonl") or "")) _check_file_sha(review_jsonl, application.get("candidate_review_jsonl_sha256"), issues, label="candidate review JSONL") review_rows = _read_review_rows(review_jsonl, issues, label="candidate review file") reference_oracle_rows = _oracle_or_reference_assisted_rows(review_rows) if reference_oracle_rows: warnings.append( "candidate review JSONL contains oracle/reference-assisted rows; " "application is diagnostic-only and must not pass operational admission" ) _check_file_sha(Path(str(application.get("book_md") or output_dir / "book.md")), application.get("book_md_sha256"), issues, label="book.md") _check_file_sha( Path(str(application.get("book_body_md") or output_dir / "book.body.md")), application.get("book_body_md_sha256"), issues, label="book.body.md", ) pairs_path = Path(str(application.get("insertion_pairs") or output_dir / "candidate-line-insertion-pairs.jsonl")) _check_file_sha(pairs_path, application.get("insertion_pairs_sha256"), issues, label="candidate-line-insertion-pairs.jsonl") pair_rows = _read_jsonl(pairs_path, issues, label="candidate-line-insertion-pairs.jsonl") pages = application.get("pages") if not isinstance(pages, list) or not all(isinstance(page, dict) for page in pages): issues.append("candidate-line-insertion-audit.json pages must be a list of objects") pages = [] for index, page in enumerate(pages): page_id = str(page.get("page_id") or f"page-{index}") source_document = Path(str(page.get("source_document") or "")) _check_file_sha(source_document, page.get("source_document_sha256"), issues, label=f"{page_id} source document") document_json = Path(str(page.get("document_json") or "")) _check_file_sha(document_json, page.get("document_json_sha256"), issues, label=f"{page_id} document.json") document_dir = Path(str(page.get("document_dir") or "")) _check_file_sha(document_dir / "document.md", page.get("document_md_sha256"), issues, label=f"{page_id} document.md") _check_file_sha( document_dir / "document.body.md", page.get("document_body_md_sha256"), issues, label=f"{page_id} document.body.md", ) score_path = Path(str(page.get("score_document_path") or "")) _check_file_sha(score_path, page.get("score_document_sha256"), issues, label=f"{page_id} score document") document_payload = _read_json_object(document_json, issues, label=f"{page_id} document.json") if document_payload is not None: document = Document.from_dict(document_payload) total_page_inserted = _candidate_inserted_lines(document) page_inserted = _candidate_inserted_lines_from_review(document, review_jsonl) _validate_current_inserted_lines(page_inserted, issues, page_id=page_id) total_inserted_line_count += len(total_page_inserted) inserted_line_count += len(page_inserted) expected_page_count = int(page.get("applied_count") or 0) if len(page_inserted) != expected_page_count: issues.append(f"{page_id} inserted line count mismatch: recorded={expected_page_count} actual={len(page_inserted)}") applied_count = int(application.get("applied_count") or 0) if inserted_line_count != applied_count: issues.append(f"applied_count mismatch: recorded={applied_count} inserted_lines={inserted_line_count}") if len(pair_rows) != applied_count: issues.append(f"insertion pair count mismatch: applied_count={applied_count} pairs={len(pair_rows)}") report = { "created_at_utc": datetime.now(UTC).isoformat(), "tool": "tools/audit_candidate_line_review_application.py", "output_dir": str(output_dir), "application_audit": str(application_audit_path), "audit_passed": not issues, "issues": issues, "warnings": warnings, "applied_count": application.get("applied_count") if isinstance(application, dict) else None, "inserted_line_count": inserted_line_count, "total_inserted_line_count": total_inserted_line_count, "inherited_inserted_line_count": total_inserted_line_count - inserted_line_count, "insertion_pair_count": len(pair_rows), "review_row_count": len(review_rows), "reference_oracle_review_count": len(reference_oracle_rows), "reference_oracle_review_rows": reference_oracle_rows, "operational_review_safe": not reference_oracle_rows, "claim_boundary": "artifact consistency audit for candidate-line review application; it does not certify OCR quality", } out = Path(args.out) if getattr(args, "out", None) else output_dir / "candidate-line-application-audit.json" out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return report def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("output_dir", type=Path) parser.add_argument("--out", type=Path) args = parser.parse_args() report = audit_application(args) print(f"candidate-line-application audit {'passed' if report['audit_passed'] else 'failed'} issues={len(report['issues'])}") raise SystemExit(0 if report["audit_passed"] else 1) if __name__ == "__main__": main()