ocr-projects-archive / ocr-tech /tools /audit_eval_run.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
288 kB
#!/usr/bin/env python3 """Audit a completed OCR eval output directory for review readiness.""" from __future__ import annotations import argparse import csv import hashlib import html import importlib.util import json import random import shlex import sys import unicodedata from datetime import datetime from pathlib import Path from typing import Any import regex from rapidfuzz.distance import Levenshtein from ocrtech.metrics import bbox_iou from ocrtech.references import load_reference from ocrtech.schemas import BBox, Document, TextLine TOOLS_DIR = Path(__file__).resolve().parent SUPPORTED_NORMALIZATION_PROFILE = "NFC_preserve_whitespace" UNCERTAINTY_BOOTSTRAP_SAMPLES = 200 UNCERTAINTY_BOOTSTRAP_SEED = 1729 MIN_CLAIM_SCORING_UNITS = 30 RECOGNIZER_ARTIFACTS = ( "eval-run.json", "metrics.json", "metrics.md", "predictions.tsv", "unmatched.tsv", "worst-lines.csv", "sample-review.csv", "sample-review.html", "confusions.json", "command.txt", "claim-safety.md", ) PAGE_ARTIFACTS = ( "page-eval-run.json", "metrics.json", "metrics.md", "page-metrics.csv", "reference-artifacts.tsv", "text-pairs.tsv", "missing.tsv", "worst-pages.csv", "confusions.json", "sample-review.csv", "sample-review.html", "line-alignment.csv", "command.txt", "claim-safety.md", ) RECIPE_ROOT_ARTIFACTS = ( "recipe-run.json", "recipe.json", "recipe-validation.json", "audit.json", "claim-safety.md", "materialize/output-command-run.json", "materialize/output-command-run.md", "materialize/command-template.txt", ) RECOGNIZER_REQUIRED_RUN_FIELDS = ( "created_at", "model_name", "checkpoint", "config", "dictionary", "artifact_identity", "label_file", "label_file_sha256", "prediction_file", "prediction_file_sha256", "data_dir", "script_inventory_file", "script_inventory_file_sha256", "eval_pack_version", "pack_content_sha256", "frozen_eval_pack", "unseen_for_selection", "selected_checkpoint_on_test", "normalization_profile", "eval_mode", "languages", "scripts", "domains", "gt_statuses", "splits", "script_inventory", "claim_safe", "warnings", "machine", "command", ) PAGE_REQUIRED_RUN_FIELDS = ( "created_at", "system_name", "model_name", "checkpoint", "config", "dictionary", "artifact_identity", "eval_manifest", "eval_manifest_sha256", "train_manifests", "documents_dir", "ocr_output_kind", "require_output_audit", "output_audit_filename", "output_audit_passed_field", "page_output_audit", "script_inventory_file", "script_inventory_file_sha256", "eval_pack_version", "pack_content_sha256", "frozen_eval_pack", "unseen_for_selection", "selected_checkpoint_on_test", "line_iou_threshold", "language_default", "script_default", "domain_default", "gt_status_default", "normalization_profile", "languages", "scripts", "domains", "gt_statuses", "splits", "eval_mode", "script_inventory", "claim_safe", "warnings", "machine", "command", ) RECIPE_REQUIRED_FIELDS = ( "created_at", "recipe_id", "recipe_file", "recipe_file_sha256", "root", "outputs", "train_manifests", "recipe_validation", "materialization", "audit", "score", "machine", "command", ) CLAIM_SAFE_DOMAINS = {"real_print", "real_handwritten"} CLAIM_SAFE_GT_STATUSES = {"human_verified", "machine_converted_spot_checked"} CLAIM_EVAL_SPLITS = {"eval", "test", "heldout", "held_out", "validation"} SOURCE_DOCUMENT_KEYS = ("source_document", "source_doc", "source_pdf", "source_image", "source_xml", "document", "doc_id") PAGE_KEYS = ("page", "page_index", "page_number", "source_page", "page_id") BBOX_KEYS = ("bbox", "bounding_box", "coords", "polygon") CLAIM_ELIGIBLE_KEYS = ("claim_eligible", "claim_evidence_eligible", "paper_claim_eligible") HUMAN_REVIEW_EVIDENCE_KEYS = ( "reviewed_by", "reviewer", "native_reviewer", "gt_reviewer", "review_artifact", "gt_review_artifact", "review_batch", "double_reviewed_by", "inter_annotator_agreement", "correction_rate", ) PROVENANCE_SIGNAL_KEYS = ( "label_provenance", "gt_source", "source_gt_status", "source_label_status", "reference_status", "assisted_source", "source_type", "data_origin", "evidence_type", "dataset_type", "slices", ) def _read_json(path: Path) -> Any: if not path.exists(): raise FileNotFoundError(f"JSON artifact does not exist: {path}") try: return json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise ValueError(f"invalid JSON artifact: {path}: {exc}") from exc def _load_tool(name: str) -> Any: path = TOOLS_DIR / f"{name}.py" spec = importlib.util.spec_from_file_location(name, path) if spec is None or spec.loader is None: raise RuntimeError(f"could not load tool module: {path}") module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _score_dir(run_dir: Path) -> Path: if (run_dir / "eval-run.json").exists() or (run_dir / "page-eval-run.json").exists(): return run_dir if (run_dir / "score").is_dir(): return run_dir / "score" raise FileNotFoundError(f"could not find eval-run.json, page-eval-run.json, or score/ in {run_dir}") def _mode(score_dir: Path) -> str: if (score_dir / "eval-run.json").exists(): return "recognizer_oracle_crop" if (score_dir / "page-eval-run.json").exists(): return "page_level_ocr" raise FileNotFoundError(f"score directory has no eval run JSON: {score_dir}") def _is_number(value: Any) -> bool: return isinstance(value, int | float) and not isinstance(value, bool) def _close(left: float, right: float, epsilon: float = 1e-12) -> bool: return abs(left - right) <= epsilon def _percentile(values: list[float], q: float) -> float: if not values: return 0.0 ordered = sorted(values) if len(ordered) == 1: return ordered[0] position = (len(ordered) - 1) * q lower = int(position) upper = min(lower + 1, len(ordered) - 1) weight = position - lower return ordered[lower] * (1.0 - weight) + ordered[upper] * weight def _confidence_interval(values: list[float]) -> dict[str, float]: return { "low": _percentile(values, 0.025), "high": _percentile(values, 0.975), } def _expected_uncertainty( units: list[dict[str, int]], *, method: str, metrics: tuple[str, ...], ) -> dict[str, Any]: n = len(units) samples: dict[str, list[float]] = {metric: [] for metric in metrics} if n: rng = random.Random(UNCERTAINTY_BOOTSTRAP_SEED) for _ in range(UNCERTAINTY_BOOTSTRAP_SAMPLES): totals = { "codepoint_edits": 0, "codepoint_ref_len": 0, "grapheme_edits": 0, "grapheme_ref_len": 0, "exact": 0, } for _ in range(n): unit = units[rng.randrange(n)] for key in totals: totals[key] += unit.get(key, 0) if "cer_codepoint_micro" in samples: samples["cer_codepoint_micro"].append(totals["codepoint_edits"] / max(totals["codepoint_ref_len"], 1)) if "cer_grapheme_micro" in samples: samples["cer_grapheme_micro"].append(totals["grapheme_edits"] / max(totals["grapheme_ref_len"], 1)) if "exact_line_accuracy" in samples: samples["exact_line_accuracy"].append(totals["exact"] / max(n, 1)) if "text_cer_codepoint_micro" in samples: samples["text_cer_codepoint_micro"].append(totals["codepoint_edits"] / max(totals["codepoint_ref_len"], 1)) if "text_cer_grapheme_micro" in samples: samples["text_cer_grapheme_micro"].append(totals["grapheme_edits"] / max(totals["grapheme_ref_len"], 1)) if "text_exact_accuracy" in samples: samples["text_exact_accuracy"].append(totals["exact"] / max(totals.get("text_reference", n), 1)) return { "method": method, "confidence_level": 0.95, "bootstrap_samples": UNCERTAINTY_BOOTSTRAP_SAMPLES, "seed": UNCERTAINTY_BOOTSTRAP_SEED, "scoring_units": n, "effective_sample_size": n, "min_claim_scoring_units": MIN_CLAIM_SCORING_UNITS, "claim_sample_size_ok": n >= MIN_CLAIM_SCORING_UNITS, "metrics": {metric: _confidence_interval(samples[metric]) for metric in metrics}, } def _check_uncertainty(recorded: Any, expected: dict[str, Any], label: str) -> list[str]: issues: list[str] = [] if not isinstance(recorded, dict): return [f"{label} uncertainty is missing or not an object"] for key in ( "method", "confidence_level", "bootstrap_samples", "seed", "scoring_units", "effective_sample_size", "min_claim_scoring_units", "claim_sample_size_ok", ): if recorded.get(key) != expected[key]: issues.append(f"{label} uncertainty {key} mismatch: recorded={recorded.get(key)} recomputed={expected[key]}") recorded_metrics = recorded.get("metrics") expected_metrics = expected["metrics"] if not isinstance(recorded_metrics, dict): return issues + [f"{label} uncertainty metrics missing or not an object"] missing = sorted(set(expected_metrics) - set(recorded_metrics)) extra = sorted(set(recorded_metrics) - set(expected_metrics)) if missing: issues.append(f"{label} uncertainty metrics missing fields: {missing}") if extra: issues.append(f"{label} uncertainty metrics contain extra fields: {extra}") for metric, expected_interval in expected_metrics.items(): recorded_interval = recorded_metrics.get(metric) if not isinstance(recorded_interval, dict): issues.append(f"{label} uncertainty {metric} interval missing or not an object") continue for bound in ("low", "high"): value = recorded_interval.get(bound) expected_value = expected_interval[bound] if not _is_number(value) or not _close(float(value), expected_value): issues.append(f"{label} uncertainty {metric}.{bound} mismatch: recorded={value} recomputed={expected_value}") return issues def _artifact_rows(path: Path) -> int: if not path.exists(): return 0 with path.open("r", encoding="utf-8", newline="") as handle: return max(sum(1 for _ in handle) - 1, 0) def _csv_rows(path: Path) -> list[dict[str, str]]: if not path.exists(): return [] with path.open("r", encoding="utf-8", newline="") as handle: return list(csv.DictReader(handle)) def _tsv_rows(path: Path) -> list[dict[str, str]]: if not path.exists(): return [] with path.open("r", encoding="utf-8", newline="") as handle: return list(csv.DictReader(handle, delimiter="\t")) def _short_rows(rows: list[Any], limit: int = 10) -> list[Any | str]: visible: list[Any | str] = list(rows[:limit]) if len(rows) > limit: visible.append(f"... {len(rows) - limit} more") return visible def _duplicate_values(rows: list[dict[str, str]], field: str) -> list[str]: seen: set[str] = set() duplicates: set[str] = set() for row in rows: value = row.get(field) or "" if not value: continue if value in seen: duplicates.add(value) seen.add(value) return sorted(duplicates) def _duplicate_keys(rows: list[dict[str, str]], fields: tuple[str, ...]) -> list[tuple[str, ...]]: seen: set[tuple[str, ...]] = set() duplicates: set[tuple[str, ...]] = set() for row in rows: key = tuple(row.get(field) or "" for field in fields) if not any(key): continue if key in seen: duplicates.add(key) seen.add(key) return sorted(duplicates) def _check_unique_field(label: str, rows: list[dict[str, str]], field: str) -> list[str]: duplicates = _duplicate_values(rows, field) if duplicates: return [f"{label} duplicate {field} values: {duplicates[:10]}"] return [] def _check_unique_key(label: str, rows: list[dict[str, str]], fields: tuple[str, ...]) -> list[str]: duplicates = _duplicate_keys(rows, fields) if duplicates: return [f"{label} duplicate {','.join(fields)} values: {duplicates[:10]}"] return [] def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _check_hash_field(eval_run: dict[str, Any], path_field: str, hash_field: str, label: str) -> list[str]: issues: list[str] = [] path_value = eval_run.get(path_field) expected = eval_run.get(hash_field) if path_value in (None, "") and expected in (None, ""): return issues if not isinstance(path_value, str) or not path_value: issues.append(f"{label} hash is recorded but path is missing: {hash_field}") return issues if not isinstance(expected, str) or not expected: issues.append(f"{label} path is recorded but hash is missing: {hash_field}") return issues path = Path(path_value) if not path.exists(): issues.append(f"{label} path does not exist for hash verification: {path_value}") return issues actual = _sha256_file(path) if actual != expected: issues.append(f"{label} sha256 mismatch for {path_field}: recorded={expected} actual={actual}") return issues def _read_summary_for_audit(eval_run: dict[str, Any], label: str, purpose: str) -> tuple[dict[str, Any], list[str]]: summary_file = eval_run.get("summary_file") if summary_file in (None, ""): return {}, [] if not isinstance(summary_file, str): return {}, [f"{label} summary_file is not a string"] path = Path(summary_file) if not path.exists(): return {}, [f"{label} summary file does not exist for {purpose}: {summary_file}"] try: payload = _read_json(path) except (OSError, ValueError) as exc: return {}, [f"{label} summary file could not be parsed for {purpose}: {exc}"] if not isinstance(payload, dict): return {}, [f"{label} summary file is not a JSON object for {purpose}: {summary_file}"] return payload, [] def _check_summary_metadata_consistency(eval_run: dict[str, Any], label: str) -> list[str]: summary, issues = _read_summary_for_audit(eval_run, label, "metadata audit") if issues or not summary: return issues list_fields = ( ("language", "languages"), ("script", "scripts"), ("domain", "domains"), ("gt_status", "gt_statuses"), ("split", "splits"), ) for summary_field, run_field in list_fields: summary_value = summary.get(summary_field) if summary_value in (None, ""): continue run_values = eval_run.get(run_field) if not isinstance(run_values, list): continue run_value_strings = {str(value) for value in run_values} if str(summary_value) not in run_value_strings: issues.append(f"{label} summary {summary_field} mismatch: summary={summary_value!r} run={run_values!r}") summary_version = summary.get("eval_pack_version") run_version = eval_run.get("eval_pack_version") if summary_version not in (None, "") and run_version not in (None, "") and str(summary_version) != str(run_version): issues.append(f"{label} summary eval_pack_version mismatch: summary={summary_version!r} run={run_version!r}") for field in ("frozen_eval_pack", "unseen_for_selection", "claim_eligible_default"): if field not in summary: continue summary_value = summary.get(field) run_value = eval_run.get(field) if not isinstance(summary_value, bool): issues.append(f"{label} summary {field} is not boolean: {summary_value!r}") continue if isinstance(run_value, bool) and summary_value != run_value: issues.append(f"{label} summary {field} mismatch: summary={summary_value!r} run={run_value!r}") summary_eval_pack = summary.get("eval_pack") if isinstance(summary_eval_pack, dict): summary_task = summary_eval_pack.get("task") run_task = eval_run.get("eval_mode") if summary_task not in (None, "") and run_task not in (None, "") and str(summary_task) != str(run_task): issues.append(f"{label} summary task mismatch: summary={summary_task!r} run={run_task!r}") return issues def _check_train_manifest_hashes(eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] manifests = eval_run.get("train_manifests") if manifests in (None, []): return issues if not isinstance(manifests, list): return ["train_manifests is not a list"] for index, manifest in enumerate(manifests): if not isinstance(manifest, dict): issues.append(f"train_manifests[{index}] is not an object") continue path_value = manifest.get("path") expected = manifest.get("sha256") if not isinstance(path_value, str) or not path_value: issues.append(f"train_manifests[{index}] missing path") continue if not isinstance(expected, str) or not expected: issues.append(f"train_manifests[{index}] missing sha256") continue path = Path(path_value) if not path.exists(): issues.append(f"train_manifests[{index}] path does not exist: {path_value}") continue actual = _sha256_file(path) if actual != expected: issues.append(f"train_manifests[{index}] sha256 mismatch: recorded={expected} actual={actual}") return issues def _audit_hash_text(value: str) -> str: return hashlib.sha256(_nfc(value).encode("utf-8")).hexdigest() def _audit_sample_hash(image_sha256: str, text_sha256: str) -> str: return hashlib.sha256(f"{image_sha256}\0{text_sha256}".encode("utf-8")).hexdigest() def _train_manifest_paths(eval_run: dict[str, Any]) -> tuple[list[Path], list[str]]: manifests = eval_run.get("train_manifests") if manifests in (None, []): return [], [] if not isinstance(manifests, list): return [], ["train_manifests is not a list for leakage recomputation"] paths: list[Path] = [] issues: list[str] = [] for index, manifest in enumerate(manifests): if not isinstance(manifest, dict): issues.append(f"train_manifests[{index}] is not an object for leakage recomputation") continue path_value = manifest.get("path") if not isinstance(path_value, str) or not path_value: issues.append(f"train_manifests[{index}] missing path for leakage recomputation") continue paths.append(Path(path_value)) return paths, issues def _audit_manifest_leakage_rows(path: Path, *, manifest_dir: Path | None = None, compute_missing_image_sha: bool) -> tuple[list[dict[str, str]], list[str]]: rows: list[dict[str, str]] = [] issues: list[str] = [] if not path.exists(): return rows, [f"leakage manifest does not exist: {path}"] base_dir = manifest_dir or path.parent with path.open("r", encoding="utf-8") as handle: for line_no, line in enumerate(handle, 1): line = line.strip() if not line: continue try: payload = json.loads(line) except json.JSONDecodeError as exc: issues.append(f"leakage manifest {path}:{line_no} invalid JSON: {exc}") continue if not isinstance(payload, dict): issues.append(f"leakage manifest {path}:{line_no} row is not an object") continue metadata = dict(payload.get("metadata") or {}) sample_id = str(payload.get("sample_id") or payload.get("id") or metadata.get("sample_id") or "") image_path = str(payload.get("image_path") or payload.get("image") or metadata.get("image_path") or "") image_sha256 = str(payload.get("sha256") or payload.get("image_sha256") or metadata.get("sha256") or "") if compute_missing_image_sha and not image_sha256 and image_path: resolved = _resolve_audit_manifest_path(image_path, base_dir) if resolved.exists(): image_sha256 = _sha256_file(resolved) text_value = payload.get("text") if payload.get("text") is not None else metadata.get("text") text = str(text_value or "") text_sha256 = str(payload.get("text_sha256") or metadata.get("text_sha256") or (_audit_hash_text(text) if text else "")) sample_sha256 = str(payload.get("sample_sha256") or metadata.get("sample_sha256") or "") if not sample_sha256 and image_sha256 and text_sha256: sample_sha256 = _audit_sample_hash(image_sha256, text_sha256) source_sample_id = str(payload.get("source_sample_id") or metadata.get("source_sample_id") or sample_id) rows.append( { "sample_id": sample_id, "source_sample_id": source_sample_id, "image_path": image_path, "image_sha256": image_sha256, "text_sha256": text_sha256, "sample_sha256": sample_sha256, } ) return rows, issues def _leakage_overlap(left: list[dict[str, str]], right: list[dict[str, str]], field: str) -> list[str]: left_values = {row.get(field) or "" for row in left if row.get(field)} right_values = {row.get(field) or "" for row in right if row.get(field)} return sorted(left_values & right_values) def _leakage_overlaps(train_rows: list[dict[str, str]], eval_rows: list[dict[str, str]]) -> dict[str, list[str]]: return { "sample_id": _leakage_overlap(train_rows, eval_rows, "sample_id"), "source_sample_id": _leakage_overlap(train_rows, eval_rows, "source_sample_id"), "image_sha256": _leakage_overlap(train_rows, eval_rows, "image_sha256"), "sample_sha256": _leakage_overlap(train_rows, eval_rows, "sample_sha256"), "image_path": _leakage_overlap(train_rows, eval_rows, "image_path"), "text_sha256": _leakage_overlap(train_rows, eval_rows, "text_sha256"), } def _compare_leakage_report(label: str, recorded: Any, expected: dict[str, Any]) -> list[str]: if not isinstance(recorded, dict): return [f"{label} leakage report is missing or not an object"] issues: list[str] = [] for key in ("train_manifests", "checked", "train_rows", "eval_rows"): if recorded.get(key) != expected[key]: issues.append(f"{label} leakage {key} mismatch: recorded={recorded.get(key)} recomputed={expected[key]}") recorded_overlaps = recorded.get("overlaps") if not isinstance(recorded_overlaps, dict): issues.append(f"{label} leakage overlaps missing or not an object") return issues for key, expected_values in expected["overlaps"].items(): recorded_values = recorded_overlaps.get(key) if recorded_values != expected_values: issues.append(f"{label} leakage overlaps.{key} mismatch: recorded={recorded_values} recomputed={expected_values}") return issues def _spotcheck_calibration_summary(path: Path) -> tuple[dict[str, Any] | None, list[str]]: if not path.exists(): return None, [f"spotcheck calibration file does not exist for summary audit: {path}"] try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return None, [f"spotcheck calibration file is invalid JSON for summary audit: {path}: {exc}"] if not isinstance(payload, dict): return None, [f"spotcheck calibration file must contain an object for summary audit: {path}"] return ( { "claim_evidence_eligible": payload.get("claim_evidence_eligible"), "recommended_pack_gt_status": payload.get("recommended_pack_gt_status"), "language": payload.get("language"), "script": payload.get("script"), "domain": payload.get("domain"), "source_gt_status": payload.get("source_gt_status"), "counts": payload.get("counts", {}), "machine_label_metrics": payload.get("machine_label_metrics", {}), "eligibility_reasons": payload.get("eligibility_reasons", []), }, [], ) def _check_spotcheck_calibration_summary(eval_run: dict[str, Any], metrics: dict[str, Any], label: str) -> list[str]: path_value = eval_run.get("spotcheck_calibration_file") if path_value in (None, ""): if eval_run.get("spotcheck_calibration") not in (None, {}): return [f"{label} spotcheck_calibration is recorded without spotcheck_calibration_file"] if metrics.get("spotcheck_calibration") not in (None, {}): return [f"{label} metrics spotcheck_calibration is recorded without spotcheck_calibration_file"] return [] if not isinstance(path_value, str): return [f"{label} spotcheck_calibration_file is not a string"] expected, issues = _spotcheck_calibration_summary(Path(path_value)) if issues: return issues assert expected is not None recorded_run = eval_run.get("spotcheck_calibration") recorded_metrics = metrics.get("spotcheck_calibration") if recorded_run != expected: issues.append(f"{label} eval-run spotcheck_calibration does not match calibration file: recorded={recorded_run} recomputed={expected}") if recorded_metrics != expected: issues.append(f"{label} metrics spotcheck_calibration does not match calibration file: recorded={recorded_metrics} recomputed={expected}") return issues def _check_artifact_identity(eval_run: dict[str, Any], required: tuple[str, ...], label: str) -> list[str]: issues: list[str] = [] artifacts = eval_run.get("artifact_identity") if not isinstance(artifacts, dict): return [f"{label} artifact_identity is missing or not an object"] for name in required: identity = artifacts.get(name) if not isinstance(identity, dict): issues.append(f"{label} artifact_identity.{name} is missing or not an object") continue if not identity.get("verified"): issues.append(f"{label} artifact_identity.{name} is not hash-verifiable: {identity.get('reason') or 'unverified'}") continue recorded_value = eval_run.get(name) identity_value = identity.get("value") if not isinstance(recorded_value, str) or not recorded_value: issues.append(f"{label} {name} field is missing for artifact_identity consistency") elif identity_value != recorded_value: issues.append(f"{label} artifact_identity.{name}.value does not match eval-run {name}: recorded={identity_value} eval_run={recorded_value}") path_value = identity.get("path") or identity.get("value") expected = identity.get("sha256") if not isinstance(path_value, str) or not path_value: issues.append(f"{label} artifact_identity.{name} missing path") continue if not isinstance(expected, str) or not expected: issues.append(f"{label} artifact_identity.{name} missing sha256") continue path = Path(path_value) if not path.exists(): issues.append(f"{label} artifact_identity.{name} path does not exist: {path_value}") continue actual = _sha256_file(path) if actual != expected: issues.append(f"{label} artifact_identity.{name} sha256 mismatch: recorded={expected} actual={actual}") return issues def _check_required_artifacts(score_dir: Path, names: tuple[str, ...]) -> list[str]: issues: list[str] = [] for name in names: path = score_dir / name if not path.exists(): issues.append(f"missing required artifact: {name}") elif path.is_file() and path.stat().st_size == 0: issues.append(f"empty required artifact: {name}") return issues def _fmt_report_metric(value: Any) -> str: if _is_number(value): return f"{float(value):.6f}" return str(value) def _expect_report_line(text: str, issues: list[str], line: str) -> None: if line not in text.splitlines(): issues.append(f"claim-safety.md missing or stale line: {line}") def _check_claim_safety_report_consistency(score_dir: Path, eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: issues: list[str] = [] path = score_dir / "claim-safety.md" if not path.exists(): return issues text = path.read_text(encoding="utf-8") machine = eval_run.get("machine") if isinstance(eval_run.get("machine"), dict) else {} expected_lines = [ f"- eval mode: {eval_run.get('eval_mode')}", f"- claim safe: {eval_run.get('claim_safe')}", f"- created at: {eval_run.get('created_at')}", f"- machine platform: {machine.get('platform')}", f"- machine python: {machine.get('python')}", f"- command: {eval_run.get('command')}", f"- eval pack version: {eval_run.get('eval_pack_version') or '(missing)'}", f"- frozen eval pack: {eval_run.get('frozen_eval_pack')}", f"- unseen for selection: {eval_run.get('unseen_for_selection')}", f"- selected checkpoint on test: {eval_run.get('selected_checkpoint_on_test')}", f"- languages: {eval_run.get('languages')}", f"- scripts: {eval_run.get('scripts')}", f"- domains: {eval_run.get('domains')}", f"- GT statuses: {eval_run.get('gt_statuses')}", f"- splits: {eval_run.get('splits')}", f"- metadata contradictions: {metrics.get('metadata_contradictions', [])}", ] mode = eval_run.get("eval_mode") if mode == "recognizer_oracle_crop": uncertainty = metrics.get("uncertainty") if isinstance(metrics.get("uncertainty"), dict) else {} intervals = uncertainty.get("metrics") if isinstance(uncertainty.get("metrics"), dict) else {} graph_interval = intervals.get("cer_grapheme_micro") if isinstance(intervals.get("cer_grapheme_micro"), dict) else {} inventory = metrics.get("script_inventory") if isinstance(metrics.get("script_inventory"), dict) else {} normalization = metrics.get("normalization_summary") if isinstance(metrics.get("normalization_summary"), dict) else {} dictionary_coverage = metrics.get("dictionary_coverage") if isinstance(metrics.get("dictionary_coverage"), dict) else {} expected_lines.extend( [ f"- model: {eval_run.get('model_name')}", f"- checkpoint: {eval_run.get('checkpoint') or '(not recorded)'}", f"- config: {eval_run.get('config') or '(not recorded)'}", f"- dictionary: {eval_run.get('dictionary') or '(not recorded)'}", f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}", f"- label file: {eval_run.get('label_file')}", f"- prediction file: {eval_run.get('prediction_file')}", f"- scored lines: {metrics.get('n_lines')}", f"- unmatched labels: {metrics.get('unmatched_labels')}", f"- unmatched predictions: {metrics.get('unmatched_predictions')}", f"- script inventory file: {inventory.get('inventory_file')}", f"- observed codepoints/graphemes: {inventory.get('observed_codepoint_count')}/{inventory.get('observed_grapheme_count')}", f"- missing required codepoints: {inventory.get('missing_required_codepoints')}", f"- missing required graphemes: {inventory.get('missing_required_graphemes')}", f"- normalization profile: {normalization.get('profile')}", f"- normalization changed GT/prediction/any: {normalization.get('gt_changed')}/{normalization.get('prediction_changed')}/{normalization.get('any_changed')} of {normalization.get('total')}", f"- dictionary coverage checked: {dictionary_coverage.get('checked')}", f"- dictionary GT codepoints/dictionary codepoints: {dictionary_coverage.get('gt_codepoint_count')}/{dictionary_coverage.get('dictionary_codepoint_count')}", f"- dictionary missing GT codepoints: {dictionary_coverage.get('missing_gt_codepoints')}", f"- codepoint CER micro: {_fmt_report_metric(metrics.get('cer_codepoint_micro'))}", f"- grapheme CER micro: {_fmt_report_metric(metrics.get('cer_grapheme_micro'))}", f"- exact-line accuracy: {_fmt_report_metric(metrics.get('exact_line_accuracy'))}", f"- prediction match rate: {_fmt_report_metric(metrics.get('prediction_match_rate'))}", f"- codepoint edits/ref: {metrics.get('codepoint_edits')}/{metrics.get('codepoint_ref_len')}", f"- grapheme edits/ref: {metrics.get('grapheme_edits')}/{metrics.get('grapheme_ref_len')}", f"- zero-length GT lines: {metrics.get('zero_length_gt_lines')}", f"- uncertainty method: {uncertainty.get('method')}", f"- uncertainty scoring units: {uncertainty.get('effective_sample_size')}/{uncertainty.get('min_claim_scoring_units')}", f"- grapheme CER 95% CI: {_fmt_report_metric(graph_interval.get('low'))}..{_fmt_report_metric(graph_interval.get('high'))}", "- command.txt", "- predictions.tsv", "- unmatched.tsv", "- worst-lines.csv", "- sample-review.csv", "- sample-review.html", "- confusions.json", ] ) slices = metrics.get("slices") if isinstance(metrics.get("slices"), list) else [] for row in slices: if not isinstance(row, dict): continue expected_lines.append( f"- {row.get('language')} / {row.get('script')} / {row.get('domain')} / {row.get('gt_status')} / {row.get('split')}: " f"n={row.get('n_lines')} graph_CER={_fmt_report_metric(row.get('cer_grapheme_micro'))} " f"cp_CER={_fmt_report_metric(row.get('cer_codepoint_micro'))} exact={_fmt_report_metric(row.get('exact_line_accuracy'))}" ) elif mode == "page_level_ocr": metric_means = metrics.get("metric_means") if isinstance(metrics.get("metric_means"), dict) else {} uncertainty = metrics.get("uncertainty") if isinstance(metrics.get("uncertainty"), dict) else {} intervals = uncertainty.get("metrics") if isinstance(uncertainty.get("metrics"), dict) else {} graph_interval = intervals.get("text_cer_grapheme_micro") if isinstance(intervals.get("text_cer_grapheme_micro"), dict) else {} inventory = metrics.get("script_inventory") if isinstance(metrics.get("script_inventory"), dict) else {} normalization = metrics.get("normalization_summary") if isinstance(metrics.get("normalization_summary"), dict) else {} dictionary_coverage = metrics.get("dictionary_coverage") if isinstance(metrics.get("dictionary_coverage"), dict) else {} expected_lines.extend( [ f"- system: {eval_run.get('system_name')}", f"- model: {eval_run.get('model_name')}", f"- checkpoint: {eval_run.get('checkpoint') or '(not recorded)'}", f"- config: {eval_run.get('config') or '(not recorded)'}", f"- dictionary: {eval_run.get('dictionary') or '(not recorded)'}", f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}", f"- normalization profile: {eval_run.get('normalization_profile')}", f"- eval manifest: {eval_run.get('eval_manifest')}", f"- documents dir: {eval_run.get('documents_dir')}", f"- manifest samples: {metrics.get('manifest_samples')}", f"- scored pages: {metrics.get('scored_pages')}", f"- missing documents: {metrics.get('missing_documents')}", f"- missing references: {metrics.get('missing_references')}", f"- script inventory file: {inventory.get('inventory_file')}", f"- observed codepoints/graphemes: {inventory.get('observed_codepoint_count')}/{inventory.get('observed_grapheme_count')}", f"- missing required codepoints: {inventory.get('missing_required_codepoints')}", f"- missing required graphemes: {inventory.get('missing_required_graphemes')}", f"- normalization profile: {normalization.get('profile')}", f"- normalization changed reference/prediction/any: {normalization.get('reference_changed')}/{normalization.get('prediction_changed')}/{normalization.get('any_changed')} of {normalization.get('total')}", f"- dictionary coverage checked: {dictionary_coverage.get('checked')}", f"- dictionary reference codepoints/dictionary codepoints: {dictionary_coverage.get('gt_codepoint_count')}/{dictionary_coverage.get('dictionary_codepoint_count')}", f"- dictionary missing reference codepoints: {dictionary_coverage.get('missing_gt_codepoints')}", f"- leakage checked: {metrics.get('leakage', {}).get('checked') if isinstance(metrics.get('leakage'), dict) else None}", f"- text codepoint CER micro: {_fmt_report_metric(metrics.get('text_cer_codepoint_micro'))}", f"- text grapheme CER micro: {_fmt_report_metric(metrics.get('text_cer_grapheme_micro'))}", f"- text exact accuracy: {_fmt_report_metric(metrics.get('text_exact_accuracy'))}", f"- line detection F1 mean: {_fmt_report_metric(metric_means.get('line_detection_f1'))}", f"- line mean IoU: {_fmt_report_metric(metric_means.get('line_mean_iou'))}", f"- reading-order pair accuracy: {_fmt_report_metric(metric_means.get('reading_order_pair_accuracy'))}", f"- text codepoint edits/ref: {metrics.get('text_codepoint_edits')}/{metrics.get('text_codepoint_ref_len')}", f"- text grapheme edits/ref: {metrics.get('text_grapheme_edits')}/{metrics.get('text_grapheme_ref_len')}", f"- text exact pages/ref pages: {metrics.get('text_exact_pages')}/{metrics.get('text_reference_pages')}", f"- zero-length text references: {metrics.get('zero_length_text_references')}", f"- uncertainty method: {uncertainty.get('method')}", f"- uncertainty scoring units: {uncertainty.get('effective_sample_size')}/{uncertainty.get('min_claim_scoring_units')}", f"- text grapheme CER 95% CI: {_fmt_report_metric(graph_interval.get('low'))}..{_fmt_report_metric(graph_interval.get('high'))}", "- command.txt", "- page-metrics.csv", "- reference-artifacts.tsv", "- text-pairs.tsv", "- missing.tsv", "- worst-pages.csv", "- sample-review.csv", "- sample-review.html", "- confusions.json", "- line-alignment.csv", ] ) slices = metrics.get("slices") if isinstance(metrics.get("slices"), list) else [] for row in slices: if not isinstance(row, dict): continue row_means = row.get("metric_means") if isinstance(row.get("metric_means"), dict) else {} expected_lines.append( f"- {row.get('language')} / {row.get('script')} / {row.get('domain')} / {row.get('gt_status')} / {row.get('split')}: " f"pages={row.get('scored_pages')} text_graph_CER={_fmt_report_metric(row.get('text_cer_grapheme_micro'))} " f"text_cp_CER={_fmt_report_metric(row.get('text_cer_codepoint_micro'))} " f"exact={_fmt_report_metric(row.get('text_exact_accuracy'))} line_F1={_fmt_report_metric(row_means.get('line_detection_f1', 0.0))}" ) for line in expected_lines: _expect_report_line(text, issues, line) warnings = eval_run.get("warnings") if isinstance(eval_run.get("warnings"), list) else [] if warnings: for warning in warnings: _expect_report_line(text, issues, f"- {warning}") else: _expect_report_line(text, issues, "- none") return issues def _expect_metrics_line(text: str, issues: list[str], line: str) -> None: if line not in text.splitlines(): issues.append(f"metrics.md missing or stale line: {line}") def _check_metrics_md_consistency(score_dir: Path, eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: issues: list[str] = [] path = score_dir / "metrics.md" if not path.exists(): return issues text = path.read_text(encoding="utf-8") expected_lines: list[str] = [] mode = eval_run.get("eval_mode") if mode == "recognizer_oracle_crop": uncertainty = metrics.get("uncertainty") if isinstance(metrics.get("uncertainty"), dict) else {} intervals = uncertainty.get("metrics") if isinstance(uncertainty.get("metrics"), dict) else {} graph_interval = intervals.get("cer_grapheme_micro") if isinstance(intervals.get("cer_grapheme_micro"), dict) else {} inventory = metrics.get("script_inventory") if isinstance(metrics.get("script_inventory"), dict) else {} normalization = metrics.get("normalization_summary") if isinstance(metrics.get("normalization_summary"), dict) else {} dictionary_coverage = metrics.get("dictionary_coverage") if isinstance(metrics.get("dictionary_coverage"), dict) else {} expected_lines.extend( [ "# Recognizer Eval Metrics", f"- lines: {metrics.get('n_lines')}", f"- label lines: {metrics.get('label_lines')}", f"- prediction match rate: {_fmt_report_metric(metrics.get('prediction_match_rate'))}", f"- unmatched labels: {metrics.get('unmatched_labels')}", f"- unmatched predictions: {metrics.get('unmatched_predictions')}", f"- codepoint CER: {_fmt_report_metric(metrics.get('cer_codepoint_micro'))}", f"- grapheme CER: {_fmt_report_metric(metrics.get('cer_grapheme_micro'))}", f"- exact-line accuracy: {_fmt_report_metric(metrics.get('exact_line_accuracy'))}", f"- codepoint edits/ref: {metrics.get('codepoint_edits')}/{metrics.get('codepoint_ref_len')}", f"- grapheme edits/ref: {metrics.get('grapheme_edits')}/{metrics.get('grapheme_ref_len')}", f"- zero-length GT lines: {metrics.get('zero_length_gt_lines')}", f"- normalization profile: {normalization.get('profile')}", f"- normalization changed GT/prediction/any: {normalization.get('gt_changed')}/{normalization.get('prediction_changed')}/{normalization.get('any_changed')} of {normalization.get('total')}", f"- dictionary coverage checked: {dictionary_coverage.get('checked')}", f"- dictionary GT codepoints/dictionary codepoints: {dictionary_coverage.get('gt_codepoint_count')}/{dictionary_coverage.get('dictionary_codepoint_count')}", f"- dictionary missing GT codepoints: {dictionary_coverage.get('missing_gt_codepoints')}", f"- uncertainty method: {uncertainty.get('method')}", f"- uncertainty scoring units: {uncertainty.get('effective_sample_size')}/{uncertainty.get('min_claim_scoring_units')}", f"- grapheme CER 95% CI: {_fmt_report_metric(graph_interval.get('low'))}..{_fmt_report_metric(graph_interval.get('high'))}", f"- script inventory file: {inventory.get('inventory_file')}", f"- observed codepoints/graphemes: {inventory.get('observed_codepoint_count')}/{inventory.get('observed_grapheme_count')}", f"- missing required codepoints: {inventory.get('missing_required_codepoints')}", f"- missing required graphemes: {inventory.get('missing_required_graphemes')}", ] ) slices = metrics.get("slices") if isinstance(metrics.get("slices"), list) else [] for row in slices: if not isinstance(row, dict): continue expected_lines.append( f"- {row.get('language')} / {row.get('script')} / {row.get('domain')} / {row.get('gt_status')} / {row.get('split')}: " f"n={row.get('n_lines')} graph_CER={_fmt_report_metric(row.get('cer_grapheme_micro'))} " f"cp_CER={_fmt_report_metric(row.get('cer_codepoint_micro'))} exact={_fmt_report_metric(row.get('exact_line_accuracy'))}" ) elif mode == "page_level_ocr": metric_means = metrics.get("metric_means") if isinstance(metrics.get("metric_means"), dict) else {} uncertainty = metrics.get("uncertainty") if isinstance(metrics.get("uncertainty"), dict) else {} intervals = uncertainty.get("metrics") if isinstance(uncertainty.get("metrics"), dict) else {} graph_interval = intervals.get("text_cer_grapheme_micro") if isinstance(intervals.get("text_cer_grapheme_micro"), dict) else {} inventory = metrics.get("script_inventory") if isinstance(metrics.get("script_inventory"), dict) else {} normalization = metrics.get("normalization_summary") if isinstance(metrics.get("normalization_summary"), dict) else {} dictionary_coverage = metrics.get("dictionary_coverage") if isinstance(metrics.get("dictionary_coverage"), dict) else {} expected_lines.extend( [ "# Page OCR Eval Metrics", f"- manifest samples: {metrics.get('manifest_samples')}", f"- scored pages: {metrics.get('scored_pages')}", f"- missing documents: {metrics.get('missing_documents')}", f"- missing references: {metrics.get('missing_references')}", f"- text codepoint CER micro: {_fmt_report_metric(metrics.get('text_cer_codepoint_micro'))}", f"- text grapheme CER micro: {_fmt_report_metric(metrics.get('text_cer_grapheme_micro'))}", f"- text exact accuracy: {_fmt_report_metric(metrics.get('text_exact_accuracy'))}", f"- text codepoint edits/ref: {metrics.get('text_codepoint_edits')}/{metrics.get('text_codepoint_ref_len')}", f"- text grapheme edits/ref: {metrics.get('text_grapheme_edits')}/{metrics.get('text_grapheme_ref_len')}", f"- text exact pages/ref pages: {metrics.get('text_exact_pages')}/{metrics.get('text_reference_pages')}", f"- zero-length text references: {metrics.get('zero_length_text_references')}", f"- normalization profile: {normalization.get('profile')}", f"- normalization changed reference/prediction/any: {normalization.get('reference_changed')}/{normalization.get('prediction_changed')}/{normalization.get('any_changed')} of {normalization.get('total')}", f"- dictionary coverage checked: {dictionary_coverage.get('checked')}", f"- dictionary reference codepoints/dictionary codepoints: {dictionary_coverage.get('gt_codepoint_count')}/{dictionary_coverage.get('dictionary_codepoint_count')}", f"- dictionary missing reference codepoints: {dictionary_coverage.get('missing_gt_codepoints')}", f"- line detection F1 mean: {_fmt_report_metric(metric_means.get('line_detection_f1', 0.0))}", f"- line mean IoU: {_fmt_report_metric(metric_means.get('line_mean_iou', 0.0))}", f"- uncertainty method: {uncertainty.get('method')}", f"- uncertainty scoring units: {uncertainty.get('effective_sample_size')}/{uncertainty.get('min_claim_scoring_units')}", f"- text grapheme CER 95% CI: {_fmt_report_metric(graph_interval.get('low'))}..{_fmt_report_metric(graph_interval.get('high'))}", f"- script inventory file: {inventory.get('inventory_file')}", f"- observed codepoints/graphemes: {inventory.get('observed_codepoint_count')}/{inventory.get('observed_grapheme_count')}", f"- missing required codepoints: {inventory.get('missing_required_codepoints')}", f"- missing required graphemes: {inventory.get('missing_required_graphemes')}", ] ) for key, value in metric_means.items(): expected_lines.append(f"- {key}: {_fmt_report_metric(value)}") slices = metrics.get("slices") if isinstance(metrics.get("slices"), list) else [] for row in slices: if not isinstance(row, dict): continue row_means = row.get("metric_means") if isinstance(row.get("metric_means"), dict) else {} expected_lines.append( f"- {row.get('language')} / {row.get('script')} / {row.get('domain')} / {row.get('gt_status')} / {row.get('split')}: " f"pages={row.get('scored_pages')} text_graph_CER={_fmt_report_metric(row.get('text_cer_grapheme_micro'))} " f"text_cp_CER={_fmt_report_metric(row.get('text_cer_codepoint_micro'))} " f"line_F1={_fmt_report_metric(row_means.get('line_detection_f1', 0.0))}" ) coverage = metrics.get("reference_coverage") if isinstance(metrics.get("reference_coverage"), dict) else {} for key, value in coverage.items(): expected_lines.append(f"- {key}: {value}") for line in expected_lines: _expect_metrics_line(text, issues, line) warnings = eval_run.get("warnings") if isinstance(eval_run.get("warnings"), list) else [] if warnings: for warning in warnings: _expect_metrics_line(text, issues, f"- {warning}") else: _expect_metrics_line(text, issues, "- none") return issues def _expect_root_report_line(text: str, issues: list[str], line: str) -> None: if line not in text.splitlines(): issues.append(f"root claim-safety.md missing or stale line: {line}") def _check_root_claim_safety_report_consistency( run_dir: Path, recipe_run: dict[str, Any], eval_run: dict[str, Any], metrics: dict[str, Any], ) -> list[str]: issues: list[str] = [] path = run_dir / "claim-safety.md" if not path.exists(): return issues text = path.read_text(encoding="utf-8") recipe: dict[str, Any] = {} if (run_dir / "recipe.json").exists(): try: raw_recipe = _read_json(run_dir / "recipe.json") except (FileNotFoundError, ValueError) as exc: issues.append(str(exc)) raw_recipe = {} if isinstance(raw_recipe, dict): recipe = raw_recipe else: issues.append("recipe.json must contain a JSON object") recipe_audit = recipe_run.get("audit") if isinstance(recipe_run.get("audit"), dict) else {} machine = eval_run.get("machine") if isinstance(eval_run.get("machine"), dict) else {} leakage = metrics.get("leakage") if isinstance(metrics.get("leakage"), dict) else {} uncertainty = metrics.get("uncertainty") if isinstance(metrics.get("uncertainty"), dict) else {} intervals = uncertainty.get("metrics") if isinstance(uncertainty.get("metrics"), dict) else {} script_inventory = metrics.get("script_inventory") if isinstance(metrics.get("script_inventory"), dict) else {} expected_lines = [ f"- recipe: {recipe_run.get('recipe_id')}", f"- task: {recipe.get('task')}", f"- audit passed: {recipe_audit.get('audit_passed')}", f"- claim safe: {eval_run.get('claim_safe')}", f"- eval mode: {eval_run.get('eval_mode')}", f"- score created at: {eval_run.get('created_at')}", f"- score machine platform: {machine.get('platform')}", f"- score machine python: {machine.get('python')}", f"- score command: {eval_run.get('command')}", ] mode = eval_run.get("eval_mode") if mode == "page_level_ocr": expected_lines.extend( [ f"- system: {eval_run.get('system_name')}", f"- model: {eval_run.get('model_name')}", f"- checkpoint: {eval_run.get('checkpoint') or '(not recorded)'}", f"- config: {eval_run.get('config') or '(not recorded)'}", f"- dictionary: {eval_run.get('dictionary') or '(not recorded)'}", f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}", f"- normalization profile: {eval_run.get('normalization_profile')}", f"- eval manifest: {eval_run.get('eval_manifest')}", f"- documents dir: {eval_run.get('documents_dir')}", f"- manifest samples: {metrics.get('manifest_samples')}", f"- scored pages: {metrics.get('scored_pages')}", f"- missing documents: {metrics.get('missing_documents')}", f"- missing references: {metrics.get('missing_references')}", ] ) else: expected_lines.extend( [ f"- model: {eval_run.get('model_name')}", f"- checkpoint: {eval_run.get('checkpoint') or '(not recorded)'}", f"- config: {eval_run.get('config') or '(not recorded)'}", f"- dictionary: {eval_run.get('dictionary') or '(not recorded)'}", f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}", f"- normalization profile: {eval_run.get('normalization_profile')}", f"- label file: {eval_run.get('label_file')}", f"- prediction file: {eval_run.get('prediction_file')}", f"- scored lines: {metrics.get('n_lines')}", f"- unmatched labels: {metrics.get('unmatched_labels')}", f"- unmatched predictions: {metrics.get('unmatched_predictions')}", ] ) expected_lines.extend( [ f"- eval pack version: {eval_run.get('eval_pack_version') or '(missing)'}", f"- frozen eval pack: {eval_run.get('frozen_eval_pack')}", f"- unseen for selection: {eval_run.get('unseen_for_selection')}", f"- selected checkpoint on test: {eval_run.get('selected_checkpoint_on_test')}", f"- languages: {eval_run.get('languages')}", f"- scripts: {eval_run.get('scripts')}", f"- domains: {eval_run.get('domains')}", f"- GT statuses: {eval_run.get('gt_statuses')}", f"- splits: {eval_run.get('splits')}", f"- script inventory file: {script_inventory.get('inventory_file')}", f"- observed codepoints/graphemes: {script_inventory.get('observed_codepoint_count')}/{script_inventory.get('observed_grapheme_count')}", f"- missing required codepoints: {script_inventory.get('missing_required_codepoints')}", f"- missing required graphemes: {script_inventory.get('missing_required_graphemes')}", f"- metadata contradictions: {metrics.get('metadata_contradictions', [])}", ] ) if mode == "page_level_ocr": metric_means = metrics.get("metric_means") if isinstance(metrics.get("metric_means"), dict) else {} normalization = metrics.get("normalization_summary") if isinstance(metrics.get("normalization_summary"), dict) else {} dictionary_coverage = metrics.get("dictionary_coverage") if isinstance(metrics.get("dictionary_coverage"), dict) else {} expected_lines.extend( [ f"- metadata coverage: {metrics.get('metadata_coverage')}", f"- reference coverage: {metrics.get('reference_coverage')}", f"- normalization changed reference/prediction/any: {normalization.get('reference_changed')}/{normalization.get('prediction_changed')}/{normalization.get('any_changed')} of {normalization.get('total')}", f"- dictionary coverage checked: {dictionary_coverage.get('checked')}", f"- dictionary reference codepoints/dictionary codepoints: {dictionary_coverage.get('gt_codepoint_count')}/{dictionary_coverage.get('dictionary_codepoint_count')}", f"- dictionary missing reference codepoints: {dictionary_coverage.get('missing_gt_codepoints')}", f"- leakage checked: {leakage.get('checked')}", f"- text codepoint CER micro: {_fmt_report_metric(metrics.get('text_cer_codepoint_micro'))}", f"- text grapheme CER micro: {_fmt_report_metric(metrics.get('text_cer_grapheme_micro'))}", f"- text exact accuracy: {_fmt_report_metric(metrics.get('text_exact_accuracy'))}", f"- line detection F1 mean: {_fmt_report_metric(metric_means.get('line_detection_f1'))}", f"- line mean IoU: {_fmt_report_metric(metric_means.get('line_mean_iou'))}", f"- reading-order pair accuracy: {_fmt_report_metric(metric_means.get('reading_order_pair_accuracy'))}", f"- text codepoint edits/ref: {metrics.get('text_codepoint_edits')}/{metrics.get('text_codepoint_ref_len')}", f"- text grapheme edits/ref: {metrics.get('text_grapheme_edits')}/{metrics.get('text_grapheme_ref_len')}", f"- text exact pages/ref pages: {metrics.get('text_exact_pages')}/{metrics.get('text_reference_pages')}", f"- zero-length text references: {metrics.get('zero_length_text_references')}", f"- uncertainty method: {uncertainty.get('method')}", f"- uncertainty scoring units: {uncertainty.get('effective_sample_size')}/{uncertainty.get('min_claim_scoring_units')}", f"- text grapheme CER 95% CI: {_fmt_report_metric((intervals.get('text_cer_grapheme_micro') or {}).get('low'))}..{_fmt_report_metric((intervals.get('text_cer_grapheme_micro') or {}).get('high'))}", ] ) slices = metrics.get("slices") if isinstance(metrics.get("slices"), list) else [] for row in slices: if not isinstance(row, dict): continue row_means = row.get("metric_means") if isinstance(row.get("metric_means"), dict) else {} expected_lines.append( f"- {row.get('language')} / {row.get('script')} / {row.get('domain')} / {row.get('gt_status')} / {row.get('split')}: " f"pages={row.get('scored_pages')} text_graph_CER={_fmt_report_metric(row.get('text_cer_grapheme_micro'))} " f"text_cp_CER={_fmt_report_metric(row.get('text_cer_codepoint_micro'))} " f"exact={_fmt_report_metric(row.get('text_exact_accuracy'))} line_F1={_fmt_report_metric(row_means.get('line_detection_f1', 0.0))}" ) expected_lines.extend( [ "- audit JSON: audit.json", "- eval run JSON: score/page-eval-run.json", "- metrics JSON: score/metrics.json", "- metrics Markdown: score/metrics.md", "- command text: score/command.txt", "- page metrics CSV: score/page-metrics.csv", "- reference artifacts TSV: score/reference-artifacts.tsv", "- text pairs TSV: score/text-pairs.tsv", "- missing TSV: score/missing.tsv", "- worst pages: score/worst-pages.csv", "- line alignment CSV: score/line-alignment.csv", "- review CSV: score/sample-review.csv", "- review HTML: score/sample-review.html", "- confusions JSON: score/confusions.json", ] ) else: normalization = metrics.get("normalization_summary") if isinstance(metrics.get("normalization_summary"), dict) else {} dictionary_coverage = metrics.get("dictionary_coverage") if isinstance(metrics.get("dictionary_coverage"), dict) else {} expected_lines.extend( [ f"- provenance coverage: {metrics.get('provenance_coverage')}", f"- normalizer counts: {metrics.get('normalizer_counts', {})}", f"- normalization changed GT/prediction/any: {normalization.get('gt_changed')}/{normalization.get('prediction_changed')}/{normalization.get('any_changed')} of {normalization.get('total')}", f"- dictionary coverage checked: {dictionary_coverage.get('checked')}", f"- dictionary GT codepoints/dictionary codepoints: {dictionary_coverage.get('gt_codepoint_count')}/{dictionary_coverage.get('dictionary_codepoint_count')}", f"- dictionary missing GT codepoints: {dictionary_coverage.get('missing_gt_codepoints')}", f"- leakage checked: {leakage.get('checked')}", f"- codepoint CER micro: {_fmt_report_metric(metrics.get('cer_codepoint_micro'))}", f"- grapheme CER micro: {_fmt_report_metric(metrics.get('cer_grapheme_micro'))}", f"- exact-line accuracy: {_fmt_report_metric(metrics.get('exact_line_accuracy'))}", f"- codepoint edits/ref: {metrics.get('codepoint_edits')}/{metrics.get('codepoint_ref_len')}", f"- grapheme edits/ref: {metrics.get('grapheme_edits')}/{metrics.get('grapheme_ref_len')}", f"- zero-length GT lines: {metrics.get('zero_length_gt_lines')}", f"- uncertainty method: {uncertainty.get('method')}", f"- uncertainty scoring units: {uncertainty.get('effective_sample_size')}/{uncertainty.get('min_claim_scoring_units')}", f"- grapheme CER 95% CI: {_fmt_report_metric((intervals.get('cer_grapheme_micro') or {}).get('low'))}..{_fmt_report_metric((intervals.get('cer_grapheme_micro') or {}).get('high'))}", ] ) slices = metrics.get("slices") if isinstance(metrics.get("slices"), list) else [] for row in slices: if not isinstance(row, dict): continue expected_lines.append( f"- {row.get('language')} / {row.get('script')} / {row.get('domain')} / {row.get('gt_status')} / {row.get('split')}: " f"n={row.get('n_lines')} graph_CER={_fmt_report_metric(row.get('cer_grapheme_micro'))} " f"cp_CER={_fmt_report_metric(row.get('cer_codepoint_micro'))} exact={_fmt_report_metric(row.get('exact_line_accuracy'))}" ) expected_lines.extend( [ "- audit JSON: audit.json", "- eval run JSON: score/eval-run.json", "- metrics JSON: score/metrics.json", "- metrics Markdown: score/metrics.md", "- command text: score/command.txt", "- predictions TSV: score/predictions.tsv", "- unmatched TSV: score/unmatched.tsv", "- worst lines: score/worst-lines.csv", "- review CSV: score/sample-review.csv", "- review HTML: score/sample-review.html", "- confusions JSON: score/confusions.json", ] ) for line in expected_lines: _expect_root_report_line(text, issues, line) warnings = eval_run.get("warnings") if isinstance(eval_run.get("warnings"), list) else [] if warnings: for warning in warnings: _expect_root_report_line(text, issues, f"- {warning}") else: _expect_root_report_line(text, issues, "- none") return issues def _check_timestamp(payload: dict[str, Any], field: str, label: str) -> list[str]: value = payload.get(field) if not isinstance(value, str) or not value: return [f"{label} {field} timestamp is missing"] try: datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: return [f"{label} {field} timestamp is invalid: {value}"] return [] def _check_required_fields(payload: dict[str, Any], fields: tuple[str, ...], label: str) -> list[str]: issues: list[str] = [] for field in fields: if field not in payload: issues.append(f"{label} missing required field: {field}") elif payload[field] in (None, "") and field not in {"checkpoint", "config", "dictionary"}: issues.append(f"{label} has empty required field: {field}") return issues def _check_normalization_profile(eval_run: dict[str, Any]) -> list[str]: profile = eval_run.get("normalization_profile") if profile != SUPPORTED_NORMALIZATION_PROFILE: return [ "eval run normalization_profile does not match supported metric recomputation profile: " f"recorded={profile!r} expected={SUPPORTED_NORMALIZATION_PROFILE!r}" ] return [] def _check_normalization_summary(recorded: Any, expected: dict[str, Any], label: str) -> list[str]: if not isinstance(recorded, dict): return [f"{label} normalization_summary missing or not an object"] issues: list[str] = [] for key, expected_value in expected.items(): if recorded.get(key) != expected_value: issues.append(f"{label} normalization_summary {key} mismatch: recorded={recorded.get(key)} recomputed={expected_value}") return issues def _expected_recognizer_normalization_summary(score_dir: Path) -> dict[str, Any]: rows = _tsv_rows(score_dir / "predictions.tsv") gt_changed = sum(1 for row in rows if (row.get("gt_text") or "") != _nfc(row.get("gt_text") or "")) prediction_changed = sum(1 for row in rows if (row.get("prediction") or "") != _nfc(row.get("prediction") or "")) return { "profile": SUPPORTED_NORMALIZATION_PROFILE, "unit": "line", "total": len(rows), "gt_changed": gt_changed, "prediction_changed": prediction_changed, "any_changed": sum( 1 for row in rows if (row.get("gt_text") or "") != _nfc(row.get("gt_text") or "") or (row.get("prediction") or "") != _nfc(row.get("prediction") or "") ), } def _expected_page_normalization_summary(score_dir: Path) -> dict[str, Any]: rows = _tsv_rows(score_dir / "text-pairs.tsv") prediction_changed = sum(1 for row in rows if (row.get("prediction_text") or "") != _nfc(row.get("prediction_text") or "")) reference_changed = sum(1 for row in rows if (row.get("reference_text") or "") != _nfc(row.get("reference_text") or "")) return { "profile": SUPPORTED_NORMALIZATION_PROFILE, "unit": "page", "total": len(rows), "prediction_changed": prediction_changed, "reference_changed": reference_changed, "any_changed": sum( 1 for row in rows if (row.get("prediction_text") or "") != _nfc(row.get("prediction_text") or "") or (row.get("reference_text") or "") != _nfc(row.get("reference_text") or "") ), } def _check_machine_metadata(payload: dict[str, Any], label: str) -> list[str]: machine = payload.get("machine") if not isinstance(machine, dict): return [f"{label} machine metadata is missing or not an object"] issues: list[str] = [] for field in ("platform", "python"): value = machine.get(field) if not isinstance(value, str) or not value: issues.append(f"{label} machine.{field} is missing") return issues def _check_common(score_dir: Path, eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: issues: list[str] = [] if eval_run.get("claim_safe") is True and eval_run.get("warnings"): issues.append("claim_safe is true but warnings are present") if eval_run.get("claim_safe") is False and not eval_run.get("warnings"): issues.append("claim_safe is false but warnings are empty") issues.extend(_check_machine_metadata(eval_run, "eval run")) if not isinstance(eval_run.get("command"), str) or not eval_run["command"]: issues.append("eval run command is missing") else: command_file = score_dir / "command.txt" if command_file.exists() and command_file.read_text(encoding="utf-8").strip() != eval_run["command"].strip(): issues.append("command.txt does not match eval run command") issues.extend(_check_normalization_profile(eval_run)) issues.extend(_check_timestamp(eval_run, "created_at", "eval run")) metrics_md = score_dir / "metrics.md" if metrics_md.exists() and not metrics_md.read_text(encoding="utf-8").strip(): issues.append("metrics.md is blank") elif metrics_md.exists(): issues.extend(_check_metrics_md_consistency(score_dir, eval_run, metrics)) claim_report = score_dir / "claim-safety.md" if claim_report.exists(): text = claim_report.read_text(encoding="utf-8") if not text.strip(): issues.append("claim-safety.md is blank") for marker in ("## What Was Scored", "## Ground Truth And Pack", "## Metrics", "## Warnings"): if marker not in text: issues.append(f"claim-safety.md missing section: {marker}") issues.extend(_check_claim_safety_report_consistency(score_dir, eval_run, metrics)) if not isinstance(metrics, dict): issues.append("metrics artifact is not a JSON object") return issues def _warning_texts(eval_run: dict[str, Any]) -> list[str]: warnings = eval_run.get("warnings") if not isinstance(warnings, list): return [] return [str(warning) for warning in warnings] def _has_warning(eval_run: dict[str, Any], prefix: str) -> bool: return any(warning.startswith(prefix) for warning in _warning_texts(eval_run)) def _claim_blocker(issues: list[str], eval_run: dict[str, Any], mode: str, reason: str, warning_prefix: str) -> None: if eval_run.get("claim_safe") is True: issues.append(f"{mode} claim_safe is true despite {reason}") if not _has_warning(eval_run, warning_prefix): issues.append(f"{mode} missing claim-safety warning for {reason}: {warning_prefix}") def _string_set(value: Any) -> set[str]: if not isinstance(value, list): return set() return {str(item) for item in value} def _check_eval_run_identity_lists( eval_run: dict[str, Any], rows: list[dict[str, str]], *, run_label: str, row_label: str, ) -> list[str]: issues: list[str] = [] if not rows: return issues mappings = ( ("languages", "language"), ("scripts", "script"), ("domains", "domain"), ("gt_statuses", "gt_status"), ("splits", "split"), ) for run_field, row_field in mappings: recorded = eval_run.get(run_field) expected = sorted({row.get(row_field) or "" for row in rows}) if not isinstance(recorded, list): issues.append(f"{run_label} {run_field} is not a list for {row_label} identity audit") continue recorded_values = [str(item) for item in recorded] if recorded_values != expected: issues.append( f"{run_label} {run_field} does not match {row_label} {row_field} values: " f"recorded={recorded_values} recomputed={expected}" ) return issues def _check_shared_claim_blockers(mode: str, eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: issues: list[str] = [] if not isinstance(eval_run.get("model_name"), str) or not eval_run["model_name"].strip(): _claim_blocker(issues, eval_run, mode, "missing model identity", "model name not recorded") if mode == "page" and (not isinstance(eval_run.get("system_name"), str) or not eval_run["system_name"].strip()): _claim_blocker(issues, eval_run, mode, "missing system identity", "system name not recorded") if not eval_run.get("eval_pack_version"): _claim_blocker(issues, eval_run, mode, "missing eval-pack version", "eval pack version not recorded") artifact_identity = eval_run.get("artifact_identity") if isinstance(artifact_identity, dict): for name, identity in sorted(artifact_identity.items()): if isinstance(identity, dict) and not identity.get("verified"): _claim_blocker(issues, eval_run, mode, f"unverified {name} artifact", f"{name} artifact is not hash-verifiable:") if eval_run.get("frozen_eval_pack") is not True: _claim_blocker(issues, eval_run, mode, "non-frozen eval pack", "eval pack is not declared frozen") if eval_run.get("unseen_for_selection") is not True: _claim_blocker(issues, eval_run, mode, "eval pack not declared unseen for selection", "eval pack is not declared unseen for model/checkpoint selection") if eval_run.get("selected_checkpoint_on_test") is True: _claim_blocker(issues, eval_run, mode, "checkpoint selection on test data", "checkpoint/model selection on this test pack is declared") if not eval_run.get("summary_file"): _claim_blocker(issues, eval_run, mode, "missing eval pack summary/datasheet", "eval pack summary/datasheet missing") if metrics.get("metadata_contradictions"): _claim_blocker(issues, eval_run, mode, "metadata contradictions", "metadata contradictions detected:") if eval_run.get("known_leakage"): _claim_blocker(issues, eval_run, mode, "known leakage acknowledgement", "known leakage acknowledged:") languages = _string_set(eval_run.get("languages")) scripts = _string_set(eval_run.get("scripts")) domains = _string_set(eval_run.get("domains")) gt_statuses = _string_set(eval_run.get("gt_statuses")) splits = _string_set(eval_run.get("splits")) if len(languages) > 1: _claim_blocker(issues, eval_run, mode, "mixed languages", "mixed languages:") if len(scripts) > 1: _claim_blocker(issues, eval_run, mode, "mixed scripts", "mixed scripts:") if len(domains) > 1: _claim_blocker(issues, eval_run, mode, "mixed eval domains", "mixed eval domains:") if len(gt_statuses) > 1: _claim_blocker(issues, eval_run, mode, "mixed GT statuses", "mixed GT statuses:") if len(splits) > 1: _claim_blocker(issues, eval_run, mode, "mixed split statuses", "mixed split statuses:") if splits and not splits <= CLAIM_EVAL_SPLITS: _claim_blocker(issues, eval_run, mode, "unexpected split status", "unexpected split status for claim evaluation:") if domains and not domains <= CLAIM_SAFE_DOMAINS: _claim_blocker(issues, eval_run, mode, "unsafe eval domain", "not claim-safe real domain:") if gt_statuses and not gt_statuses <= CLAIM_SAFE_GT_STATUSES: _claim_blocker(issues, eval_run, mode, "untrusted GT status", "not human-verified/spot-checked GT:") uncertainty = metrics.get("uncertainty") if not isinstance(uncertainty, dict): _claim_blocker(issues, eval_run, mode, "missing claim uncertainty evidence", "claim-facing uncertainty missing") elif uncertainty.get("claim_sample_size_ok") is not True: _claim_blocker(issues, eval_run, mode, "claim-facing sample size too small", "claim-facing sample size too small:") slices = metrics.get("slices") if isinstance(slices, list): for row in slices: if not isinstance(row, dict): continue slice_uncertainty = row.get("uncertainty") if isinstance(slice_uncertainty, dict): count = int(slice_uncertainty.get("effective_sample_size", 0) or 0) minimum = int(slice_uncertainty.get("min_claim_scoring_units", MIN_CLAIM_SCORING_UNITS) or MIN_CLAIM_SCORING_UNITS) ok = slice_uncertainty.get("claim_sample_size_ok") is True else: count = int(row.get("n_lines", row.get("scored_pages", 0)) or 0) minimum = MIN_CLAIM_SCORING_UNITS ok = count >= minimum if not ok or count < minimum: _claim_blocker(issues, eval_run, mode, "slice sample size too small", "slice sample size too small:") script_inventory = metrics.get("script_inventory") if not isinstance(script_inventory, dict) or not script_inventory.get("inventory_file"): _claim_blocker(issues, eval_run, mode, "missing script inventory evidence", "script inventory file missing") elif script_inventory.get("missing_required_codepoints") or script_inventory.get("missing_required_graphemes"): if script_inventory.get("missing_required_codepoints"): _claim_blocker(issues, eval_run, mode, "missing required script codepoints", "missing required script codepoints:") if script_inventory.get("missing_required_graphemes"): _claim_blocker(issues, eval_run, mode, "missing required script graphemes", "missing required script graphemes:") if "machine_converted_spot_checked" in gt_statuses: calibration = eval_run.get("spotcheck_calibration") if not isinstance(calibration, dict): _claim_blocker( issues, eval_run, mode, "spot-checked machine GT without calibration artifact", "machine_converted_spot_checked GT declared without native spot-check calibration artifact", ) elif not calibration.get("claim_evidence_eligible"): _claim_blocker( issues, eval_run, mode, "ineligible native spot-check calibration", "native spot-check calibration artifact is not claim evidence eligible", ) leakage = metrics.get("leakage") if not isinstance(leakage, dict): _claim_blocker(issues, eval_run, mode, "unrun leakage check", "train/dev leakage check not run") return issues if not leakage.get("checked"): train_manifests = eval_run.get("train_manifests") warning_prefix = ( "train/dev leakage manifests were provided but no train rows were checked" if isinstance(train_manifests, list) and train_manifests else "train/dev leakage check not run" ) _claim_blocker(issues, eval_run, mode, "unrun leakage check", warning_prefix) overlaps = leakage.get("overlaps") if not isinstance(overlaps, dict): return issues hard_leakage = { key: values for key, values in overlaps.items() if key != "text_sha256" and isinstance(values, list) and values } if hard_leakage: _claim_blocker(issues, eval_run, mode, "train/dev/eval leakage", "train/dev/eval leakage detected:") text_overlap = overlaps.get("text_sha256") if isinstance(text_overlap, list) and text_overlap: _claim_blocker(issues, eval_run, mode, "train/dev/eval text overlap", "train/dev/eval text overlap detected:") return issues def _check_recognizer_claim_blockers(eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: issues = _check_shared_claim_blockers("recognizer", eval_run, metrics) matched_lines = metrics.get("matched_lines") if matched_lines == 0: _claim_blocker(issues, eval_run, "recognizer", "no matched rows scored", "no matched rows scored") unmatched_count = int(metrics.get("unmatched_labels", 0) or 0) + int(metrics.get("unmatched_predictions", 0) or 0) if unmatched_count: _claim_blocker(issues, eval_run, "recognizer", "unmatched prediction/label rows", "unmatched rows present:") match_rate = metrics.get("prediction_match_rate") min_match_rate = eval_run.get("min_match_rate") if _is_number(match_rate) and _is_number(min_match_rate) and float(match_rate) < float(min_match_rate): _claim_blocker(issues, eval_run, "recognizer", "low prediction match rate", "low prediction match rate:") if not eval_run.get("manifest_file"): _claim_blocker(issues, eval_run, "recognizer", "missing eval manifest", "eval manifest missing") if eval_run.get("claim_eligible_default") is not True: _claim_blocker(issues, eval_run, "recognizer", "rows not guaranteed claim eligible", "not all rows are claim eligible:") coverage = metrics.get("provenance_coverage") if isinstance(coverage, dict): for key, count in sorted(coverage.items()): if count: _claim_blocker(issues, eval_run, "recognizer", f"missing provenance field {key}", f"{key}:") return issues def _check_recognizer_row_claim_eligibility_blockers(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: rows = _tsv_rows(score_dir / "predictions.tsv") if not rows: return [] ineligible = [ row.get("sample_id") or row.get("image_path") or f"row-{index}" for index, row in enumerate(rows, 1) if row.get("claim_eligible") != "True" ] if not ineligible: return [] issues: list[str] = [] _claim_blocker(issues, eval_run, "recognizer", "row-level claim ineligible values", "not all rows are claim eligible:") return issues def _read_page_output_audit_for_replay( sample_id: str, output_dir: Path, *, filename: str, passed_field: str, ) -> dict[str, Any]: audit_path = output_dir / filename row: dict[str, Any] = { "sample_id": sample_id, "output_dir": str(output_dir), "audit_path": str(audit_path), "exists": audit_path.is_file(), "passed": False, "sha256": None, "size_bytes": None, "issue": "", } if not audit_path.is_file(): row["issue"] = f"missing page output audit: {audit_path}" return row row["sha256"] = _sha256_file(audit_path) row["size_bytes"] = audit_path.stat().st_size try: payload = json.loads(audit_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: row["issue"] = f"page output audit is invalid JSON: {audit_path}: {exc}" return row if not isinstance(payload, dict): row["issue"] = f"page output audit must contain a JSON object: {audit_path}" return row passed = payload.get(passed_field) row["passed"] = passed is True if passed is not True: row["issue"] = f"page output audit did not pass: {audit_path} {passed_field}={passed!r}" return row def _expected_page_output_audit(score_dir: Path, eval_run: dict[str, Any]) -> dict[str, Any]: required = bool(eval_run.get("require_output_audit")) filename = str(eval_run.get("output_audit_filename") or "limbu-output-audit.json") passed_field = str(eval_run.get("output_audit_passed_field") or "passed") output_audit_rows: list[dict[str, Any]] = [] page_rows = _csv_rows(score_dir / "page-metrics.csv") if required: for row in page_rows: sample_id = str(row.get("sample_id") or "") document_path_value = row.get("document_path") or "" if not document_path_value: output_audit_rows.append( { "sample_id": sample_id, "output_dir": "", "audit_path": filename, "exists": False, "passed": False, "sha256": None, "size_bytes": None, "issue": f"page-metrics.csv row has no document_path for output audit replay: {sample_id}", } ) continue output_audit_rows.append( _read_page_output_audit_for_replay( sample_id, Path(document_path_value).parent, filename=filename, passed_field=passed_field, ) ) return { "required": required, "filename": filename, "passed_field": passed_field, "expected_count": len(page_rows) if required else 0, "checked_count": len(output_audit_rows), "passed_count": sum(1 for row in output_audit_rows if row.get("passed") is True), "missing_count": sum(1 for row in output_audit_rows if row.get("exists") is not True), "failed_count": sum(1 for row in output_audit_rows if row.get("exists") is True and row.get("passed") is not True), "issues": [str(row.get("issue")) for row in output_audit_rows if row.get("issue")], "artifacts": output_audit_rows, } def _check_page_output_audit(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] expected = _expected_page_output_audit(score_dir, eval_run) recorded_metrics = metrics.get("page_output_audit") if not isinstance(recorded_metrics, dict): issues.append("metrics.json missing page_output_audit object") elif recorded_metrics != expected: issues.append("metrics.json page_output_audit does not match current page output audit artifacts") recorded_eval_run = eval_run.get("page_output_audit") if not isinstance(recorded_eval_run, dict): issues.append("page-eval-run.json missing page_output_audit object") elif recorded_eval_run != expected: issues.append("page-eval-run.json page_output_audit does not match current page output audit artifacts") if expected["required"] and expected["passed_count"] != expected["expected_count"]: issues.append( "replayed page output audits are not all passing: " f"{expected['passed_count']}/{expected['expected_count']} filename={expected['filename']!r}" ) return issues def _check_page_claim_blockers(eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: issues = _check_shared_claim_blockers("page", eval_run, metrics) if int(metrics.get("missing_documents", 0) or 0): _claim_blocker(issues, eval_run, "page", "missing document outputs", "missing document outputs:") if int(metrics.get("missing_references", 0) or 0): _claim_blocker(issues, eval_run, "page", "missing references", "missing references:") scored_pages = metrics.get("scored_pages") manifest_samples = metrics.get("manifest_samples") if _is_number(scored_pages) and _is_number(manifest_samples) and int(scored_pages) < int(manifest_samples): _claim_blocker(issues, eval_run, "page", "incomplete page scoring coverage", "incomplete page scoring coverage:") if metrics.get("document_source_path_mismatches"): _claim_blocker(issues, eval_run, "page", "document source_path mismatches", "document source_path mismatches detected:") if eval_run.get("claim_eligible_default") is not True: _claim_blocker(issues, eval_run, "page", "eval pack not declared claim eligible", "eval pack is not declared claim eligible") claim_eligible_rows = metrics.get("claim_eligible_rows") if _is_number(claim_eligible_rows) and _is_number(manifest_samples) and int(claim_eligible_rows) != int(manifest_samples): _claim_blocker(issues, eval_run, "page", "not all manifest rows are claim eligible", "not all manifest rows are claim eligible:") metadata_coverage = metrics.get("metadata_coverage") if isinstance(metadata_coverage, dict): for key, count in sorted(metadata_coverage.items()): if count: _claim_blocker(issues, eval_run, "page", f"missing row metadata {key}", f"{key}:") reference_coverage = metrics.get("reference_coverage") if isinstance(reference_coverage, dict): if not reference_coverage.get("text"): _claim_blocker(issues, eval_run, "page", "no text references scored", "no text references scored") if not reference_coverage.get("reading_order"): _claim_blocker(issues, eval_run, "page", "no reading-order references scored", "no reading-order references scored") if not reference_coverage.get("line_alignment_scored"): _claim_blocker(issues, eval_run, "page", "no line-bbox alignment references scored", "no line-bbox alignment references scored") if not (reference_coverage.get("tables") or reference_coverage.get("figures")): _claim_blocker(issues, eval_run, "page", "no layout detection references scored", "no layout detection references scored") return issues def _check_ratio(metrics: dict[str, Any], numerator: str, denominator: str, ratio: str) -> list[str]: issues: list[str] = [] for key in (numerator, denominator, ratio): if key not in metrics or not _is_number(metrics[key]): issues.append(f"metrics missing numeric field: {key}") return issues expected = metrics[numerator] / max(metrics[denominator], 1) if not _close(float(metrics[ratio]), float(expected)): issues.append(f"metrics {ratio} does not equal {numerator}/{denominator}") return issues def _nfc(value: str) -> str: return unicodedata.normalize("NFC", value or "") def _graphemes(value: str) -> list[str]: return regex.findall(r"\X", _nfc(value)) def _codepoint_id(ch: str) -> str: return f"U+{ord(ch):04X}" def _hash_existing_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _read_script_inventory(path: Path | None) -> tuple[dict[str, Any], list[str]]: if path is None: return {}, [] if not path.exists(): return {}, [f"script inventory file does not exist: {path}"] try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return {}, [f"script inventory file is invalid JSON: {path}: {exc}"] if not isinstance(payload, dict): return {}, [f"script inventory file must contain a JSON object: {path}"] return payload, [] def _required_codepoints(payload: dict[str, Any]) -> tuple[list[str], list[str]]: raw = payload.get("required_codepoints", []) if not isinstance(raw, list): return [], ["script inventory required_codepoints must be a list"] required: set[str] = set() issues: list[str] = [] for item in raw: if not isinstance(item, str): issues.append(f"script inventory codepoint entry must be a string: {item!r}") continue value = item.strip() if not value: continue if value.upper().startswith("U+"): try: required.add(f"U+{int(value[2:], 16):04X}") except ValueError: issues.append(f"script inventory codepoint entry is not valid hex: {item!r}") continue normalized = _nfc(value) if len(normalized) != 1: issues.append(f"script inventory codepoint entry must be one character or U+XXXX: {item!r}") continue required.add(_codepoint_id(normalized)) return sorted(required), issues def _required_graphemes(payload: dict[str, Any]) -> tuple[list[str], list[str]]: raw = payload.get("required_graphemes", []) if not isinstance(raw, list): return [], ["script inventory required_graphemes must be a list"] required: set[str] = set() issues: list[str] = [] for item in raw: if not isinstance(item, str): issues.append(f"script inventory grapheme entry must be a string: {item!r}") continue value = _nfc(item) if value.strip(): required.add(value) return sorted(required), issues def _expected_script_inventory(texts: list[str], inventory_path: Path | None) -> tuple[dict[str, Any], list[str]]: payload, issues = _read_script_inventory(inventory_path) required_cps, cp_issues = _required_codepoints(payload) required_graphs, graph_issues = _required_graphemes(payload) issues.extend(cp_issues) issues.extend(graph_issues) text = _nfc("\n".join(texts)) codepoints = sorted({_codepoint_id(ch) for ch in text if not ch.isspace()}) graphemes = sorted({cluster for cluster in _graphemes(text) if cluster.strip()}) categories = [unicodedata.category(ch) for ch in text if not ch.isspace()] return { "inventory_file": str(inventory_path) if inventory_path else None, "inventory_file_sha256": _hash_existing_file(inventory_path) if inventory_path and inventory_path.exists() else None, "observed_codepoint_count": len(codepoints), "observed_grapheme_count": len(graphemes), "observed_codepoints": codepoints, "observed_graphemes": graphemes, "required_codepoints": required_cps, "required_graphemes": required_graphs, "missing_required_codepoints": sorted(set(required_cps) - set(codepoints)), "missing_required_graphemes": sorted(set(required_graphs) - set(graphemes)), "digit_codepoint_count": sum(1 for ch in text if not ch.isspace() and unicodedata.category(ch) == "Nd"), "punctuation_codepoint_count": sum(1 for category in categories if category.startswith("P")), "combining_mark_codepoint_count": sum(1 for category in categories if category.startswith("M")), }, issues def _check_script_inventory(recorded: Any, expected: dict[str, Any], label: str) -> list[str]: if not isinstance(recorded, dict): return [f"{label} script_inventory missing or not an object"] issues: list[str] = [] for key, expected_value in expected.items(): if recorded.get(key) != expected_value: issues.append(f"{label} script_inventory {key} mismatch: recorded={recorded.get(key)} recomputed={expected_value}") return issues def _dictionary_codepoints(dictionary: str) -> set[str] | None: path = Path(str(dictionary or "")) if not path.exists() or not path.is_file(): return None codepoints: set[str] = set() for line in path.read_text(encoding="utf-8").splitlines(): token = _nfc(line.strip()) if not token: continue for ch in token: if not ch.isspace(): codepoints.add(_codepoint_id(ch)) return codepoints def _expected_dictionary_coverage(texts: list[str], dictionary: Any) -> dict[str, Any]: dictionary_text = str(dictionary or "") dictionary_path = Path(dictionary_text) gt_codepoints = sorted({_codepoint_id(ch) for text in texts for ch in _nfc(text) if not ch.isspace()}) dictionary_codepoints = _dictionary_codepoints(dictionary_text) if dictionary_codepoints is None: return { "dictionary": dictionary_text, "dictionary_file_sha256": None, "checked": False, "gt_codepoint_count": len(gt_codepoints), "missing_gt_codepoints": [], "missing_gt_codepoint_count": 0, "reason": "dictionary file is not readable", } missing = sorted(set(gt_codepoints) - dictionary_codepoints) return { "dictionary": dictionary_text, "dictionary_file_sha256": _hash_existing_file(dictionary_path), "checked": True, "dictionary_codepoint_count": len(dictionary_codepoints), "gt_codepoint_count": len(gt_codepoints), "missing_gt_codepoints": missing, "missing_gt_codepoint_count": len(missing), } def _check_dictionary_coverage(recorded: Any, expected: dict[str, Any], label: str) -> list[str]: if not isinstance(recorded, dict): return [f"{label} dictionary_coverage missing or not an object"] issues: list[str] = [] for key, expected_value in expected.items(): if recorded.get(key) != expected_value: issues.append(f"{label} dictionary_coverage {key} mismatch: recorded={recorded.get(key)} recomputed={expected_value}") return issues def _bool_text(value: str) -> bool | None: lowered = str(value).strip().lower() if lowered in {"true", "1", "yes", "y"}: return True if lowered in {"false", "0", "no", "n"}: return False return None def _metadata_bool_value(metadata: dict[str, Any], keys: tuple[str, ...], fallback: Any) -> bool: for key in keys: if key not in metadata: continue value = metadata[key] if isinstance(value, bool): return value if isinstance(value, str): parsed = _bool_text(value) if parsed is not None: return parsed return bool(value) return fallback is True def _metadata_signal_text(metadata: dict[str, Any]) -> str: values: list[str] = [] for key in PROVENANCE_SIGNAL_KEYS: value = metadata.get(key) if value in (None, "", [], {}): continue if isinstance(value, list): values.extend(str(item) for item in value) elif isinstance(value, dict): values.extend(f"{item_key}:{item_value}" for item_key, item_value in value.items()) else: values.append(str(value)) return " ".join(values).lower().replace("_", " ").replace("-", " ") def _metadata_contradictions_for(sample_id: str, metadata: dict[str, Any], domain: str, gt_status: str) -> list[str]: signal_text = _metadata_signal_text(metadata) if not signal_text: return [] contradictions: list[str] = [] if domain in CLAIM_SAFE_DOMAINS and any(term in signal_text for term in ("synthetic", "transliterated", "transliteration")): contradictions.append(f"{sample_id}: synthetic/transliterated provenance is declared as real domain {domain}") if gt_status == "human_verified" and any( term in signal_text for term in ("machine converted", "machine", "synthetic", "transliterated", "transliteration") ): contradictions.append(f"{sample_id}: machine/synthetic provenance is declared as human_verified GT") return contradictions def _metadata_has_any(metadata: dict[str, Any], keys: tuple[str, ...]) -> bool: for key in keys: if metadata.get(key) not in (None, "", [], {}): return True return False def _metadata_value_for_keys(metadata: dict[str, Any], keys: tuple[str, ...]) -> str: for key in keys: value = metadata.get(key) if value not in (None, "", [], {}): if isinstance(value, (list, dict)): return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) return str(value) return "" def _bbox_text(bbox: BBox | None) -> str: if bbox is None: return "" return json.dumps(bbox.to_list(), ensure_ascii=False, separators=(",", ":")) def _int_text(value: str, field: str, row_index: int, issues: list[str]) -> int | None: try: return int(value) except (TypeError, ValueError): issues.append(f"predictions.tsv row {row_index} has non-integer {field}: {value!r}") return None def _float_text(value: str, field: str, row_index: int, issues: list[str]) -> float | None: try: return float(value) except (TypeError, ValueError): issues.append(f"CSV row {row_index} has non-numeric {field}: {value!r}") return None def _check_recognizer_predictions_recompute(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] rows = _tsv_rows(score_dir / "predictions.tsv") if not rows: return ["predictions.tsv has no scored rows"] cp_edits_total = 0 cp_len_total = 0 gr_edits_total = 0 gr_len_total = 0 exact_total = 0 zero_length_gt = 0 slice_totals: dict[tuple[str, str, str, str, str], dict[str, int]] = {} required = { "sample_id", "image_path", "image_sha256", "language", "script", "domain", "gt_status", "split", "claim_eligible", "source_document", "source_page", "source_bbox", "gt_text", "prediction", "codepoint_edits", "grapheme_edits", "exact", } for row_index, row in enumerate(rows, 1): missing = sorted(required - set(row)) if missing: issues.append(f"predictions.tsv row {row_index} missing columns: {missing}") continue gt_text = row.get("gt_text") or "" prediction = row.get("prediction") or "" prediction_nfc = _nfc(prediction) gt_nfc = _nfc(gt_text) cp_edits = Levenshtein.distance(prediction_nfc, gt_nfc) cp_len = len(gt_nfc) if cp_len == 0: zero_length_gt += 1 pred_graph = _graphemes(prediction_nfc) gt_graph = _graphemes(gt_nfc) gr_edits = Levenshtein.distance(pred_graph, gt_graph) gr_len = len(gt_graph) exact = prediction_nfc == gt_nfc recorded_cp = _int_text(row.get("codepoint_edits", ""), "codepoint_edits", row_index, issues) recorded_gr = _int_text(row.get("grapheme_edits", ""), "grapheme_edits", row_index, issues) recorded_exact = _bool_text(row.get("exact", "")) sample_id = row.get("sample_id") or f"row {row_index}" if recorded_cp is not None and recorded_cp != cp_edits: issues.append(f"predictions.tsv {sample_id} codepoint_edits mismatch: recorded={recorded_cp} recomputed={cp_edits}") if recorded_gr is not None and recorded_gr != gr_edits: issues.append(f"predictions.tsv {sample_id} grapheme_edits mismatch: recorded={recorded_gr} recomputed={gr_edits}") if recorded_exact is None: issues.append(f"predictions.tsv {sample_id} exact is not boolean-like: {row.get('exact')!r}") elif recorded_exact != exact: issues.append(f"predictions.tsv {sample_id} exact mismatch: recorded={recorded_exact} recomputed={exact}") cp_edits_total += cp_edits cp_len_total += cp_len gr_edits_total += gr_edits gr_len_total += gr_len exact_total += int(exact) slice_key = ( row.get("language") or "", row.get("script") or "", row.get("domain") or "", row.get("gt_status") or "", row.get("split") or "", ) bucket = slice_totals.setdefault( slice_key, { "n_lines": 0, "matched_lines": 0, "codepoint_edits": 0, "codepoint_ref_len": 0, "grapheme_edits": 0, "grapheme_ref_len": 0, "exact": 0, }, ) bucket["n_lines"] += 1 bucket["matched_lines"] += 1 bucket["codepoint_edits"] += cp_edits bucket["codepoint_ref_len"] += cp_len bucket["grapheme_edits"] += gr_edits bucket["grapheme_ref_len"] += gr_len bucket["exact"] += int(exact) expected_values = { "n_lines": len(rows), "matched_lines": len(rows), "zero_length_gt_lines": zero_length_gt, "codepoint_edits": cp_edits_total, "codepoint_ref_len": cp_len_total, "grapheme_edits": gr_edits_total, "grapheme_ref_len": gr_len_total, } for key, expected in expected_values.items(): if metrics.get(key) != expected: issues.append(f"metrics {key} does not match recomputed predictions.tsv value: recorded={metrics.get(key)} recomputed={expected}") if zero_length_gt: _claim_blocker(issues, eval_run, "recognizer", "zero-length GT rows", "zero-length GT rows present:") expected_exact_accuracy = exact_total / max(len(rows), 1) if not _is_number(metrics.get("exact_line_accuracy")) or not _close(float(metrics["exact_line_accuracy"]), expected_exact_accuracy): issues.append( "metrics exact_line_accuracy does not match recomputed predictions.tsv value: " f"recorded={metrics.get('exact_line_accuracy')} recomputed={expected_exact_accuracy}" ) expected_cp_cer = cp_edits_total / max(cp_len_total, 1) if not _is_number(metrics.get("cer_codepoint_micro")) or not _close(float(metrics["cer_codepoint_micro"]), expected_cp_cer): issues.append( "metrics cer_codepoint_micro does not match recomputed predictions.tsv value: " f"recorded={metrics.get('cer_codepoint_micro')} recomputed={expected_cp_cer}" ) expected_gr_cer = gr_edits_total / max(gr_len_total, 1) if not _is_number(metrics.get("cer_grapheme_micro")) or not _close(float(metrics["cer_grapheme_micro"]), expected_gr_cer): issues.append( "metrics cer_grapheme_micro does not match recomputed predictions.tsv value: " f"recorded={metrics.get('cer_grapheme_micro')} recomputed={expected_gr_cer}" ) issues.extend(_check_recognizer_slice_metrics(slice_totals, metrics.get("slices"))) return issues def _check_recognizer_uncertainty(score_dir: Path, metrics: dict[str, Any]) -> list[str]: units: list[dict[str, int]] = [] issues: list[str] = [] for row_index, row in enumerate(_tsv_rows(score_dir / "predictions.tsv"), 1): prediction_nfc = _nfc(row.get("prediction") or "") gt_nfc = _nfc(row.get("gt_text") or "") pred_graph = _graphemes(prediction_nfc) gt_graph = _graphemes(gt_nfc) units.append( { "codepoint_edits": Levenshtein.distance(prediction_nfc, gt_nfc), "codepoint_ref_len": len(gt_nfc), "grapheme_edits": Levenshtein.distance(pred_graph, gt_graph), "grapheme_ref_len": len(gt_graph), "exact": int(row.get("exact") == "True"), } ) expected = _expected_uncertainty( units, method="deterministic_row_bootstrap", metrics=("cer_codepoint_micro", "cer_grapheme_micro", "exact_line_accuracy"), ) issues.extend(_check_uncertainty(metrics.get("uncertainty"), expected, "recognizer")) for index, row in enumerate(metrics.get("slices") if isinstance(metrics.get("slices"), list) else [], 1): if not isinstance(row, dict): continue slice_key = ( str(row.get("language") or ""), str(row.get("script") or ""), str(row.get("domain") or ""), str(row.get("gt_status") or ""), str(row.get("split") or ""), ) slice_units: list[dict[str, int]] = [] for source_row in _tsv_rows(score_dir / "predictions.tsv"): source_key = ( str(source_row.get("language") or ""), str(source_row.get("script") or ""), str(source_row.get("domain") or ""), str(source_row.get("gt_status") or ""), str(source_row.get("split") or ""), ) if source_key != slice_key: continue prediction_nfc = _nfc(source_row.get("prediction") or "") gt_nfc = _nfc(source_row.get("gt_text") or "") pred_graph = _graphemes(prediction_nfc) gt_graph = _graphemes(gt_nfc) slice_units.append( { "codepoint_edits": Levenshtein.distance(prediction_nfc, gt_nfc), "codepoint_ref_len": len(gt_nfc), "grapheme_edits": Levenshtein.distance(pred_graph, gt_graph), "grapheme_ref_len": len(gt_graph), "exact": int(source_row.get("exact") == "True"), } ) expected_slice = _expected_uncertainty( slice_units, method="deterministic_row_bootstrap", metrics=("cer_codepoint_micro", "cer_grapheme_micro", "exact_line_accuracy"), ) issues.extend(_check_uncertainty(row.get("uncertainty"), expected_slice, f"recognizer slices[{index}]")) return issues def _check_recognizer_script_inventory(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: inventory_path = Path(eval_run["script_inventory_file"]) if isinstance(eval_run.get("script_inventory_file"), str) and eval_run.get("script_inventory_file") else None expected, issues = _expected_script_inventory([row.get("gt_text") or "" for row in _tsv_rows(score_dir / "predictions.tsv")], inventory_path) issues.extend(_check_script_inventory(metrics.get("script_inventory"), expected, "recognizer metrics")) issues.extend(_check_script_inventory(eval_run.get("script_inventory"), expected, "recognizer eval-run")) return issues def _check_recognizer_dictionary_coverage(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: expected = _expected_dictionary_coverage( [row.get("gt_text") or "" for row in _tsv_rows(score_dir / "predictions.tsv")], eval_run.get("dictionary"), ) issues = _check_dictionary_coverage(metrics.get("dictionary_coverage"), expected, "recognizer metrics") if expected.get("checked") and expected.get("missing_gt_codepoints"): _claim_blocker(issues, eval_run, "recognizer", "GT codepoints missing from dictionary", "GT codepoints missing from dictionary:") return issues def _recognizer_confusions_from_predictions(score_dir: Path, limit: int = 200) -> dict[str, int]: counts: dict[str, int] = {} for row in _tsv_rows(score_dir / "predictions.tsv"): gt = _nfc(row.get("gt_text") or "") pred = _nfc(row.get("prediction") or "") for opcode in Levenshtein.opcodes(pred, gt): tag, i1, i2, j1, j2 = opcode if tag == "equal": continue left = pred[i1:i2] or "<eps>" right = gt[j1:j2] or "<eps>" key = f"{left}->{right}" counts[key] = counts.get(key, 0) + 1 return dict(sorted(counts.items(), key=lambda item: item[1], reverse=True)[:limit]) def _check_recognizer_confusions(score_dir: Path) -> list[str]: path = score_dir / "confusions.json" if not path.exists(): return [] try: payload = _read_json(path) except (FileNotFoundError, ValueError) as exc: return [str(exc)] if not isinstance(payload, dict): return ["confusions.json must contain a JSON object"] issues: list[str] = [] recorded: dict[str, int] = {} for key, value in payload.items(): if isinstance(value, bool) or not isinstance(value, int): issues.append(f"confusions.json {key!r} count is not an integer: {value!r}") continue recorded[str(key)] = value expected = _recognizer_confusions_from_predictions(score_dir) if recorded != expected: issues.append(f"confusions.json does not match recomputed predictions.tsv confusions: recorded={recorded} recomputed={expected}") return issues def _check_recognizer_slice_metrics(slice_totals: dict[tuple[str, str, str, str, str], dict[str, int]], slices: Any) -> list[str]: issues: list[str] = [] if not isinstance(slices, list) or not slices: return ["recognizer slice metrics are missing"] recorded_by_key: dict[tuple[str, str, str, str, str], dict[str, Any]] = {} for index, row in enumerate(slices, 1): if not isinstance(row, dict): issues.append(f"recognizer slices[{index}] is not an object") continue key = ( str(row.get("language") or ""), str(row.get("script") or ""), str(row.get("domain") or ""), str(row.get("gt_status") or ""), str(row.get("split") or ""), ) if key in recorded_by_key: issues.append(f"duplicate recognizer slice metric for {key}") recorded_by_key[key] = row expected_keys = set(slice_totals) recorded_keys = set(recorded_by_key) missing = sorted(expected_keys - recorded_keys) extra = sorted(recorded_keys - expected_keys) if missing: issues.append(f"recognizer slice metrics missing slices: {missing[:10]}") if extra: issues.append(f"recognizer slice metrics contain extra slices: {extra[:10]}") for key in sorted(expected_keys & recorded_keys): totals = slice_totals[key] recorded = recorded_by_key[key] expected_numbers: dict[str, float | int] = { "n_lines": totals["n_lines"], "matched_lines": totals["matched_lines"], "codepoint_edits": totals["codepoint_edits"], "codepoint_ref_len": totals["codepoint_ref_len"], "cer_codepoint_micro": totals["codepoint_edits"] / max(totals["codepoint_ref_len"], 1), "grapheme_edits": totals["grapheme_edits"], "grapheme_ref_len": totals["grapheme_ref_len"], "cer_grapheme_micro": totals["grapheme_edits"] / max(totals["grapheme_ref_len"], 1), "exact_line_accuracy": totals["exact"] / max(totals["n_lines"], 1), } for field, expected in expected_numbers.items(): value = recorded.get(field) if isinstance(expected, int): if value != expected: issues.append(f"recognizer slice {key} {field} mismatch: recorded={value} recomputed={expected}") elif not _is_number(value) or not _close(float(value), expected): issues.append(f"recognizer slice {key} {field} mismatch: recorded={value} recomputed={expected}") return issues def _expected_recognizer_worst_rows(score_dir: Path) -> tuple[list[dict[str, Any]], list[str]]: issues: list[str] = [] expected_rows: list[dict[str, Any]] = [] for row_index, row in enumerate(_tsv_rows(score_dir / "predictions.tsv"), 1): sample_id = row.get("sample_id") or f"row {row_index}" gt_text = row.get("gt_text") or "" prediction = row.get("prediction") or "" prediction_nfc = _nfc(prediction) gt_nfc = _nfc(gt_text) cp_edits = Levenshtein.distance(prediction_nfc, gt_nfc) cp_len = len(gt_nfc) pred_graph = _graphemes(prediction_nfc) gt_graph = _graphemes(gt_nfc) gr_edits = Levenshtein.distance(pred_graph, gt_graph) gr_len = len(gt_graph) expected_rows.append( { "sample_id": sample_id, "image_path": row.get("image_path") or "", "image_sha256": row.get("image_sha256") or "", "language": row.get("language") or "", "script": row.get("script") or "", "domain": row.get("domain") or "", "gt_status": row.get("gt_status") or "", "split": row.get("split") or "", "claim_eligible": row.get("claim_eligible") or "", "source_document": row.get("source_document") or "", "source_page": row.get("source_page") or "", "source_bbox": row.get("source_bbox") or "", "gt_text": gt_text, "prediction": prediction, "prediction_score": row.get("prediction_score") or "", "cer_codepoint": cp_edits / max(cp_len, 1), "cer_grapheme": gr_edits / max(gr_len, 1), "sort_key": (gr_edits / max(gr_len, 1), gr_edits), } ) expected_rows.sort(key=lambda row: row["sort_key"], reverse=True) return expected_rows, issues def _check_sample_id_order(label: str, expected_rows: list[dict[str, Any]], recorded_rows: list[dict[str, str]]) -> list[str]: if expected_rows and not recorded_rows: return [f"{label} has no review rows"] if len(recorded_rows) > len(expected_rows): return [f"{label} has more rows than scored examples: recorded={len(recorded_rows)} scored={len(expected_rows)}"] expected_ids = [str(row["sample_id"]) for row in expected_rows[: len(recorded_rows)]] recorded_ids = [row.get("sample_id") or "" for row in recorded_rows] if recorded_ids != expected_ids: return [f"{label} sample order does not match recomputed worst examples: recorded={recorded_ids} recomputed={expected_ids}"] return [] def _check_review_csv_fields( label: str, expected_rows: list[dict[str, Any]], recorded_rows: list[dict[str, str]], fields: tuple[str, ...], ) -> list[str]: issues: list[str] = [] expected_by_id = {str(row["sample_id"]): row for row in expected_rows} for row_index, row in enumerate(recorded_rows, 1): sample_id = row.get("sample_id") or f"row {row_index}" expected = expected_by_id.get(sample_id) if expected is None: continue for field in fields: expected_value = expected.get(field) if isinstance(expected_value, float): recorded = _float_text(row.get(field, ""), field, row_index, issues) if recorded is not None and not _close(recorded, expected_value): issues.append(f"{label} {sample_id} {field} mismatch: recorded={recorded} recomputed={expected_value}") elif (row.get(field) or "") != str(expected_value or ""): issues.append(f"{label} {sample_id} {field} mismatch") return issues def _check_review_html_contents( score_dir: Path, label: str, expected_rows: list[dict[str, Any]], review_rows: list[dict[str, str]], fields: tuple[str, ...], ) -> list[str]: path = score_dir / "sample-review.html" if not path.exists() or not review_rows: return [] try: content = path.read_text(encoding="utf-8") except OSError as exc: return [f"sample-review.html could not be read: {exc}"] issues: list[str] = [] for row in expected_rows[: len(review_rows)]: sample_id = str(row["sample_id"]) missing_fields: list[str] = [] for field in fields: value = str(row.get(field) or "") if value and html.escape(value) not in content: missing_fields.append(field) if missing_fields: issues.append(f"sample-review.html missing {label} review sample {sample_id} fields: {missing_fields}") return issues def _check_recognizer_worst_artifacts(score_dir: Path) -> list[str]: issues: list[str] = [] expected_rows, row_issues = _expected_recognizer_worst_rows(score_dir) issues.extend(row_issues) worst_rows = _csv_rows(score_dir / "worst-lines.csv") review_rows = _csv_rows(score_dir / "sample-review.csv") issues.extend(_check_sample_id_order("worst-lines.csv", expected_rows, worst_rows)) issues.extend(_check_sample_id_order("sample-review.csv", expected_rows, review_rows)) issues.extend(_check_review_csv_fields("sample-review.csv", expected_rows, review_rows, ("image_path", "gt_text", "prediction", "prediction_score"))) issues.extend( _check_review_html_contents( score_dir, "recognizer", expected_rows, review_rows, ("sample_id", "image_path", "gt_text", "prediction"), ) ) expected_by_id = {str(row["sample_id"]): row for row in expected_rows} for row_index, row in enumerate(worst_rows, 1): sample_id = row.get("sample_id") or f"row {row_index}" expected = expected_by_id.get(sample_id) if expected is None: continue for field in ("cer_grapheme", "cer_codepoint"): recorded = _float_text(row.get(field, ""), field, row_index, issues) if recorded is not None and not _close(recorded, float(expected[field])): issues.append(f"worst-lines.csv {sample_id} {field} mismatch: recorded={recorded} recomputed={expected[field]}") for field in ( "image_path", "image_sha256", "language", "script", "domain", "gt_status", "split", "claim_eligible", "source_document", "source_page", "source_bbox", "gt_text", "prediction", "prediction_score", ): if (row.get(field) or "") != str(expected[field]): issues.append(f"worst-lines.csv {sample_id} {field} mismatch") return issues def _check_recognizer_output_duplicates(score_dir: Path) -> list[str]: issues: list[str] = [] predictions = _tsv_rows(score_dir / "predictions.tsv") worst_rows = _csv_rows(score_dir / "worst-lines.csv") review_rows = _csv_rows(score_dir / "sample-review.csv") unmatched_rows = _tsv_rows(score_dir / "unmatched.tsv") issues.extend(_check_unique_field("predictions.tsv", predictions, "sample_id")) issues.extend(_check_unique_field("predictions.tsv", predictions, "image_path")) issues.extend(_check_unique_field("worst-lines.csv", worst_rows, "sample_id")) issues.extend(_check_unique_field("sample-review.csv", review_rows, "sample_id")) issues.extend(_check_unique_key("unmatched.tsv", unmatched_rows, ("kind", "image_path"))) return issues def _read_recognizer_label_source(path: Path) -> tuple[list[dict[str, str]], list[str]]: issues: list[str] = [] rows: list[dict[str, str]] = [] seen: set[str] = set() if not path.exists(): return rows, [f"recognizer label source does not exist for unmatched audit: {path}"] with path.open("r", encoding="utf-8", newline="") as handle: for line_no, line in enumerate(handle, 1): line = line.rstrip("\n") if not line: continue if "\t" not in line: issues.append(f"recognizer label source {path}:{line_no} is not tab-separated") continue image_path, text = line.split("\t", 1) image_key = Path(image_path).name if image_key in seen: issues.append(f"recognizer label source duplicate image basename: {image_key}") seen.add(image_key) rows.append({"image_key": image_key, "image_path": image_path, "text": text}) return rows, issues def _read_recognizer_prediction_source(path: Path) -> tuple[list[dict[str, str]], list[str]]: issues: list[str] = [] rows: list[dict[str, str]] = [] seen: set[str] = set() if not path.exists(): return rows, [f"recognizer prediction source does not exist for unmatched audit: {path}"] with path.open("r", encoding="utf-8", newline="") as handle: for line_no, line in enumerate(handle, 1): line = line.rstrip("\n") if not line: continue parts = line.split("\t") if len(parts) < 2: issues.append(f"recognizer prediction source {path}:{line_no} is not tab-separated") continue image_path = parts[0] image_key = Path(image_path).name if image_key in seen: issues.append(f"recognizer prediction source duplicate image basename: {image_key}") seen.add(image_key) rows.append({"image_key": image_key, "image_path": image_path, "text": parts[1], "score": parts[2] if len(parts) > 2 else ""}) return rows, issues def _read_recognizer_manifest_source(path_value: Any) -> tuple[dict[str, dict[str, Any]], list[str]]: if not isinstance(path_value, str) or not path_value: return {}, [] path = Path(path_value) if not path.exists(): return {}, [f"recognizer manifest source does not exist for predictions.tsv audit: {path}"] rows: dict[str, dict[str, Any]] = {} seen_ids: set[str] = set() issues: list[str] = [] with path.open("r", encoding="utf-8") as handle: for line_no, line in enumerate(handle, 1): line = line.strip() if not line: continue try: payload = json.loads(line) except json.JSONDecodeError as exc: issues.append(f"recognizer manifest source {path}:{line_no} invalid JSON: {exc}") continue if not isinstance(payload, dict): issues.append(f"recognizer manifest source {path}:{line_no} row is not an object") continue sample_id = str(payload.get("sample_id") or "") if sample_id: if sample_id in seen_ids: issues.append(f"recognizer manifest source duplicate sample_id: {sample_id}") seen_ids.add(sample_id) rows[sample_id] = payload image_path = str(payload.get("image_path") or payload.get("image") or "") if image_path: rows[image_path] = payload rows[Path(image_path).name] = payload return rows, issues def _recognizer_metadata_for(label: dict[str, str], manifest: dict[str, dict[str, Any]]) -> dict[str, Any]: payload = manifest.get(label["image_path"]) or manifest.get(label["image_key"]) or {} metadata = dict(payload.get("metadata") or {}) for key, value in payload.items(): if key == "metadata": continue if key not in metadata: metadata[key] = value for key in ("sample_id", "dataset", "split", "image_path", "sha256"): if key in payload and key not in metadata: metadata[key] = payload[key] return metadata def _recognizer_eval_run_fallback(eval_run: dict[str, Any], key: str) -> str: values = eval_run.get(key) if isinstance(values, list) and len(values) == 1: return str(values[0]) return "" def _recognizer_domain(metadata: dict[str, Any], fallback: str) -> str: for key in ("domain", "eval_domain"): if metadata.get(key): return str(metadata[key]) slices = metadata.get("slices") if isinstance(slices, list): if "synthetic" in slices: return "synthetic" if "real" in slices: return fallback return fallback def _recognizer_gt_status(metadata: dict[str, Any], fallback: str) -> str: for key in ("gt_status", "label_status"): if metadata.get(key): return str(metadata[key]) provenance = str(metadata.get("label_provenance") or "").lower() if "human" in provenance and "pending" not in provenance: return "human_verified" return fallback def _expected_recognizer_prediction_rows(eval_run: dict[str, Any]) -> tuple[list[dict[str, str]], list[str]]: label_file = eval_run.get("label_file") prediction_file = eval_run.get("prediction_file") data_dir = eval_run.get("data_dir") if not isinstance(label_file, str) or not label_file: return [], ["eval-run.json missing label_file for predictions.tsv source audit"] if not isinstance(prediction_file, str) or not prediction_file: return [], ["eval-run.json missing prediction_file for predictions.tsv source audit"] if not isinstance(data_dir, str) or not data_dir: return [], ["eval-run.json missing data_dir for predictions.tsv source audit"] labels, label_issues = _read_recognizer_label_source(Path(label_file)) predictions, prediction_issues = _read_recognizer_prediction_source(Path(prediction_file)) manifest, manifest_issues = _read_recognizer_manifest_source(eval_run.get("manifest_file")) issues = [*label_issues, *prediction_issues, *manifest_issues] if issues: return [], issues prediction_by_key = {row["image_key"]: row for row in predictions} language_fallback = _recognizer_eval_run_fallback(eval_run, "languages") script_fallback = _recognizer_eval_run_fallback(eval_run, "scripts") domain_fallback = _recognizer_eval_run_fallback(eval_run, "domains") gt_status_fallback = _recognizer_eval_run_fallback(eval_run, "gt_statuses") split_fallback = _recognizer_eval_run_fallback(eval_run, "splits") expected: list[dict[str, str]] = [] seen_sample_ids: set[str] = set() for index, label in enumerate(labels): prediction = prediction_by_key.get(label["image_key"]) if prediction is None: continue metadata = _recognizer_metadata_for(label, manifest) sample_id = str(metadata.get("sample_id") or f"row-{index:06d}") if sample_id in seen_sample_ids: issues.append(f"recognizer source rows duplicate resolved sample_id: {sample_id}") seen_sample_ids.add(sample_id) image_path = Path(data_dir) / label["image_path"] image_sha256 = _sha256_file(image_path) if image_path.exists() else "" declared_image_sha256 = str(metadata.get("sha256") or metadata.get("image_sha256") or "") if declared_image_sha256 and declared_image_sha256 != image_sha256: issues.append( f"recognizer manifest {sample_id} sha256 mismatch: " f"recorded={declared_image_sha256} actual={image_sha256}" ) text_sha256 = _audit_hash_text(label["text"]) declared_text_sha256 = str(metadata.get("text_sha256") or "") if declared_text_sha256 and declared_text_sha256 != text_sha256: issues.append( f"recognizer manifest {sample_id} text_sha256 mismatch: " f"recorded={declared_text_sha256} actual={text_sha256}" ) declared_sample_sha256 = str(metadata.get("sample_sha256") or "") if declared_sample_sha256: actual_sample_sha256 = _audit_sample_hash(image_sha256, text_sha256) if declared_sample_sha256 != actual_sample_sha256: issues.append( f"recognizer manifest {sample_id} sample_sha256 mismatch: " f"recorded={declared_sample_sha256} actual={actual_sample_sha256}" ) expected.append( { "sample_id": sample_id, "image_path": label["image_path"], "image_sha256": image_sha256, "language": str(metadata.get("language") or language_fallback), "script": str(metadata.get("script") or script_fallback), "domain": _recognizer_domain(metadata, domain_fallback), "gt_status": _recognizer_gt_status(metadata, gt_status_fallback), "split": str(metadata.get("split") or metadata.get("split_role") or split_fallback), "claim_eligible": str(_metadata_bool_value(metadata, CLAIM_ELIGIBLE_KEYS, eval_run.get("claim_eligible_default"))), "source_document": _metadata_value_for_keys(metadata, SOURCE_DOCUMENT_KEYS), "source_page": _metadata_value_for_keys(metadata, PAGE_KEYS), "source_bbox": _metadata_value_for_keys(metadata, BBOX_KEYS), "gt_text": label["text"], "prediction": prediction["text"], "prediction_score": prediction["score"], } ) return expected, issues def _recognizer_pack_content_digest(rows: list[dict[str, str]]) -> str: canonical_rows = [ { "sample_id": row.get("sample_id") or "", "image_sha256": row.get("image_sha256") or "", "gt_text_sha256_nfc": _audit_hash_text(row.get("gt_text") or ""), "language": row.get("language") or "", "script": row.get("script") or "", "domain": row.get("domain") or "", "gt_status": row.get("gt_status") or "", "split": row.get("split") or "", "claim_eligible": _bool_text(row.get("claim_eligible") or ""), } for row in rows ] payload = "\n".join( json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) for row in sorted(canonical_rows, key=lambda item: item["sample_id"]) ) return hashlib.sha256(payload.encode("utf-8")).hexdigest() def _summary_pack_content_sha256(summary: dict[str, Any]) -> str: value = summary.get("pack_content_sha256") if isinstance(value, str) and value: return value eval_pack = summary.get("eval_pack") if isinstance(eval_pack, dict): nested = eval_pack.get("pack_content_sha256") if isinstance(nested, str) and nested: return nested return "" def _check_recognizer_pack_content_digest(eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: expected_rows, issues = _expected_recognizer_prediction_rows(eval_run) if issues: return issues expected = _recognizer_pack_content_digest(expected_rows) for label, payload in (("eval-run.json", eval_run), ("metrics.json", metrics)): recorded = payload.get("pack_content_sha256") if recorded != expected: issues.append(f"{label} pack_content_sha256 mismatch: recorded={recorded!r} recomputed={expected}") summary, summary_issues = _read_summary_for_audit(eval_run, "recognizer", "pack content digest audit") issues.extend(summary_issues) if summary: summary_digest = _summary_pack_content_sha256(summary) if not summary_digest: issues.append("recognizer summary missing pack_content_sha256") elif summary_digest != expected: issues.append(f"recognizer summary pack_content_sha256 mismatch: recorded={summary_digest} recomputed={expected}") return issues def _check_recognizer_prediction_source_rows(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: recorded = _tsv_rows(score_dir / "predictions.tsv") expected, issues = _expected_recognizer_prediction_rows(eval_run) if issues: return issues if len(recorded) != len(expected): issues.append(f"predictions.tsv row count does not match source matched rows: recorded={len(recorded)} recomputed={len(expected)}") for index in range(min(len(recorded), len(expected))): recorded_row = recorded[index] expected_row = expected[index] sample_id = recorded_row.get("sample_id") or expected_row["sample_id"] or f"row {index + 1}" for field in ( "sample_id", "image_path", "image_sha256", "language", "script", "domain", "gt_status", "split", "claim_eligible", "source_document", "source_page", "source_bbox", "gt_text", "prediction", "prediction_score", ): if (recorded_row.get(field) or "") != expected_row[field]: issues.append(f"predictions.tsv {sample_id} {field} does not match source files") return issues def _check_recognizer_metadata_contradictions(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: manifest, manifest_issues = _read_recognizer_manifest_source(eval_run.get("manifest_file")) if manifest_issues: return manifest_issues expected: list[str] = [] for row in _tsv_rows(score_dir / "predictions.tsv"): image_path = row.get("image_path") or "" label = {"image_path": image_path, "image_key": Path(image_path).name} metadata = _recognizer_metadata_for(label, manifest) sample_id = row.get("sample_id") or str(metadata.get("sample_id") or "") expected.extend( _metadata_contradictions_for( sample_id, metadata, row.get("domain") or "", row.get("gt_status") or "", ) ) recorded = metrics.get("metadata_contradictions") if recorded != expected: return [f"recognizer metadata_contradictions do not match source metadata: recorded={recorded} recomputed={expected}"] return [] def _check_recognizer_provenance_coverage(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: manifest, manifest_issues = _read_recognizer_manifest_source(eval_run.get("manifest_file")) if manifest_issues: return manifest_issues rows = _tsv_rows(score_dir / "predictions.tsv") expected = { "missing_source_document": 0, "missing_page": 0, "missing_bbox": 0, "missing_claim_eligibility": 0, "missing_human_review_evidence": 0, } for row in rows: image_path = row.get("image_path") or "" label = {"image_path": image_path, "image_key": Path(image_path).name} metadata = _recognizer_metadata_for(label, manifest) if not _metadata_has_any(metadata, SOURCE_DOCUMENT_KEYS): expected["missing_source_document"] += 1 if not _metadata_has_any(metadata, PAGE_KEYS): expected["missing_page"] += 1 if not _metadata_has_any(metadata, BBOX_KEYS): expected["missing_bbox"] += 1 if not any(key in metadata for key in CLAIM_ELIGIBLE_KEYS): expected["missing_claim_eligibility"] += 1 if _recognizer_gt_status(metadata, "") == "human_verified" and not _metadata_has_any(metadata, HUMAN_REVIEW_EVIDENCE_KEYS): expected["missing_human_review_evidence"] += 1 recorded = metrics.get("provenance_coverage") if recorded != expected: return [f"recognizer provenance_coverage does not match source metadata: recorded={recorded} recomputed={expected}"] return [] def _recognizer_eval_leakage_rows(score_dir: Path) -> list[dict[str, str]]: rows: list[dict[str, str]] = [] for row in _tsv_rows(score_dir / "predictions.tsv"): image_sha256 = row.get("image_sha256") or "" text_sha256 = _audit_hash_text(row.get("gt_text") or "") rows.append( { "sample_id": row.get("sample_id") or "", "source_sample_id": row.get("source_sample_id") or row.get("sample_id") or "", "image_path": row.get("image_path") or "", "image_sha256": image_sha256, "text_sha256": text_sha256, "sample_sha256": _audit_sample_hash(image_sha256, text_sha256) if image_sha256 and text_sha256 else "", } ) return rows def _check_recognizer_leakage_report(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: train_paths, issues = _train_manifest_paths(eval_run) train_rows: list[dict[str, str]] = [] for path in train_paths: rows, row_issues = _audit_manifest_leakage_rows(path, compute_missing_image_sha=False) train_rows.extend(rows) issues.extend(row_issues) if issues: return issues eval_rows = _recognizer_eval_leakage_rows(score_dir) if train_rows else [] expected = { "train_manifests": [str(path) for path in train_paths], "checked": bool(train_rows), "train_rows": len(train_rows), "eval_rows": len(eval_rows), "overlaps": _leakage_overlaps(train_rows, eval_rows), } return _compare_leakage_report("recognizer", metrics.get("leakage"), expected) def _check_recognizer_unmatched_artifact(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: path = score_dir / "unmatched.tsv" if not path.exists(): return [] label_file = eval_run.get("label_file") prediction_file = eval_run.get("prediction_file") data_dir = eval_run.get("data_dir") if not isinstance(label_file, str) or not label_file: return ["eval-run.json missing label_file for unmatched.tsv audit"] if not isinstance(prediction_file, str) or not prediction_file: return ["eval-run.json missing prediction_file for unmatched.tsv audit"] if not isinstance(data_dir, str) or not data_dir: return ["eval-run.json missing data_dir for unmatched.tsv audit"] labels, label_issues = _read_recognizer_label_source(Path(label_file)) predictions, prediction_issues = _read_recognizer_prediction_source(Path(prediction_file)) issues = [*label_issues, *prediction_issues] if issues: return issues prediction_by_key = {row["image_key"]: row for row in predictions} label_keys = {row["image_key"] for row in labels} expected = [ { "kind": "label_without_prediction", "image_key": row["image_key"], "image_path": row["image_path"], "image_sha256": _sha256_file(Path(data_dir) / row["image_path"]) if (Path(data_dir) / row["image_path"]).exists() else "", "text": row["text"], "prediction_score": "", } for row in labels if row["image_key"] not in prediction_by_key ] expected.extend( { "kind": "prediction_without_label", "image_key": row["image_key"], "image_path": row["image_path"], "image_sha256": "", "text": row["text"], "prediction_score": row["score"], } for row in predictions if row["image_key"] not in label_keys ) fields = ["kind", "image_key", "image_path", "image_sha256", "text", "prediction_score"] recorded = [{field: row.get(field) or "" for field in fields} for row in _tsv_rows(path)] if recorded != expected: issues.append( "unmatched.tsv rows do not match recomputed label/prediction misses: " f"recorded={_short_rows(recorded)} recomputed={_short_rows(expected)}" ) return issues def _check_recognizer_match_count_metrics(metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: label_file = eval_run.get("label_file") prediction_file = eval_run.get("prediction_file") if not isinstance(label_file, str) or not label_file: return ["eval-run.json missing label_file for recognizer match-count audit"] if not isinstance(prediction_file, str) or not prediction_file: return ["eval-run.json missing prediction_file for recognizer match-count audit"] labels, label_issues = _read_recognizer_label_source(Path(label_file)) predictions, prediction_issues = _read_recognizer_prediction_source(Path(prediction_file)) issues = [*label_issues, *prediction_issues] if issues: return issues label_keys = {row["image_key"] for row in labels} prediction_keys = {row["image_key"] for row in predictions} matched = len(label_keys & prediction_keys) expected_counts = { "label_lines": len(labels), "n_lines": matched, "matched_lines": matched, "unmatched_labels": len(label_keys - prediction_keys), "unmatched_predictions": len(prediction_keys - label_keys), } for field, expected in expected_counts.items(): recorded = metrics.get(field) if not _is_number(recorded): issues.append(f"metrics missing numeric field: {field}") elif float(recorded) != float(expected): issues.append( f"metrics {field} does not match recomputed recognizer source value: " f"recorded={recorded} recomputed={expected}" ) expected_match_rate = matched / max(len(labels), 1) recorded_match_rate = metrics.get("prediction_match_rate") if not _is_number(recorded_match_rate): issues.append("metrics missing numeric field: prediction_match_rate") elif not _close(float(recorded_match_rate), expected_match_rate): issues.append( "metrics prediction_match_rate does not match recomputed recognizer source value: " f"recorded={recorded_match_rate} recomputed={expected_match_rate}" ) return issues def _check_recognizer_row_hashes(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] data_dir = eval_run.get("data_dir") if not isinstance(data_dir, str) or not data_dir: return ["recognizer data_dir is missing for row hash verification"] for row_index, row in enumerate(_tsv_rows(score_dir / "predictions.tsv"), 1): sample_id = row.get("sample_id") or f"row {row_index}" image_path = row.get("image_path") expected = row.get("image_sha256") if not image_path: issues.append(f"predictions.tsv {sample_id} missing image_path") continue if not expected: issues.append(f"predictions.tsv {sample_id} missing image_sha256") continue path = Path(data_dir) / image_path if not path.exists(): issues.append(f"predictions.tsv {sample_id} image path does not exist: {path}") continue actual = _sha256_file(path) if actual != expected: issues.append(f"predictions.tsv {sample_id} image_sha256 mismatch: recorded={expected} actual={actual}") return issues def _detection_values(predicted_count: int, reference_count: int, matches: int) -> dict[str, float | int]: precision = matches / predicted_count if predicted_count else (1.0 if reference_count == 0 else 0.0) recall = matches / reference_count if reference_count else (1.0 if predicted_count == 0 else 0.0) f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 return { "line_reference_count": reference_count, "line_prediction_count": predicted_count, "line_match_count": matches, "line_detection_precision": precision, "line_detection_recall": recall, "line_detection_f1": f1, } def _audit_reference_payload(input_path: Path, metadata: dict[str, Any], manifest_dir: Path) -> tuple[dict[str, Any] | None, list[str]]: issues: list[str] = [] explicit = metadata.get("reference_path") candidates: list[Path] = [] if isinstance(explicit, str) and explicit: candidates.append(_resolve_audit_manifest_path(explicit, manifest_dir)) candidates.extend([input_path.with_suffix(input_path.suffix + ".ref.json"), input_path.with_suffix(".ref.json")]) for candidate in candidates: if not candidate.exists() or candidate.suffix.lower() != ".json": continue try: payload = json.loads(candidate.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: issues.append(f"reference JSON is invalid for line-alignment audit: {candidate}: {exc}") return None, issues if not isinstance(payload, dict): issues.append(f"reference JSON must contain an object for line-alignment audit: {candidate}") return None, issues return payload, issues return None, issues def _audit_reference_lines(payload: dict[str, Any] | None) -> list[dict[str, Any]]: if payload is None: return [] candidates = payload.get("reading_order") if candidates is None: candidates = payload.get("lines") if not isinstance(candidates, list): return [] rows: list[dict[str, Any]] = [] for index, item in enumerate(candidates): if not isinstance(item, dict): continue raw_bbox = item.get("bbox") if raw_bbox is None: continue try: bbox = BBox.from_any(raw_bbox) except Exception: continue if bbox.w <= 0 or bbox.h <= 0: continue rows.append( { "line_id": str(item.get("id") or item.get("line_id") or f"ref-{index:04d}"), "text": str(item.get("text") or ""), "bbox": bbox, "page_index": int(item.get("page_index", 0)), } ) return rows def _audit_predicted_lines(document_path: Path) -> tuple[list[TextLine], list[str]]: try: payload = json.loads(document_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return [], [f"document JSON is invalid for line-alignment audit: {document_path}: {exc}"] if not isinstance(payload, dict): return [], [f"document JSON must contain an object for line-alignment audit: {document_path}"] try: document = Document.from_dict(payload) except Exception as exc: return [], [f"document JSON failed schema parse for line-alignment audit: {document_path}: {exc}"] predicted: list[TextLine] = [] for page in document.pages: predicted.extend(page.text_lines) return predicted, [] def _codepoint_cer(prediction: str, reference: str) -> float: pred_nfc = _nfc(prediction) ref_nfc = _nfc(reference) return Levenshtein.distance(pred_nfc, ref_nfc) / max(len(ref_nfc), 1) def _text_micro_counts(prediction: str, reference: str) -> tuple[int, int, int, int]: pred_nfc = _nfc(prediction) ref_nfc = _nfc(reference) pred_graph = _graphemes(pred_nfc) ref_graph = _graphemes(ref_nfc) return ( Levenshtein.distance(pred_nfc, ref_nfc), len(ref_nfc), Levenshtein.distance(pred_graph, ref_graph), len(ref_graph), ) def _audit_document_text(document_path: Path) -> tuple[str | None, list[str]]: try: payload = json.loads(document_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return None, [f"document JSON is invalid for text metric audit: {document_path}: {exc}"] if not isinstance(payload, dict): return None, [f"document JSON must contain an object for text metric audit: {document_path}"] try: document = Document.from_dict(payload) except Exception as exc: return None, [f"document JSON failed schema parse for text metric audit: {document_path}: {exc}"] return document.text, [] def _audit_document_source_path(document_path: Path) -> tuple[str | None, list[str]]: try: payload = json.loads(document_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return None, [f"document JSON is invalid for source-path audit: {document_path}: {exc}"] if not isinstance(payload, dict): return None, [f"document JSON must contain an object for source-path audit: {document_path}"] try: document = Document.from_dict(payload) except Exception as exc: return None, [f"document JSON failed schema parse for source-path audit: {document_path}: {exc}"] return document.source_path, [] def _audit_reference_text(input_path: Path, metadata: dict[str, Any], manifest_dir: Path) -> tuple[str | None, list[str]]: explicit = metadata.get("reference_path") explicit_path = _resolve_audit_manifest_path(explicit, manifest_dir) if isinstance(explicit, str) and explicit else None try: reference = load_reference(input_path, explicit_path=explicit_path) except Exception as exc: return None, [f"reference failed to parse for text metric audit: {input_path}: {exc}"] if reference is None: return None, [f"reference is missing for text metric audit: {input_path}"] return reference.text, [] def _audit_reference_path(input_path: Path, metadata: dict[str, Any], manifest_dir: Path) -> Path | None: explicit = metadata.get("reference_path") candidates: list[Path] = [] if isinstance(explicit, str) and explicit: candidates.append(_resolve_audit_manifest_path(explicit, manifest_dir)) candidates.extend( [ input_path.with_suffix(input_path.suffix + ".ref.json"), input_path.with_suffix(".ref.json"), input_path.with_suffix(input_path.suffix + ".ref.txt"), input_path.with_suffix(".ref.txt"), ] ) for candidate in candidates: if candidate.exists(): return candidate return None def _text_sha256(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() def _expected_page_reference_artifact_rows(eval_run: dict[str, Any]) -> tuple[list[dict[str, str]], list[str]]: issues: list[str] = [] eval_manifest = eval_run.get("eval_manifest") if not isinstance(eval_manifest, str) or not eval_manifest: return [], ["page-eval-run.json missing eval_manifest for reference-artifacts.tsv audit"] manifest_path = Path(eval_manifest) manifest_rows, manifest_issues = _read_page_manifest_source(manifest_path) issues.extend(manifest_issues) manifest_dir = manifest_path.parent expected: list[dict[str, str]] = [] for manifest_row in manifest_rows: sample_id = str(manifest_row["sample_id"]) metadata = manifest_row["metadata"] input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) reference_path = _audit_reference_path(input_path, metadata, manifest_dir) if reference_path is None: continue try: reference = load_reference(input_path, explicit_path=reference_path) except Exception as exc: issues.append(f"reference failed to parse for reference-artifacts.tsv audit: {input_path}: {exc}") continue if reference is None: continue fields, field_issues = _audit_reference_fields(input_path, metadata, manifest_dir) issues.extend(field_issues) if field_issues: continue text = reference.text or "" claim_value = (reference.metadata or {}).get("claim_evidence_eligible") expected.append( { "sample_id": sample_id, "input_path": str(input_path), "reference_path": str(reference_path), "reference_sha256": _sha256_file(reference_path), "reference_fields": ";".join(fields), "reference_text_sha256": _text_sha256(text), "reference_status": str((reference.metadata or {}).get("reference_status") or ""), "reference_claim_eligible": str(claim_value) if claim_value is not None else "", } ) return expected, issues def _check_page_reference_artifacts(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] recorded = _tsv_rows(score_dir / "reference-artifacts.tsv") expected, expected_issues = _expected_page_reference_artifact_rows(eval_run) issues.extend(expected_issues) if not recorded: return [*issues, "reference-artifacts.tsv has no scored reference rows"] recorded_by_id: dict[str, dict[str, str]] = {} for row_index, row in enumerate(recorded, 1): sample_id = row.get("sample_id") or f"row {row_index}" if sample_id not in recorded_by_id: recorded_by_id[sample_id] = row expected_by_id = {row["sample_id"]: row for row in expected} missing = sorted(set(expected_by_id) - set(recorded_by_id)) extra = sorted(set(recorded_by_id) - set(expected_by_id)) if missing: issues.append(f"reference-artifacts.tsv missing scored reference rows: {missing[:10]}") if extra: issues.append(f"reference-artifacts.tsv has rows not backed by scored references: {extra[:10]}") issues.extend(_check_unique_field("reference-artifacts.tsv", recorded, "sample_id")) required = { "sample_id", "input_path", "reference_path", "reference_sha256", "reference_fields", "reference_text_sha256", "reference_status", "reference_claim_eligible", } for row_index, row in enumerate(recorded, 1): sample_id = row.get("sample_id") or f"row {row_index}" missing_fields = sorted(required - set(row)) if missing_fields: issues.append(f"reference-artifacts.tsv {sample_id} missing columns: {missing_fields}") continue expected_row = expected_by_id.get(sample_id) if expected_row is None: continue for field in ("input_path", "reference_path"): if not _same_path(row.get(field), Path(expected_row[field])): issues.append( f"reference-artifacts.tsv {sample_id} {field} does not match reference source: " f"recorded={row.get(field) or ''} recomputed={expected_row[field]}" ) for field in ("reference_sha256", "reference_fields", "reference_text_sha256", "reference_status", "reference_claim_eligible"): if (row.get(field) or "") != expected_row[field]: issues.append( f"reference-artifacts.tsv {sample_id} {field} does not match reference source: " f"recorded={row.get(field) or ''} recomputed={expected_row[field]}" ) return issues def _page_pack_content_digest(metric_rows: list[dict[str, str]], reference_rows: list[dict[str, str]]) -> str: references_by_id = {row["sample_id"]: row for row in reference_rows} canonical_rows: list[dict[str, Any]] = [] for row in metric_rows: reference = references_by_id.get(row["sample_id"], {}) canonical_rows.append( { "sample_id": row.get("sample_id") or "", "input_sha256": row.get("input_sha256") or "", "reference_sha256": reference.get("reference_sha256") or "", "reference_text_sha256": reference.get("reference_text_sha256") or "", "language": row.get("language") or "", "script": row.get("script") or "", "domain": row.get("domain") or "", "gt_status": row.get("gt_status") or "", "split": row.get("split") or "", "claim_eligible": _bool_text(row.get("claim_eligible") or ""), "source_document": row.get("source_document") or "", "source_page": row.get("source_page") or "", "source_bbox": row.get("source_bbox") or "", "reference_fields": reference.get("reference_fields") or row.get("reference_fields") or "", "reference_status": reference.get("reference_status") or "", "reference_claim_eligible": reference.get("reference_claim_eligible") or "", } ) payload = "\n".join( json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) for row in sorted(canonical_rows, key=lambda item: item["sample_id"]) ) return hashlib.sha256(payload.encode("utf-8")).hexdigest() def _check_page_pack_content_digest(eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: metric_rows, metric_issues = _expected_page_metric_identity_rows(eval_run) reference_rows, reference_issues = _expected_page_reference_artifact_rows(eval_run) issues = [*metric_issues, *reference_issues] if issues: return issues expected = _page_pack_content_digest(metric_rows, reference_rows) for label, payload in (("page-eval-run.json", eval_run), ("metrics.json", metrics)): recorded = payload.get("pack_content_sha256") if recorded != expected: issues.append(f"{label} pack_content_sha256 mismatch: recorded={recorded!r} recomputed={expected}") summary, summary_issues = _read_summary_for_audit(eval_run, "page", "pack content digest audit") issues.extend(summary_issues) if summary: summary_digest = _summary_pack_content_sha256(summary) if not summary_digest: issues.append("page summary missing pack_content_sha256") elif summary_digest != expected: issues.append(f"page summary pack_content_sha256 mismatch: recorded={summary_digest} recomputed={expected}") return issues def _check_page_text_pairs_artifact(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] page_rows = _csv_rows(score_dir / "page-metrics.csv") text_rows = _tsv_rows(score_dir / "text-pairs.tsv") if not page_rows: return ["page-metrics.csv has no scored rows for text-pairs.tsv audit"] if not text_rows: return ["text-pairs.tsv has no scored rows"] page_rows_by_id = {row.get("sample_id") or "": row for row in page_rows} text_rows_by_id = {row.get("sample_id") or "": row for row in text_rows} missing = sorted(set(page_rows_by_id) - set(text_rows_by_id)) extra = sorted(set(text_rows_by_id) - set(page_rows_by_id)) if missing: issues.append(f"text-pairs.tsv missing scored page rows: {missing[:10]}") if extra: issues.append(f"text-pairs.tsv has rows not present in page-metrics.csv: {extra[:10]}") issues.extend(_check_unique_field("text-pairs.tsv", text_rows, "sample_id")) manifest_rows_by_id: dict[str, dict[str, Any]] = {} manifest_dir = Path("") eval_manifest = eval_run.get("eval_manifest") if isinstance(eval_manifest, str) and eval_manifest: manifest_path = Path(eval_manifest) manifest_dir = manifest_path.parent manifest_rows, manifest_issues = _read_page_manifest_source(manifest_path) issues.extend(manifest_issues) manifest_rows_by_id = {str(row["sample_id"]): row for row in manifest_rows} else: issues.append("page-eval-run.json missing eval_manifest for text-pairs.tsv audit") required = { "sample_id", "input_path", "document_path", "prediction_text_sha256", "reference_text_sha256", "prediction_text", "reference_text", "text_codepoint_edits", "text_codepoint_ref_len", "text_grapheme_edits", "text_grapheme_ref_len", "text_exact", } for row_index, row in enumerate(text_rows, 1): sample_id = row.get("sample_id") or f"row {row_index}" missing_fields = sorted(required - set(row)) if missing_fields: issues.append(f"text-pairs.tsv {sample_id} missing columns: {missing_fields}") continue prediction_text = row.get("prediction_text") or "" reference_text = row.get("reference_text") or "" if row.get("prediction_text_sha256") != _text_sha256(prediction_text): issues.append(f"text-pairs.tsv {sample_id} prediction_text_sha256 mismatch") if row.get("reference_text_sha256") != _text_sha256(reference_text): issues.append(f"text-pairs.tsv {sample_id} reference_text_sha256 mismatch") cp_edits = _int_text(row.get("text_codepoint_edits", ""), "text_codepoint_edits", row_index, issues) cp_len = _int_text(row.get("text_codepoint_ref_len", ""), "text_codepoint_ref_len", row_index, issues) gr_edits = _int_text(row.get("text_grapheme_edits", ""), "text_grapheme_edits", row_index, issues) gr_len = _int_text(row.get("text_grapheme_ref_len", ""), "text_grapheme_ref_len", row_index, issues) exact = _bool_text(row.get("text_exact", "")) if exact is None: issues.append(f"text-pairs.tsv {sample_id} text_exact is not boolean-like: {row.get('text_exact')!r}") expected_counts = _text_micro_counts(prediction_text, reference_text) for field, recorded, expected in ( ("text_codepoint_edits", cp_edits, expected_counts[0]), ("text_codepoint_ref_len", cp_len, expected_counts[1]), ("text_grapheme_edits", gr_edits, expected_counts[2]), ("text_grapheme_ref_len", gr_len, expected_counts[3]), ): if recorded is not None and recorded != expected: issues.append(f"text-pairs.tsv {sample_id} {field} mismatch: recorded={recorded} recomputed={expected}") expected_exact = _nfc(prediction_text) == _nfc(reference_text) if exact is not None and exact != expected_exact: issues.append(f"text-pairs.tsv {sample_id} text_exact mismatch: recorded={exact} recomputed={expected_exact}") page_row = page_rows_by_id.get(sample_id) if page_row is not None: for field in ("input_path", "document_path", "text_codepoint_edits", "text_codepoint_ref_len", "text_grapheme_edits", "text_grapheme_ref_len", "text_exact"): if str(row.get(field) or "") != str(page_row.get(field) or ""): issues.append( f"text-pairs.tsv {sample_id} {field} does not match page-metrics.csv: " f"recorded={row.get(field)!r} page_metrics={page_row.get(field)!r}" ) manifest_row = manifest_rows_by_id.get(sample_id) if manifest_row is None: issues.append(f"text-pairs.tsv {sample_id} missing manifest row for source text audit") continue document_path = Path(row.get("document_path") or "") input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) document_text, document_issues = _audit_document_text(document_path) reference_source_text, reference_issues = _audit_reference_text(input_path, manifest_row["metadata"], manifest_dir) issues.extend(document_issues) issues.extend(reference_issues) if not document_issues and document_text is not None and prediction_text != document_text: issues.append(f"text-pairs.tsv {sample_id} prediction_text does not match document JSON text") if not reference_issues and reference_source_text is not None and reference_text != reference_source_text: issues.append(f"text-pairs.tsv {sample_id} reference_text does not match reference source text") return issues def _page_confusions_from_text_pairs(score_dir: Path, limit: int = 200) -> dict[str, int]: counts: dict[str, int] = {} for row in _tsv_rows(score_dir / "text-pairs.tsv"): prediction = _nfc(row.get("prediction_text") or "") reference = _nfc(row.get("reference_text") or "") for opcode in Levenshtein.opcodes(prediction, reference): tag, i1, i2, j1, j2 = opcode if tag == "equal": continue left = prediction[i1:i2] or "<eps>" right = reference[j1:j2] or "<eps>" key = f"{left}->{right}" counts[key] = counts.get(key, 0) + 1 return dict(sorted(counts.items(), key=lambda item: item[1], reverse=True)[:limit]) def _check_page_confusions(score_dir: Path) -> list[str]: path = score_dir / "confusions.json" if not path.exists(): return [] try: payload = _read_json(path) except (FileNotFoundError, ValueError) as exc: return [str(exc)] if not isinstance(payload, dict): return ["confusions.json must contain a JSON object"] issues: list[str] = [] recorded: dict[str, int] = {} for key, value in payload.items(): if isinstance(value, bool) or not isinstance(value, int): issues.append(f"confusions.json {key!r} count is not an integer: {value!r}") continue recorded[str(key)] = value expected = _page_confusions_from_text_pairs(score_dir) if recorded != expected: issues.append(f"confusions.json does not match recomputed text-pairs.tsv confusions: recorded={recorded} recomputed={expected}") return issues def _expected_line_alignment_rows(sample_id: str, reference_lines: list[dict[str, Any]], predicted_lines: list[TextLine], iou_threshold: float) -> list[dict[str, Any]]: unmatched_predicted = set(range(len(predicted_lines))) rows: list[dict[str, Any]] = [] for reference in reference_lines: best_index: int | None = None best_iou = 0.0 for predicted_index in sorted(unmatched_predicted): predicted = predicted_lines[predicted_index] if predicted.page_index != reference["page_index"]: continue overlap = bbox_iou(predicted.bbox, reference["bbox"]) if overlap > best_iou: best_iou = overlap best_index = predicted_index if best_index is None or best_iou < iou_threshold: rows.append( { "sample_id": sample_id, "kind": "reference_unmatched", "reference_line_id": reference["line_id"], "predicted_line_id": "", "reference_bbox": _bbox_text(reference["bbox"]), "predicted_bbox": "", "page_index": reference["page_index"], "iou": best_iou, "reference_text": reference["text"], "predicted_text": "", "text_cer_codepoint": 1.0 if reference["text"] else 0.0, } ) continue predicted = predicted_lines[best_index] unmatched_predicted.remove(best_index) rows.append( { "sample_id": sample_id, "kind": "matched", "reference_line_id": reference["line_id"], "predicted_line_id": predicted.line_id or "", "reference_bbox": _bbox_text(reference["bbox"]), "predicted_bbox": _bbox_text(predicted.bbox), "page_index": reference["page_index"], "iou": best_iou, "reference_text": reference["text"], "predicted_text": predicted.text, "text_cer_codepoint": _codepoint_cer(predicted.text, reference["text"]), } ) for predicted_index in sorted(unmatched_predicted): predicted = predicted_lines[predicted_index] rows.append( { "sample_id": sample_id, "kind": "prediction_unmatched", "reference_line_id": "", "predicted_line_id": predicted.line_id or "", "reference_bbox": "", "predicted_bbox": _bbox_text(predicted.bbox), "page_index": predicted.page_index, "iou": 0.0, "reference_text": "", "predicted_text": predicted.text, "text_cer_codepoint": 1.0 if predicted.text else 0.0, } ) return rows def _compare_line_alignment_rows(sample_id: str, recorded_rows: list[dict[str, str]], expected_rows: list[dict[str, Any]], issues: list[str]) -> None: if len(recorded_rows) != len(expected_rows): issues.append(f"line-alignment.csv {sample_id} row count mismatch: recorded={len(recorded_rows)} recomputed={len(expected_rows)}") comparable = min(len(recorded_rows), len(expected_rows)) for index in range(comparable): recorded = recorded_rows[index] expected = expected_rows[index] row_label = f"line-alignment.csv {sample_id} row {index + 1}" for field in ( "sample_id", "kind", "reference_line_id", "predicted_line_id", "reference_bbox", "predicted_bbox", "reference_text", "predicted_text", ): if (recorded.get(field) or "") != str(expected[field]): issues.append(f"{row_label} {field} mismatch") recorded_page = _int_text(recorded.get("page_index", ""), "page_index", index + 1, issues) if recorded_page is not None and recorded_page != int(expected["page_index"]): issues.append(f"{row_label} page_index mismatch: recorded={recorded_page} recomputed={expected['page_index']}") for field in ("iou", "text_cer_codepoint"): recorded_float = _float_text(recorded.get(field, ""), field, index + 1, issues) if recorded_float is not None and not _close(recorded_float, float(expected[field])): issues.append(f"{row_label} {field} mismatch: recorded={recorded_float} recomputed={expected[field]}") def _check_page_line_alignment_recompute(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] page_rows = _csv_rows(score_dir / "page-metrics.csv") line_rows = _csv_rows(score_dir / "line-alignment.csv") if not page_rows: return ["page-metrics.csv has no scored rows"] if not line_rows: return ["line-alignment.csv has no alignment rows"] line_rows_by_sample: dict[str, list[dict[str, str]]] = {} for row in line_rows: sample_id = row.get("sample_id") or "" if sample_id: line_rows_by_sample.setdefault(sample_id, []).append(row) page_sample_ids = {row.get("sample_id") or "" for row in page_rows} extra_line_samples = sorted(set(line_rows_by_sample) - page_sample_ids) if extra_line_samples: issues.append(f"line-alignment.csv has samples not present in page-metrics.csv: {extra_line_samples[:10]}") iou_threshold = eval_run.get("line_iou_threshold") if not _is_number(iou_threshold): issues.append(f"page-eval-run.json line_iou_threshold is not numeric: {iou_threshold!r}") iou_threshold = 0.0 manifest_rows_by_id: dict[str, dict[str, Any]] = {} eval_manifest = eval_run.get("eval_manifest") manifest_dir = Path("") if isinstance(eval_manifest, str) and eval_manifest: manifest_path = Path(eval_manifest) manifest_dir = manifest_path.parent manifest_rows, manifest_issues = _read_page_manifest_source(manifest_path) issues.extend(manifest_issues) manifest_rows_by_id = {str(row["sample_id"]): row for row in manifest_rows} else: issues.append("page-eval-run.json missing eval_manifest for line-alignment audit") metric_values: dict[str, list[float]] = {} for row_index, page_row in enumerate(page_rows, 1): sample_id = page_row.get("sample_id") or f"row {row_index}" sample_line_rows = line_rows_by_sample.get(sample_id, []) manifest_row = manifest_rows_by_id.get(sample_id) document_path_value = page_row.get("document_path") if manifest_row is None: issues.append(f"line-alignment.csv {sample_id} missing manifest row for recomputation") elif not document_path_value: issues.append(f"page-metrics.csv {sample_id} missing document_path for line-alignment recomputation") else: input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) reference_payload, reference_issues = _audit_reference_payload(input_path, manifest_row["metadata"], manifest_dir) issues.extend(reference_issues) predicted_lines, predicted_issues = _audit_predicted_lines(Path(document_path_value)) issues.extend(predicted_issues) if not reference_issues and not predicted_issues: expected_rows = _expected_line_alignment_rows( sample_id, _audit_reference_lines(reference_payload), predicted_lines, float(iou_threshold), ) _compare_line_alignment_rows(sample_id, sample_line_rows, expected_rows, issues) matched_rows = [row for row in sample_line_rows if row.get("kind") == "matched"] reference_count = sum(1 for row in sample_line_rows if row.get("kind") in {"matched", "reference_unmatched"}) prediction_count = sum(1 for row in sample_line_rows if row.get("kind") in {"matched", "prediction_unmatched"}) match_count = len(matched_rows) matched_ious: list[float] = [] for line_index, line_row in enumerate(matched_rows, 1): parsed_iou = _float_text(line_row.get("iou", ""), "iou", line_index, issues) if parsed_iou is not None: matched_ious.append(parsed_iou) recomputed = _detection_values(prediction_count, reference_count, match_count) recomputed["line_mean_iou"] = sum(matched_ious) / len(matched_ious) if matched_ious else 0.0 for field, expected in recomputed.items(): if field not in page_row: issues.append(f"page-metrics.csv {sample_id} missing {field}") continue recorded = _float_text(page_row.get(field, ""), field, row_index, issues) if recorded is not None and not _close(recorded, float(expected)): issues.append(f"page-metrics.csv {sample_id} {field} mismatch: recorded={recorded} recomputed={expected}") for field, value in page_row.items(): if field in { "sample_id", "input_path", "input_sha256", "document_path", "document_sha256", "language", "script", "domain", "gt_status", "split", "reference_fields", "text_cer_codepoint", "text_cer_grapheme", }: continue if value in (None, ""): continue try: metric_values.setdefault(field, []).append(float(value)) except ValueError: continue metric_means = metrics.get("metric_means") if not isinstance(metric_means, dict): return issues for field, values in sorted(metric_values.items()): if field not in metric_means: continue expected = sum(values) / len(values) recorded = metric_means.get(field) if not _is_number(recorded) or not _close(float(recorded), expected): issues.append(f"metrics metric_means.{field} does not match page-metrics.csv mean: recorded={recorded} recomputed={expected}") return issues def _check_page_text_metrics_recompute(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] page_rows = _csv_rows(score_dir / "page-metrics.csv") if not page_rows: return ["page-metrics.csv has no scored rows"] manifest_rows_by_id: dict[str, dict[str, Any]] = {} eval_manifest = eval_run.get("eval_manifest") manifest_dir = Path("") if isinstance(eval_manifest, str) and eval_manifest: manifest_path = Path(eval_manifest) manifest_dir = manifest_path.parent manifest_rows, manifest_issues = _read_page_manifest_source(manifest_path) issues.extend(manifest_issues) manifest_rows_by_id = {str(row["sample_id"]): row for row in manifest_rows} else: issues.append("page-eval-run.json missing eval_manifest for text metric audit") cp_edits_total = 0 cp_len_total = 0 gr_edits_total = 0 gr_len_total = 0 text_reference_pages = 0 text_exact_pages = 0 zero_length_text_references = 0 for row_index, row in enumerate(page_rows, 1): sample_id = row.get("sample_id") or f"row {row_index}" required = { "text_codepoint_edits", "text_codepoint_ref_len", "text_cer_codepoint", "text_grapheme_edits", "text_grapheme_ref_len", "text_cer_grapheme", "text_exact", } missing = sorted(required - set(row)) if missing: issues.append(f"page-metrics.csv {sample_id} missing text metric columns: {missing}") continue cp_edits = _int_text(row.get("text_codepoint_edits", ""), "text_codepoint_edits", row_index, issues) cp_len = _int_text(row.get("text_codepoint_ref_len", ""), "text_codepoint_ref_len", row_index, issues) gr_edits = _int_text(row.get("text_grapheme_edits", ""), "text_grapheme_edits", row_index, issues) gr_len = _int_text(row.get("text_grapheme_ref_len", ""), "text_grapheme_ref_len", row_index, issues) cp_cer = _float_text(row.get("text_cer_codepoint", ""), "text_cer_codepoint", row_index, issues) gr_cer = _float_text(row.get("text_cer_grapheme", ""), "text_cer_grapheme", row_index, issues) recorded_exact = _bool_text(row.get("text_exact", "")) if recorded_exact is None: issues.append(f"page-metrics.csv {sample_id} text_exact is not boolean-like: {row.get('text_exact')!r}") if cp_edits is None or cp_len is None or gr_edits is None or gr_len is None: continue expected_counts = (cp_edits, cp_len, gr_edits, gr_len) expected_exact = False has_text_reference = "text" in {field for field in (row.get("reference_fields") or "").split(";") if field} manifest_row = manifest_rows_by_id.get(sample_id) document_path_value = row.get("document_path") if manifest_row is None: issues.append(f"page-metrics.csv {sample_id} missing manifest row for text metric recomputation") elif not document_path_value: issues.append(f"page-metrics.csv {sample_id} missing document_path for text metric recomputation") else: input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) document_text, document_issues = _audit_document_text(Path(document_path_value)) reference_text, reference_issues = _audit_reference_text(input_path, manifest_row["metadata"], manifest_dir) issues.extend(document_issues) issues.extend(reference_issues) if not document_issues and not reference_issues: if reference_text is None: expected_counts = (0, 0, 0, 0) expected_exact = False has_text_reference = False else: expected_counts = _text_micro_counts(document_text or "", reference_text) expected_exact = _nfc(document_text or "") == _nfc(reference_text) has_text_reference = True if len(_nfc(reference_text)) == 0: zero_length_text_references += 1 expected_cp_edits, expected_cp_len, expected_gr_edits, expected_gr_len = expected_counts for field, recorded, expected in ( ("text_codepoint_edits", cp_edits, expected_cp_edits), ("text_codepoint_ref_len", cp_len, expected_cp_len), ("text_grapheme_edits", gr_edits, expected_gr_edits), ("text_grapheme_ref_len", gr_len, expected_gr_len), ): if recorded != expected: issues.append(f"page-metrics.csv {sample_id} {field} mismatch: recorded={recorded} recomputed={expected}") expected_cp_cer = expected_counts[0] / max(expected_counts[1], 1) expected_gr_cer = expected_counts[2] / max(expected_counts[3], 1) if cp_cer is not None and not _close(cp_cer, expected_cp_cer): issues.append(f"page-metrics.csv {sample_id} text_cer_codepoint mismatch: recorded={cp_cer} recomputed={expected_cp_cer}") if gr_cer is not None and not _close(gr_cer, expected_gr_cer): issues.append(f"page-metrics.csv {sample_id} text_cer_grapheme mismatch: recorded={gr_cer} recomputed={expected_gr_cer}") if recorded_exact is not None and recorded_exact != expected_exact: issues.append(f"page-metrics.csv {sample_id} text_exact mismatch: recorded={recorded_exact} recomputed={expected_exact}") cp_edits_total += expected_counts[0] cp_len_total += expected_counts[1] gr_edits_total += expected_counts[2] gr_len_total += expected_counts[3] if has_text_reference: text_reference_pages += 1 text_exact_pages += int(expected_exact) expected_values = { "text_codepoint_edits": cp_edits_total, "text_codepoint_ref_len": cp_len_total, "text_grapheme_edits": gr_edits_total, "text_grapheme_ref_len": gr_len_total, "text_reference_pages": text_reference_pages, "text_exact_pages": text_exact_pages, "zero_length_text_references": zero_length_text_references, } for key, expected in expected_values.items(): if metrics.get(key) != expected: issues.append(f"metrics {key} does not match page-metrics.csv sum: recorded={metrics.get(key)} recomputed={expected}") if zero_length_text_references: _claim_blocker(issues, eval_run, "page", "zero-length text references", "zero-length text references present:") expected_cp_cer = cp_edits_total / max(cp_len_total, 1) if not _is_number(metrics.get("text_cer_codepoint_micro")) or not _close(float(metrics["text_cer_codepoint_micro"]), expected_cp_cer): issues.append( "metrics text_cer_codepoint_micro does not match page-metrics.csv sum: " f"recorded={metrics.get('text_cer_codepoint_micro')} recomputed={expected_cp_cer}" ) expected_gr_cer = gr_edits_total / max(gr_len_total, 1) if not _is_number(metrics.get("text_cer_grapheme_micro")) or not _close(float(metrics["text_cer_grapheme_micro"]), expected_gr_cer): issues.append( "metrics text_cer_grapheme_micro does not match page-metrics.csv sum: " f"recorded={metrics.get('text_cer_grapheme_micro')} recomputed={expected_gr_cer}" ) expected_exact_accuracy = text_exact_pages / max(text_reference_pages, 1) if not _is_number(metrics.get("text_exact_accuracy")) or not _close(float(metrics["text_exact_accuracy"]), expected_exact_accuracy): issues.append( "metrics text_exact_accuracy does not match page-metrics.csv sum: " f"recorded={metrics.get('text_exact_accuracy')} recomputed={expected_exact_accuracy}" ) return issues def _page_uncertainty_units(rows: list[dict[str, str]]) -> tuple[list[dict[str, int]], list[str]]: issues: list[str] = [] units: list[dict[str, int]] = [] for row_index, row in enumerate(rows, 1): cp_edits = _int_text(row.get("text_codepoint_edits", ""), "text_codepoint_edits", row_index, issues) cp_len = _int_text(row.get("text_codepoint_ref_len", ""), "text_codepoint_ref_len", row_index, issues) gr_edits = _int_text(row.get("text_grapheme_edits", ""), "text_grapheme_edits", row_index, issues) gr_len = _int_text(row.get("text_grapheme_ref_len", ""), "text_grapheme_ref_len", row_index, issues) exact = _bool_text(row.get("text_exact", "")) if exact is None: issues.append(f"page-metrics.csv row {row_index} text_exact is not boolean-like: {row.get('text_exact')!r}") if cp_edits is None or cp_len is None or gr_edits is None or gr_len is None: continue units.append( { "codepoint_edits": cp_edits, "codepoint_ref_len": cp_len, "grapheme_edits": gr_edits, "grapheme_ref_len": gr_len, "exact": int(bool(exact)), "text_reference": int("text" in {field for field in (row.get("reference_fields") or "").split(";") if field}), } ) return units, issues def _check_page_uncertainty(score_dir: Path, metrics: dict[str, Any]) -> list[str]: rows = _csv_rows(score_dir / "page-metrics.csv") units, issues = _page_uncertainty_units(rows) expected = _expected_uncertainty( units, method="deterministic_page_bootstrap", metrics=("text_cer_codepoint_micro", "text_cer_grapheme_micro", "text_exact_accuracy"), ) issues.extend(_check_uncertainty(metrics.get("uncertainty"), expected, "page")) for index, row in enumerate(metrics.get("slices") if isinstance(metrics.get("slices"), list) else [], 1): if not isinstance(row, dict): continue slice_key = ( str(row.get("language") or ""), str(row.get("script") or ""), str(row.get("domain") or ""), str(row.get("gt_status") or ""), str(row.get("split") or ""), ) slice_rows = [ source_row for source_row in rows if ( str(source_row.get("language") or ""), str(source_row.get("script") or ""), str(source_row.get("domain") or ""), str(source_row.get("gt_status") or ""), str(source_row.get("split") or ""), ) == slice_key ] slice_units, slice_issues = _page_uncertainty_units(slice_rows) issues.extend(slice_issues) expected_slice = _expected_uncertainty( slice_units, method="deterministic_page_bootstrap", metrics=("text_cer_codepoint_micro", "text_cer_grapheme_micro", "text_exact_accuracy"), ) issues.extend(_check_uncertainty(row.get("uncertainty"), expected_slice, f"page slices[{index}]")) return issues def _check_page_script_inventory(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] inventory_path = Path(eval_run["script_inventory_file"]) if isinstance(eval_run.get("script_inventory_file"), str) and eval_run.get("script_inventory_file") else None manifest_rows_by_id: dict[str, dict[str, Any]] = {} eval_manifest = eval_run.get("eval_manifest") manifest_dir = Path("") if isinstance(eval_manifest, str) and eval_manifest: manifest_path = Path(eval_manifest) manifest_dir = manifest_path.parent manifest_rows, manifest_issues = _read_page_manifest_source(manifest_path) issues.extend(manifest_issues) manifest_rows_by_id = {str(row["sample_id"]): row for row in manifest_rows} else: issues.append("page-eval-run.json missing eval_manifest for script inventory audit") reference_texts: list[str] = [] for row in _csv_rows(score_dir / "page-metrics.csv"): sample_id = row.get("sample_id") or "" manifest_row = manifest_rows_by_id.get(sample_id) if manifest_row is None: issues.append(f"page-metrics.csv {sample_id or '(missing sample_id)'} missing manifest row for script inventory audit") continue input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) reference_text, reference_issues = _audit_reference_text(input_path, manifest_row["metadata"], manifest_dir) issues.extend(reference_issues) if reference_text is not None: reference_texts.append(reference_text) expected, inventory_issues = _expected_script_inventory(reference_texts, inventory_path) issues.extend(inventory_issues) issues.extend(_check_script_inventory(metrics.get("script_inventory"), expected, "page metrics")) issues.extend(_check_script_inventory(eval_run.get("script_inventory"), expected, "page eval-run")) return issues def _check_page_dictionary_coverage(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: reference_texts = [row.get("reference_text") or "" for row in _tsv_rows(score_dir / "text-pairs.tsv")] expected = _expected_dictionary_coverage(reference_texts, eval_run.get("dictionary")) issues = _check_dictionary_coverage(metrics.get("dictionary_coverage"), expected, "page metrics") if expected.get("checked") and expected.get("missing_gt_codepoints"): _claim_blocker(issues, eval_run, "page", "GT codepoints missing from dictionary", "GT codepoints missing from dictionary:") return issues def _expected_page_worst_rows(score_dir: Path) -> tuple[list[dict[str, Any]], list[str]]: issues: list[str] = [] expected_rows: list[dict[str, Any]] = [] reference_by_id = {row.get("sample_id") or "": row for row in _tsv_rows(score_dir / "reference-artifacts.tsv")} for row_index, row in enumerate(_csv_rows(score_dir / "page-metrics.csv"), 1): sample_id = row.get("sample_id") or f"row {row_index}" reference_row = reference_by_id.get(sample_id, {}) cp_edits = _int_text(row.get("text_codepoint_edits", ""), "text_codepoint_edits", row_index, issues) cp_len = _int_text(row.get("text_codepoint_ref_len", ""), "text_codepoint_ref_len", row_index, issues) gr_edits = _int_text(row.get("text_grapheme_edits", ""), "text_grapheme_edits", row_index, issues) gr_len = _int_text(row.get("text_grapheme_ref_len", ""), "text_grapheme_ref_len", row_index, issues) if cp_edits is None or cp_len is None or gr_edits is None or gr_len is None: continue expected_rows.append( { "sample_id": sample_id, "input_path": row.get("input_path") or "", "input_sha256": row.get("input_sha256") or "", "input_uri": Path(row.get("input_path") or "").resolve().as_uri() if row.get("input_path") else "", "document_path": row.get("document_path") or "", "document_sha256": row.get("document_sha256") or "", "document_source_path": row.get("document_source_path") or "", "language": row.get("language") or "", "script": row.get("script") or "", "domain": row.get("domain") or "", "gt_status": row.get("gt_status") or "", "split": row.get("split") or "", "claim_eligible": row.get("claim_eligible") or "", "source_document": row.get("source_document") or "", "source_page": row.get("source_page") or "", "source_bbox": row.get("source_bbox") or "", "reference_path": reference_row.get("reference_path") or "", "reference_sha256": reference_row.get("reference_sha256") or "", "reference_status": reference_row.get("reference_status") or "", "reference_claim_eligible": reference_row.get("reference_claim_eligible") or "", "reference_fields": row.get("reference_fields") or "", "text_cer_codepoint": cp_edits / max(cp_len, 1), "text_cer_grapheme": gr_edits / max(gr_len, 1), "sort_key": gr_edits / max(gr_len, 1), } ) expected_rows.sort(key=lambda row: row["sort_key"], reverse=True) return expected_rows, issues def _check_page_worst_artifacts(score_dir: Path) -> list[str]: issues: list[str] = [] expected_rows, row_issues = _expected_page_worst_rows(score_dir) issues.extend(row_issues) worst_rows = _csv_rows(score_dir / "worst-pages.csv") review_rows = _csv_rows(score_dir / "sample-review.csv") issues.extend(_check_sample_id_order("worst-pages.csv", expected_rows, worst_rows)) issues.extend(_check_sample_id_order("sample-review.csv", expected_rows, review_rows)) issues.extend( _check_review_csv_fields( "sample-review.csv", expected_rows, review_rows, ("input_path", "document_path", "document_source_path", "text_cer_grapheme", "text_cer_codepoint", "reference_fields"), ) ) issues.extend( _check_review_html_contents( score_dir, "page", expected_rows, review_rows, ("sample_id", "input_uri", "document_path", "document_source_path", "reference_fields"), ) ) expected_by_id = {str(row["sample_id"]): row for row in expected_rows} for row_index, row in enumerate(worst_rows, 1): sample_id = row.get("sample_id") or f"row {row_index}" expected = expected_by_id.get(sample_id) if expected is None: continue for field in ("text_cer_grapheme", "text_cer_codepoint"): recorded = _float_text(row.get(field, ""), field, row_index, issues) if recorded is not None and not _close(recorded, float(expected[field])): issues.append(f"worst-pages.csv {sample_id} {field} mismatch: recorded={recorded} recomputed={expected[field]}") for field in ( "input_path", "input_sha256", "document_path", "document_sha256", "document_source_path", "language", "script", "domain", "gt_status", "split", "claim_eligible", "source_document", "source_page", "source_bbox", "reference_path", "reference_sha256", "reference_status", "reference_claim_eligible", "reference_fields", ): if (row.get(field) or "") != str(expected[field]): issues.append(f"worst-pages.csv {sample_id} {field} mismatch") return issues def _check_page_output_duplicates(score_dir: Path) -> list[str]: issues: list[str] = [] page_rows = _csv_rows(score_dir / "page-metrics.csv") missing_rows = _csv_rows(score_dir / "missing.tsv") worst_rows = _csv_rows(score_dir / "worst-pages.csv") review_rows = _csv_rows(score_dir / "sample-review.csv") issues.extend(_check_unique_field("page-metrics.csv", page_rows, "sample_id")) issues.extend(_check_unique_field("page-metrics.csv", page_rows, "input_path")) issues.extend(_check_unique_field("page-metrics.csv", page_rows, "document_path")) issues.extend(_check_unique_key("missing.tsv", missing_rows, ("kind", "sample_id"))) issues.extend(_check_unique_field("worst-pages.csv", worst_rows, "sample_id")) issues.extend(_check_unique_field("sample-review.csv", review_rows, "sample_id")) return issues def _resolve_audit_manifest_path(path_value: str, manifest_dir: Path) -> Path: path = Path(path_value) if path.is_absolute() or path.exists(): return path candidate = manifest_dir / path if candidate.exists(): return candidate return path def _page_document_candidates(sample_id: str, input_path: Path, documents_dir: Path) -> list[Path]: return [ documents_dir / sample_id / "document.json", documents_dir / input_path.stem / "document.json", documents_dir / f"{sample_id}.json", documents_dir / f"{input_path.stem}.json", ] def _page_reference_exists(input_path: Path, metadata: dict[str, Any], manifest_dir: Path) -> bool: explicit = metadata.get("reference_path") candidates: list[Path] = [] if isinstance(explicit, str) and explicit: candidates.append(_resolve_audit_manifest_path(explicit, manifest_dir)) candidates.extend( [ input_path.with_suffix(input_path.suffix + ".ref.json"), input_path.with_suffix(".ref.json"), input_path.with_suffix(input_path.suffix + ".ref.txt"), input_path.with_suffix(".ref.txt"), ] ) return any(candidate.exists() for candidate in candidates) def _read_page_manifest_source(path: Path) -> tuple[list[dict[str, Any]], list[str]]: issues: list[str] = [] rows: list[dict[str, Any]] = [] seen: set[str] = set() seen_image_paths: set[str] = set() if not path.exists(): return rows, [f"page eval manifest does not exist for missing.tsv audit: {path}"] with path.open("r", encoding="utf-8") as handle: for line_no, line in enumerate(handle, 1): line = line.strip() if not line: continue try: payload = json.loads(line) except json.JSONDecodeError as exc: issues.append(f"page eval manifest {path}:{line_no} invalid JSON: {exc}") continue if not isinstance(payload, dict): issues.append(f"page eval manifest {path}:{line_no} row is not an object") continue sample_id = str(payload.get("sample_id") or payload.get("id") or "") image_path = str(payload.get("image_path") or payload.get("image") or "") if not sample_id: issues.append(f"page eval manifest {path}:{line_no} missing sample_id") continue if not image_path: issues.append(f"page eval manifest {path}:{line_no} missing image_path") continue if sample_id in seen: issues.append(f"page eval manifest duplicate sample_id: {sample_id}") seen.add(sample_id) if image_path in seen_image_paths: issues.append(f"page eval manifest duplicate image_path: {image_path}") seen_image_paths.add(image_path) declared_sha256 = str(payload.get("sha256") or "") if declared_sha256: resolved_image = _resolve_audit_manifest_path(image_path, path.parent) if resolved_image.exists(): actual_sha256 = _sha256_file(resolved_image) if actual_sha256 != declared_sha256: issues.append( f"page eval manifest {sample_id} sha256 mismatch: " f"recorded={declared_sha256} actual={actual_sha256}" ) rows.append( { "sample_id": sample_id, "image_path": image_path, "sha256": declared_sha256, "split": str(payload.get("split") or ""), "metadata": dict(payload.get("metadata") or {}), } ) return rows, issues def _audit_page_metadata_value(metadata: dict[str, Any], key: str, fallback: Any) -> str: value = metadata.get(key) if value in (None, ""): return str(fallback or "") return str(value) def _audit_page_split(manifest_row: dict[str, Any]) -> str: metadata = manifest_row["metadata"] value = metadata.get("split") or metadata.get("split_role") or manifest_row.get("split") or "" return str(value) def _audit_reference_fields(input_path: Path, metadata: dict[str, Any], manifest_dir: Path) -> tuple[list[str], list[str]]: issues: list[str] = [] explicit = metadata.get("reference_path") explicit_path = _resolve_audit_manifest_path(explicit, manifest_dir) if isinstance(explicit, str) and explicit else None try: reference = load_reference(input_path, explicit_path=explicit_path) except Exception as exc: return [], [f"reference failed to parse for page-metrics identity audit: {input_path}: {exc}"] if reference is None: return [], [f"reference is missing for page-metrics identity audit: {input_path}"] fields: list[str] = [] if reference.text is not None: fields.append("text") if reference.markdown is not None: fields.append("markdown") if reference.reading_order: fields.append("reading_order") if reference.tables: fields.append("tables") if reference.figures: fields.append("figures") payload, payload_issues = _audit_reference_payload(input_path, metadata, manifest_dir) issues.extend(payload_issues) if _audit_reference_lines(payload): fields.append("line_bboxes") return fields, issues def _expected_page_metric_identity_rows(eval_run: dict[str, Any]) -> tuple[list[dict[str, str]], list[str]]: issues: list[str] = [] eval_manifest = eval_run.get("eval_manifest") documents_dir_value = eval_run.get("documents_dir") if not isinstance(eval_manifest, str) or not eval_manifest: return [], ["page-eval-run.json missing eval_manifest for page-metrics identity audit"] if not isinstance(documents_dir_value, str) or not documents_dir_value: return [], ["page-eval-run.json missing documents_dir for page-metrics identity audit"] manifest_path = Path(eval_manifest) manifest_rows, manifest_issues = _read_page_manifest_source(manifest_path) issues.extend(manifest_issues) manifest_dir = manifest_path.parent documents_dir = Path(documents_dir_value) expected: list[dict[str, str]] = [] for manifest_row in manifest_rows: sample_id = str(manifest_row["sample_id"]) metadata = manifest_row["metadata"] input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) document_path = next( (candidate for candidate in _page_document_candidates(sample_id, input_path, documents_dir) if candidate.exists()), None, ) if document_path is None: continue document_source_path, document_source_issues = _audit_document_source_path(document_path) issues.extend(document_source_issues) if document_source_issues: continue reference_fields, reference_issues = _audit_reference_fields(input_path, metadata, manifest_dir) issues.extend(reference_issues) if reference_issues: continue expected.append( { "sample_id": sample_id, "input_path": str(input_path), "input_sha256": _sha256_file(input_path) if input_path.exists() else "", "document_path": str(document_path), "document_source_path": document_source_path or "", "ocr_output_kind": str(eval_run.get("ocr_output_kind") or ""), "language": _audit_page_metadata_value(metadata, "language", eval_run.get("language_default")), "script": _audit_page_metadata_value(metadata, "script", eval_run.get("script_default")), "domain": _audit_page_metadata_value(metadata, "domain", eval_run.get("domain_default")), "gt_status": _audit_page_metadata_value(metadata, "gt_status", eval_run.get("gt_status_default")), "split": _audit_page_split(manifest_row), "claim_eligible": str( _metadata_bool_value(metadata, CLAIM_ELIGIBLE_KEYS, eval_run.get("claim_eligible_default")) ), "source_document": _metadata_value_for_keys(metadata, SOURCE_DOCUMENT_KEYS), "source_page": _metadata_value_for_keys(metadata, PAGE_KEYS), "source_bbox": _metadata_value_for_keys(metadata, BBOX_KEYS), "reference_fields": ";".join(reference_fields), } ) return expected, issues def _check_page_metadata_contradictions(score_dir: Path, metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: eval_manifest = eval_run.get("eval_manifest") if not isinstance(eval_manifest, str) or not eval_manifest: return ["page-eval-run.json missing eval_manifest for metadata contradiction audit"] manifest_path = Path(eval_manifest) manifest_rows, issues = _read_page_manifest_source(manifest_path) if issues: return issues manifest_dir = manifest_path.parent page_rows = {row.get("sample_id") or "": row for row in _csv_rows(score_dir / "page-metrics.csv")} expected: list[str] = [] for manifest_row in manifest_rows: sample_id = str(manifest_row["sample_id"]) page_row = page_rows.get(sample_id) if page_row is None: continue metadata = manifest_row["metadata"] domain = page_row.get("domain") or "" gt_status = page_row.get("gt_status") or "" expected.extend(_metadata_contradictions_for(sample_id, metadata, domain, gt_status)) input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) reference_payload, reference_issues = _audit_reference_payload(input_path, metadata, manifest_dir) issues.extend(reference_issues) if isinstance(reference_payload, dict) and isinstance(reference_payload.get("metadata"), dict): expected.extend(_metadata_contradictions_for(sample_id, reference_payload["metadata"], domain, gt_status)) if issues: return issues recorded = metrics.get("metadata_contradictions") if recorded != expected: return [f"page metadata_contradictions do not match source metadata: recorded={recorded} recomputed={expected}"] return [] def _check_page_metadata_coverage(metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: eval_manifest = eval_run.get("eval_manifest") if not isinstance(eval_manifest, str) or not eval_manifest: return ["page-eval-run.json missing eval_manifest for metadata coverage audit"] manifest_rows, issues = _read_page_manifest_source(Path(eval_manifest)) if issues: return issues def missing(key: str) -> int: return sum(1 for row in manifest_rows if row["metadata"].get(key) in (None, "")) def missing_any(keys: tuple[str, ...]) -> int: return sum(1 for row in manifest_rows if not _metadata_has_any(row["metadata"], keys)) def missing_human_review_evidence() -> int: return sum( 1 for row in manifest_rows if row["metadata"].get("gt_status") == "human_verified" and not _metadata_has_any(row["metadata"], HUMAN_REVIEW_EVIDENCE_KEYS) ) expected = { "missing_source_document": missing_any(SOURCE_DOCUMENT_KEYS), "missing_page": missing_any(PAGE_KEYS), "missing_bbox": missing_any(BBOX_KEYS), "missing_claim_eligibility": missing_any(CLAIM_ELIGIBLE_KEYS), "missing_human_review_evidence": missing_human_review_evidence(), "missing_language": missing("language"), "missing_script": missing("script"), "missing_domain": missing("domain"), "missing_gt_status": missing("gt_status"), } recorded = metrics.get("metadata_coverage") if recorded != expected: return [f"page metadata_coverage does not match eval manifest: recorded={recorded} recomputed={expected}"] return [] def _check_page_reference_coverage(score_dir: Path, metrics: dict[str, Any]) -> list[str]: issues: list[str] = [] rows = _csv_rows(score_dir / "page-metrics.csv") fields = ("text", "markdown", "reading_order", "tables", "figures", "line_bboxes") expected = dict.fromkeys(fields, 0) line_alignment_scored = 0 for row_index, row in enumerate(rows, 1): reference_fields = {field for field in (row.get("reference_fields") or "").split(";") if field} for field in fields: if field in reference_fields: expected[field] += 1 raw_reference_count = row.get("line_reference_count") if raw_reference_count in (None, ""): continue reference_count = _int_text(raw_reference_count, "line_reference_count", row_index, issues) if reference_count is not None and reference_count: line_alignment_scored += 1 expected["line_alignment_scored"] = line_alignment_scored recorded = metrics.get("reference_coverage") if recorded != expected: issues.append(f"page reference_coverage does not match page-metrics.csv: recorded={recorded} recomputed={expected}") return issues def _check_page_metric_identity_rows(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] recorded_rows = _csv_rows(score_dir / "page-metrics.csv") expected_rows, expected_issues = _expected_page_metric_identity_rows(eval_run) issues.extend(expected_issues) recorded_by_id: dict[str, dict[str, str]] = {} for row_index, row in enumerate(recorded_rows, 1): sample_id = row.get("sample_id") or f"row {row_index}" if sample_id not in recorded_by_id: recorded_by_id[sample_id] = row expected_by_id = {row["sample_id"]: row for row in expected_rows} missing = sorted(set(expected_by_id) - set(recorded_by_id)) extra = sorted(set(recorded_by_id) - set(expected_by_id)) if missing: issues.append(f"page-metrics.csv missing scored manifest rows: {missing[:10]}") if extra: issues.append(f"page-metrics.csv has rows not backed by scored manifest/reference/documents: {extra[:10]}") for sample_id in sorted(set(expected_by_id) & set(recorded_by_id)): expected = expected_by_id[sample_id] recorded = recorded_by_id[sample_id] for field in ( "sample_id", "input_sha256", "language", "script", "domain", "gt_status", "split", "claim_eligible", "source_document", "source_page", "source_bbox", "reference_fields", "document_source_path", ): if (recorded.get(field) or "") != expected[field]: issues.append( f"page-metrics.csv {sample_id} {field} does not match eval manifest/reference: " f"recorded={recorded.get(field) or ''} recomputed={expected[field]}" ) for field in ("input_path", "document_path"): recorded_path = recorded.get(field) or "" if not _same_path(recorded_path, Path(expected[field])): issues.append( f"page-metrics.csv {sample_id} {field} does not match eval manifest/reference: " f"recorded={recorded_path} recomputed={expected[field]}" ) return issues def _expected_page_document_source_path_mismatches(eval_run: dict[str, Any]) -> tuple[list[str], list[str]]: eval_manifest = eval_run.get("eval_manifest") documents_dir_value = eval_run.get("documents_dir") if not isinstance(eval_manifest, str) or not eval_manifest: return [], ["page-eval-run.json missing eval_manifest for document source-path audit"] if not isinstance(documents_dir_value, str) or not documents_dir_value: return [], ["page-eval-run.json missing documents_dir for document source-path audit"] manifest_path = Path(eval_manifest) manifest_rows, issues = _read_page_manifest_source(manifest_path) manifest_dir = manifest_path.parent documents_dir = Path(documents_dir_value) mismatches: list[str] = [] for manifest_row in manifest_rows: sample_id = str(manifest_row["sample_id"]) input_path = _resolve_audit_manifest_path(str(manifest_row["image_path"]), manifest_dir) document_path = next( (candidate for candidate in _page_document_candidates(sample_id, input_path, documents_dir) if candidate.exists()), None, ) if document_path is None: continue document_source_path, document_source_issues = _audit_document_source_path(document_path) issues.extend(document_source_issues) if document_source_issues: continue if not _same_path(document_source_path or "", input_path): mismatches.append( f"{sample_id}: document.source_path {document_source_path!r} does not match eval input {str(input_path)!r}" ) return mismatches, issues def _check_page_document_source_path_mismatches(metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: expected, issues = _expected_page_document_source_path_mismatches(eval_run) if issues: return issues recorded = metrics.get("document_source_path_mismatches") if recorded != expected: issues.append(f"page document_source_path_mismatches do not match scored documents: recorded={recorded} recomputed={expected}") recorded_count = metrics.get("document_source_path_mismatch_count") if recorded_count != len(expected): issues.append( "page document_source_path_mismatch_count does not match scored documents: " f"recorded={recorded_count} recomputed={len(expected)}" ) return issues def _check_page_missing_artifact(score_dir: Path, eval_run: dict[str, Any]) -> list[str]: path = score_dir / "missing.tsv" if not path.exists(): return [] eval_manifest = eval_run.get("eval_manifest") documents_dir_value = eval_run.get("documents_dir") if not isinstance(eval_manifest, str) or not eval_manifest: return ["page-eval-run.json missing eval_manifest for missing.tsv audit"] if not isinstance(documents_dir_value, str) or not documents_dir_value: return ["page-eval-run.json missing documents_dir for missing.tsv audit"] manifest_path = Path(eval_manifest) manifest_rows, issues = _read_page_manifest_source(manifest_path) if issues: return issues manifest_dir = manifest_path.parent documents_dir = Path(documents_dir_value) expected: list[tuple[str, str]] = [] for row in manifest_rows: sample_id = str(row["sample_id"]) input_path = _resolve_audit_manifest_path(str(row["image_path"]), manifest_dir) document_exists = any(candidate.exists() for candidate in _page_document_candidates(sample_id, input_path, documents_dir)) if not document_exists: expected.append(("missing_document", sample_id)) continue if not _page_reference_exists(input_path, row["metadata"], manifest_dir): expected.append(("missing_reference", sample_id)) recorded = [(row.get("kind") or "", row.get("sample_id") or "") for row in _csv_rows(path)] if recorded != expected: issues.append( "missing.tsv rows do not match recomputed manifest/document/reference misses: " f"recorded={_short_rows(recorded)} recomputed={_short_rows(expected)}" ) return issues def _check_page_manifest_count_metrics(metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: eval_manifest = eval_run.get("eval_manifest") documents_dir_value = eval_run.get("documents_dir") if not isinstance(eval_manifest, str) or not eval_manifest: return ["page-eval-run.json missing eval_manifest for manifest count audit"] if not isinstance(documents_dir_value, str) or not documents_dir_value: return ["page-eval-run.json missing documents_dir for manifest count audit"] manifest_path = Path(eval_manifest) manifest_rows, issues = _read_page_manifest_source(manifest_path) if issues: return issues manifest_dir = manifest_path.parent documents_dir = Path(documents_dir_value) missing_documents = 0 missing_references = 0 for row in manifest_rows: sample_id = str(row["sample_id"]) input_path = _resolve_audit_manifest_path(str(row["image_path"]), manifest_dir) document_exists = any(candidate.exists() for candidate in _page_document_candidates(sample_id, input_path, documents_dir)) if not document_exists: missing_documents += 1 continue if not _page_reference_exists(input_path, row["metadata"], manifest_dir): missing_references += 1 expected = { "manifest_samples": len(manifest_rows), "missing_documents": missing_documents, "missing_references": missing_references, "claim_eligible_rows": sum( 1 for row in manifest_rows if _metadata_bool_value(row["metadata"], CLAIM_ELIGIBLE_KEYS, eval_run.get("claim_eligible_default")) ), } expected["scored_pages"] = expected["manifest_samples"] - expected["missing_documents"] - expected["missing_references"] count_issues: list[str] = [] for key, value in expected.items(): if metrics.get(key) != value: count_issues.append(f"page metrics {key} does not match manifest/doc/reference sources: recorded={metrics.get(key)} recomputed={value}") return count_issues def _check_page_leakage_report(metrics: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: eval_manifest = eval_run.get("eval_manifest") if not isinstance(eval_manifest, str) or not eval_manifest: return ["page-eval-run.json missing eval_manifest for leakage audit"] train_paths, issues = _train_manifest_paths(eval_run) eval_manifest_path = Path(eval_manifest) eval_rows, eval_issues = _audit_manifest_leakage_rows( eval_manifest_path, manifest_dir=eval_manifest_path.parent, compute_missing_image_sha=True, ) issues.extend(eval_issues) train_rows: list[dict[str, str]] = [] for path in train_paths: rows, row_issues = _audit_manifest_leakage_rows(path, manifest_dir=path.parent, compute_missing_image_sha=True) train_rows.extend(rows) issues.extend(row_issues) if issues: return issues expected = { "train_manifests": [str(path) for path in train_paths], "checked": bool(train_paths), "train_rows": len(train_rows), "eval_rows": len(eval_rows), "overlaps": _leakage_overlaps(train_rows, eval_rows), } return _compare_leakage_report("page", metrics.get("leakage"), expected) def _check_page_row_hashes(score_dir: Path) -> list[str]: issues: list[str] = [] page_rows = _csv_rows(score_dir / "page-metrics.csv") for row_index, row in enumerate(page_rows, 1): sample_id = row.get("sample_id") or f"row {row_index}" for path_field, hash_field, label in ( ("input_path", "input_sha256", "input image"), ("document_path", "document_sha256", "document output"), ): path_value = row.get(path_field) expected = row.get(hash_field) if not path_value: issues.append(f"page-metrics.csv {sample_id} missing {path_field}") continue if not expected: issues.append(f"page-metrics.csv {sample_id} missing {hash_field}") continue path = Path(path_value) if not path.exists(): issues.append(f"page-metrics.csv {sample_id} {label} path does not exist: {path_value}") continue actual = _sha256_file(path) if actual != expected: issues.append(f"page-metrics.csv {sample_id} {hash_field} mismatch: recorded={expected} actual={actual}") return issues PAGE_SLICE_IDENTITY_FIELDS = { "sample_id", "input_path", "input_sha256", "document_path", "document_sha256", "document_source_path", "ocr_output_kind", "language", "script", "domain", "gt_status", "split", "claim_eligible", "source_document", "source_page", "source_bbox", "reference_fields", } PAGE_SLICE_TEXT_FIELDS = { "text_codepoint_edits", "text_codepoint_ref_len", "text_cer_codepoint", "text_grapheme_edits", "text_grapheme_ref_len", "text_cer_grapheme", "text_exact", } def _page_metric_mean_values(row: dict[str, str], row_index: int, issues: list[str]) -> dict[str, float]: values: dict[str, float] = {} excluded = PAGE_SLICE_IDENTITY_FIELDS | PAGE_SLICE_TEXT_FIELDS for field, value in row.items(): if field in excluded or value in (None, ""): continue parsed = _float_text(value, field, row_index, issues) if parsed is not None: values[field] = parsed return values def _check_page_slice_metrics(score_dir: Path, slices: Any) -> list[str]: issues: list[str] = [] page_rows = _csv_rows(score_dir / "page-metrics.csv") if not page_rows: return ["page-metrics.csv has no scored rows"] if not isinstance(slices, list) or not slices: return ["page slice metrics are missing"] groups: dict[tuple[str, str, str, str, str], dict[str, Any]] = {} for row_index, row in enumerate(page_rows, 1): key = ( row.get("language") or "", row.get("script") or "", row.get("domain") or "", row.get("gt_status") or "", row.get("split") or "", ) bucket = groups.setdefault( key, { "scored_pages": 0, "text_codepoint_edits": 0, "text_codepoint_ref_len": 0, "text_grapheme_edits": 0, "text_grapheme_ref_len": 0, "text_reference_pages": 0, "text_exact_pages": 0, "metric_values": {}, }, ) bucket["scored_pages"] += 1 cp_edits = _int_text(row.get("text_codepoint_edits", ""), "text_codepoint_edits", row_index, issues) cp_len = _int_text(row.get("text_codepoint_ref_len", ""), "text_codepoint_ref_len", row_index, issues) gr_edits = _int_text(row.get("text_grapheme_edits", ""), "text_grapheme_edits", row_index, issues) gr_len = _int_text(row.get("text_grapheme_ref_len", ""), "text_grapheme_ref_len", row_index, issues) if cp_edits is not None: bucket["text_codepoint_edits"] += cp_edits if cp_len is not None: bucket["text_codepoint_ref_len"] += cp_len if gr_edits is not None: bucket["text_grapheme_edits"] += gr_edits if gr_len is not None: bucket["text_grapheme_ref_len"] += gr_len if "text" in {field for field in (row.get("reference_fields") or "").split(";") if field}: bucket["text_reference_pages"] += 1 exact = _bool_text(row.get("text_exact", "")) if exact is None: issues.append(f"page-metrics.csv row {row_index} text_exact is not boolean-like: {row.get('text_exact')!r}") else: bucket["text_exact_pages"] += int(exact) metric_values: dict[str, list[float]] = bucket["metric_values"] for field, value in _page_metric_mean_values(row, row_index, issues).items(): metric_values.setdefault(field, []).append(value) expected_by_key: dict[tuple[str, str, str, str, str], dict[str, Any]] = {} for key, bucket in groups.items(): cp_edits = bucket["text_codepoint_edits"] cp_len = bucket["text_codepoint_ref_len"] gr_edits = bucket["text_grapheme_edits"] gr_len = bucket["text_grapheme_ref_len"] text_reference_pages = bucket["text_reference_pages"] text_exact_pages = bucket["text_exact_pages"] metric_values = bucket["metric_values"] expected_by_key[key] = { "scored_pages": bucket["scored_pages"], "text_codepoint_edits": cp_edits, "text_codepoint_ref_len": cp_len, "text_cer_codepoint_micro": cp_edits / max(cp_len, 1), "text_grapheme_edits": gr_edits, "text_grapheme_ref_len": gr_len, "text_cer_grapheme_micro": gr_edits / max(gr_len, 1), "text_reference_pages": text_reference_pages, "text_exact_pages": text_exact_pages, "text_exact_accuracy": text_exact_pages / max(text_reference_pages, 1), "metric_means": { field: sum(values) / len(values) for field, values in sorted(metric_values.items()) if values }, } recorded_by_key: dict[tuple[str, str, str, str, str], dict[str, Any]] = {} for index, row in enumerate(slices, 1): if not isinstance(row, dict): issues.append(f"page slices[{index}] is not an object") continue key = ( str(row.get("language") or ""), str(row.get("script") or ""), str(row.get("domain") or ""), str(row.get("gt_status") or ""), str(row.get("split") or ""), ) if key in recorded_by_key: issues.append(f"duplicate page slice metric for {key}") recorded_by_key[key] = row expected_keys = set(expected_by_key) recorded_keys = set(recorded_by_key) missing = sorted(expected_keys - recorded_keys) extra = sorted(recorded_keys - expected_keys) if missing: issues.append(f"page slice metrics missing slices: {missing[:10]}") if extra: issues.append(f"page slice metrics contain extra slices: {extra[:10]}") for key in sorted(expected_keys & recorded_keys): expected = expected_by_key[key] recorded = recorded_by_key[key] for field in ( "scored_pages", "text_codepoint_edits", "text_codepoint_ref_len", "text_grapheme_edits", "text_grapheme_ref_len", "text_reference_pages", "text_exact_pages", ): if recorded.get(field) != expected[field]: issues.append(f"page slice {key} {field} mismatch: recorded={recorded.get(field)} recomputed={expected[field]}") for field in ("text_cer_codepoint_micro", "text_cer_grapheme_micro", "text_exact_accuracy"): value = recorded.get(field) if not _is_number(value) or not _close(float(value), float(expected[field])): issues.append(f"page slice {key} {field} mismatch: recorded={value} recomputed={expected[field]}") expected_means = expected["metric_means"] recorded_means = recorded.get("metric_means") if not isinstance(recorded_means, dict): issues.append(f"page slice {key} metric_means missing or not an object") continue missing_means = sorted(set(expected_means) - set(recorded_means)) extra_means = sorted(set(recorded_means) - set(expected_means)) if missing_means: issues.append(f"page slice {key} metric_means missing metrics: {missing_means[:10]}") if extra_means: issues.append(f"page slice {key} metric_means contain extra metrics: {extra_means[:10]}") for field in sorted(set(expected_means) & set(recorded_means)): value = recorded_means.get(field) expected_value = expected_means[field] if not _is_number(value) or not _close(float(value), expected_value): issues.append(f"page slice {key} metric_means.{field} mismatch: recorded={value} recomputed={expected_value}") return issues def _normalizer_default_counts(payload: dict[str, Any]) -> dict[str, int]: raw_counts = payload.get("normalizer_counts") if not isinstance(raw_counts, dict): return {} defaults: dict[str, int] = {} for key in ( "defaulted_language", "defaulted_script", "defaulted_domain", "defaulted_gt_status", "defaulted_split", "defaulted_claim_eligible", ): value = raw_counts.get(key, 0) or 0 if isinstance(value, bool): continue try: count = int(value) except (TypeError, ValueError): defaults[key] = 1 continue if count: defaults[key] = count return defaults def _recognizer_summary_counts(eval_run: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: summary, issues = _read_summary_for_audit(eval_run, "recognizer", "normalizer_counts audit") if issues or not summary: return {}, issues counts = summary.get("counts") if counts is None: return {}, [] if not isinstance(counts, dict): summary_file = eval_run.get("summary_file") return {}, [f"recognizer summary counts is not an object: {summary_file}"] return counts, [] def _check_recognizer_summary_counts(eval_run: dict[str, Any], metrics: dict[str, Any]) -> list[str]: summary_counts, issues = _recognizer_summary_counts(eval_run) if issues or not summary_counts: return issues metric_counts = metrics.get("normalizer_counts") run_counts = eval_run.get("normalizer_counts") if metric_counts != summary_counts: issues.append("metrics.json normalizer_counts do not match recognizer summary counts") if run_counts != summary_counts: issues.append("eval-run.json normalizer_counts do not match recognizer summary counts") return issues def _audit_recognizer(score_dir: Path) -> tuple[list[str], dict[str, Any], dict[str, Any]]: issues = _check_required_artifacts(score_dir, RECOGNIZER_ARTIFACTS) eval_run = _read_json(score_dir / "eval-run.json") metrics = _read_json(score_dir / "metrics.json") if not isinstance(eval_run, dict) or not isinstance(metrics, dict): return ["recognizer eval artifacts must contain JSON objects"], {}, {} issues.extend(_check_required_fields(eval_run, RECOGNIZER_REQUIRED_RUN_FIELDS, "eval-run.json")) issues.extend(_check_common(score_dir, eval_run, metrics)) issues.extend( _check_eval_run_identity_lists( eval_run, _tsv_rows(score_dir / "predictions.tsv"), run_label="eval-run.json", row_label="predictions.tsv", ) ) issues.extend(_check_recognizer_metadata_contradictions(score_dir, metrics, eval_run)) issues.extend(_check_recognizer_provenance_coverage(score_dir, metrics, eval_run)) issues.extend(_check_recognizer_claim_blockers(eval_run, metrics)) if eval_run.get("eval_mode") != "recognizer_oracle_crop": issues.append(f"unexpected recognizer eval_mode: {eval_run.get('eval_mode')}") for path_field, hash_field, label in ( ("label_file", "label_file_sha256", "recognizer label file"), ("prediction_file", "prediction_file_sha256", "recognizer prediction file"), ("manifest_file", "manifest_file_sha256", "recognizer manifest file"), ("summary_file", "summary_file_sha256", "recognizer summary file"), ("script_inventory_file", "script_inventory_file_sha256", "recognizer script inventory file"), ("spotcheck_calibration_file", "spotcheck_calibration_file_sha256", "recognizer spotcheck calibration file"), ): issues.extend(_check_hash_field(eval_run, path_field, hash_field, label)) issues.extend(_check_summary_metadata_consistency(eval_run, "recognizer")) issues.extend(_check_spotcheck_calibration_summary(eval_run, metrics, "recognizer")) issues.extend(_check_artifact_identity(eval_run, ("checkpoint", "config", "dictionary"), "recognizer")) issues.extend(_check_train_manifest_hashes(eval_run)) issues.extend(_check_recognizer_leakage_report(score_dir, metrics, eval_run)) issues.extend(_check_recognizer_output_duplicates(score_dir)) issues.extend(_check_recognizer_prediction_source_rows(score_dir, eval_run)) issues.extend(_check_recognizer_pack_content_digest(eval_run, metrics)) issues.extend(_check_recognizer_row_claim_eligibility_blockers(score_dir, eval_run)) issues.extend(_check_recognizer_unmatched_artifact(score_dir, eval_run)) issues.extend(_check_recognizer_match_count_metrics(metrics, eval_run)) issues.extend(_check_recognizer_row_hashes(score_dir, eval_run)) issues.extend(_check_recognizer_predictions_recompute(score_dir, metrics, eval_run)) issues.extend( _check_normalization_summary( metrics.get("normalization_summary"), _expected_recognizer_normalization_summary(score_dir), "recognizer metrics", ) ) issues.extend(_check_recognizer_uncertainty(score_dir, metrics)) issues.extend(_check_recognizer_script_inventory(score_dir, metrics, eval_run)) issues.extend(_check_recognizer_dictionary_coverage(score_dir, metrics, eval_run)) issues.extend(_check_recognizer_confusions(score_dir)) issues.extend(_check_recognizer_worst_artifacts(score_dir)) issues.extend(_check_ratio(metrics, "codepoint_edits", "codepoint_ref_len", "cer_codepoint_micro")) issues.extend(_check_ratio(metrics, "grapheme_edits", "grapheme_ref_len", "cer_grapheme_micro")) if metrics.get("n_lines") != metrics.get("matched_lines"): issues.append("recognizer n_lines and matched_lines disagree") if metrics.get("matched_lines") != _artifact_rows(score_dir / "predictions.tsv"): issues.append("predictions.tsv row count does not match matched_lines") unmatched_expected = int(metrics.get("unmatched_labels", 0)) + int(metrics.get("unmatched_predictions", 0)) if unmatched_expected != _artifact_rows(score_dir / "unmatched.tsv"): issues.append("unmatched.tsv row count does not match unmatched counts") coverage = metrics.get("provenance_coverage") if not isinstance(coverage, dict): issues.append("recognizer provenance coverage is missing") run_default_counts = _normalizer_default_counts(eval_run) metric_default_counts = _normalizer_default_counts(metrics) if run_default_counts: warnings = eval_run.get("warnings") if isinstance(eval_run.get("warnings"), list) else [] for key in sorted(run_default_counts): expected = f"normalizer_{key}:" if not any(str(warning).startswith(expected) for warning in warnings): issues.append(f"recognizer normalizer default count missing warning: {key}") if eval_run.get("claim_safe") is True: issues.append("recognizer claim_safe is true despite normalizer-defaulted metadata") if run_default_counts != metric_default_counts: issues.append("recognizer normalizer_counts disagree between eval-run.json and metrics.json") issues.extend(_check_recognizer_summary_counts(eval_run, metrics)) leakage = metrics.get("leakage") if not isinstance(leakage, dict) or not leakage.get("checked"): issues.append("recognizer train/dev leakage check was not run") return issues, eval_run, metrics def _check_materialization_records(payload: dict[str, Any], materialize_dir: Path | None = None) -> list[str]: issues: list[str] = [] mode = payload.get("mode") records = payload.get("records") if not isinstance(records, list) or not records: return ["materialization records are missing"] record_count = payload.get("record_count") if not isinstance(record_count, int) or isinstance(record_count, bool): issues.append(f"materialization record_count is missing or not an integer: {record_count!r}") elif record_count != len(records): issues.append(f"materialization record_count does not match records length: recorded={record_count} actual={len(records)}") for index, record in enumerate(records): if not isinstance(record, dict): issues.append(f"materialization records[{index}] is not an object") continue command = record.get("command") if not isinstance(command, list) or not command or not all(isinstance(part, str) and part for part in command): issues.append(f"materialization records[{index}] command is missing or invalid") else: command_template = payload.get("command_template") if isinstance(command_template, str) and command_template.strip() and materialize_dir is not None: values: dict[str, str] | None = None if mode == "recognizer": label_file = payload.get("label_file") data_dir = payload.get("data_dir") prediction_file = payload.get("prediction_file") if all(isinstance(value, str) and value for value in (label_file, data_dir, prediction_file)): values = { "label_file": str(label_file), "data_dir": str(data_dir), "prediction_file": str(prediction_file), "out_dir": str(materialize_dir), } else: issues.append(f"materialization records[{index}] missing recognizer fields required to replay command_template") elif mode == "page": output_path = record.get("output_path") input_path = record.get("input_path") sample_id = record.get("sample_id") documents_dir = payload.get("documents_dir") if all(isinstance(value, str) and value for value in (output_path, input_path, sample_id, documents_dir)): values = { "input_path": str(input_path), "sample_id": str(sample_id), "output_dir": str(Path(str(output_path)).parent), "document_json": str(output_path), "documents_dir": str(documents_dir), "out_dir": str(materialize_dir), } else: issues.append(f"materialization records[{index}] missing page fields required to replay command_template") if values is not None: try: expected_command = shlex.split(command_template.format(**values)) except (KeyError, ValueError) as exc: issues.append(f"materialization records[{index}] command_template could not be replayed: {exc}") else: if command != expected_command: issues.append(f"materialization records[{index}] command does not match materialization command_template") for log_field in ("stdout_path", "stderr_path"): log_path = record.get(log_field) sha_field = log_field.replace("_path", "_sha256") log_sha256 = record.get(sha_field) if not isinstance(log_path, str) or not log_path: issues.append(f"materialization records[{index}] missing {log_field}") elif not Path(log_path).exists(): issues.append(f"materialization records[{index}] {log_field} does not exist: {log_path}") elif isinstance(log_sha256, str) and log_sha256: actual_log_sha256 = _sha256_file(Path(log_path)) if actual_log_sha256 != log_sha256: issues.append( f"materialization records[{index}] {sha_field} mismatch: " f"recorded={log_sha256} actual={actual_log_sha256}" ) else: issues.append(f"materialization records[{index}] missing {sha_field}") output_path = record.get("output_path") output_sha256 = record.get("output_sha256") returncode = record.get("returncode") if returncode != 0: issues.append(f"materialization records[{index}] returncode is not zero: {returncode}") if not isinstance(output_path, str) or not output_path: issues.append(f"materialization records[{index}] missing output_path") continue if not isinstance(output_sha256, str) or not output_sha256: issues.append(f"materialization records[{index}] missing output_sha256") continue path = Path(output_path) if not path.exists(): issues.append(f"materialization records[{index}] output_path does not exist: {output_path}") continue actual = _sha256_file(path) if actual != output_sha256: issues.append(f"materialization records[{index}] output_sha256 mismatch: recorded={output_sha256} actual={actual}") if mode == "page": input_path = record.get("input_path") input_sha256 = record.get("input_sha256") if not isinstance(input_path, str) or not input_path: issues.append(f"materialization records[{index}] missing input_path for page mode") continue if not isinstance(input_sha256, str) or not input_sha256: issues.append(f"materialization records[{index}] missing input_sha256 for page mode") continue source_path = Path(input_path) if not source_path.exists(): issues.append(f"materialization records[{index}] input_path does not exist: {input_path}") continue actual_input_sha256 = _sha256_file(source_path) if actual_input_sha256 != input_sha256: issues.append( f"materialization records[{index}] input_sha256 mismatch: " f"recorded={input_sha256} actual={actual_input_sha256}" ) return issues def _check_materialization_provenance(payload: dict[str, Any]) -> list[str]: issues: list[str] = [] issues.extend(_check_timestamp(payload, "created_at", "materialization run")) for field in ("mode", "system_name", "command_template", "invocation"): value = payload.get(field) if not isinstance(value, str) or not value: issues.append(f"materialization missing required field: {field}") issues.extend(_check_machine_metadata(payload, "materialization")) return issues def _same_path(left: Any, right: Path) -> bool: if not isinstance(left, str) or not left: return False try: return Path(left).resolve() == right.resolve() except OSError: return False def _is_relative_to(path: Path, parent: Path) -> bool: try: path.resolve().relative_to(parent.resolve()) except ValueError: return False except OSError: return False return True def _check_recipe_context_identity(recipe_run: dict[str, Any]) -> list[str]: issues: list[str] = [] root_value = recipe_run.get("root") recipe_file_value = recipe_run.get("recipe_file") if not isinstance(root_value, str) or not root_value: return ["recipe-run.json root is missing"] root_path = Path(root_value) if not root_path.exists() or not root_path.is_dir(): issues.append(f"recipe-run.json root does not exist or is not a directory: {root_value}") return issues if not isinstance(recipe_file_value, str) or not recipe_file_value: return issues recipe_file_path = Path(recipe_file_value) if recipe_file_path.exists() and not _is_relative_to(recipe_file_path, root_path): issues.append("recipe-run.json recipe_file is not under declared root") return issues def _check_recipe_output_lineage(run_dir: Path, recipe_run: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] outputs = recipe_run.get("outputs") materialization = recipe_run.get("materialization") if not isinstance(outputs, dict) or not isinstance(materialization, dict): return issues materialization_file = run_dir / "materialize" / "output-command-run.json" if materialization_file.exists(): try: materialization_payload = _read_json(materialization_file) except (FileNotFoundError, ValueError) as exc: issues.append(str(exc)) else: if materialization_payload != materialization: issues.append("materialize/output-command-run.json does not match recipe-run.json materialization") command_template = materialization.get("command_template") command_template_file = run_dir / "materialize" / "command-template.txt" if command_template_file.exists() and isinstance(command_template, str): if command_template_file.read_text(encoding="utf-8").strip() != command_template.strip(): issues.append("materialize/command-template.txt does not match recorded materialization command_template") report_file = run_dir / "materialize" / "output-command-run.md" records = materialization.get("records") if report_file.exists() and isinstance(records, list): report_text = report_file.read_text(encoding="utf-8") expected_record_line = f"- records: {len(records)}" if expected_record_line not in report_text: issues.append(f"materialize/output-command-run.md missing or stale line: {expected_record_line}") for index, record in enumerate(records): if not isinstance(record, dict): continue output_path = record.get("output_path") output_sha256 = record.get("output_sha256") input_path = record.get("input_path") input_sha256 = record.get("input_sha256") sample_id = record.get("sample_id") if isinstance(output_path, str) and isinstance(output_sha256, str) and isinstance(sample_id, str): if isinstance(input_path, str) and isinstance(input_sha256, str): expected_output_line = ( f"- {sample_id}: input `{input_path}` sha256={input_sha256}; " f"output `{output_path}` sha256={output_sha256}" ) else: expected_output_line = f"- {sample_id}: `{output_path}` sha256={output_sha256}" if expected_output_line not in report_text: issues.append(f"materialize/output-command-run.md missing or stale output line for record {index}") stdout_path = record.get("stdout_path") stdout_sha256 = record.get("stdout_sha256") if isinstance(stdout_path, str) and isinstance(stdout_sha256, str): expected_stdout = f"stdout `{stdout_path}` sha256={stdout_sha256}" if expected_stdout not in report_text: issues.append(f"materialize/output-command-run.md missing or stale stdout line for record {index}") stderr_path = record.get("stderr_path") stderr_sha256 = record.get("stderr_sha256") if isinstance(stderr_path, str) and isinstance(stderr_sha256, str): expected_stderr = f"stderr `{stderr_path}` sha256={stderr_sha256}" if expected_stderr not in report_text: issues.append(f"materialize/output-command-run.md missing or stale stderr line for record {index}") if eval_run.get("eval_mode") == "page_level_ocr": documents_value = outputs.get("documents_dir") eval_documents_dir = eval_run.get("documents_dir") if isinstance(documents_value, str) and documents_value: if isinstance(eval_documents_dir, str) and eval_documents_dir and not _same_path(documents_value, Path(eval_documents_dir)): issues.append("recipe-run.json outputs.documents_dir does not match score eval-run documents_dir") if not Path(documents_value).exists(): issues.append(f"recipe-run.json output path does not exist for documents_dir: {documents_value}") if isinstance(records, list): for index, record in enumerate(records): if not isinstance(record, dict): continue output_path = record.get("output_path") if isinstance(output_path, str) and output_path and not _is_relative_to(Path(output_path), Path(documents_value)): issues.append(f"recipe-run.json materialization record {index} output_path is not under outputs.documents_dir") return issues prediction_value = outputs.get("prediction_file") eval_prediction = eval_run.get("prediction_file") eval_prediction_sha256 = eval_run.get("prediction_file_sha256") if isinstance(prediction_value, str) and prediction_value: prediction_path = Path(prediction_value) if isinstance(eval_prediction, str) and eval_prediction and not _same_path(prediction_value, Path(eval_prediction)): issues.append("recipe-run.json outputs.prediction_file does not match score eval-run prediction_file") if isinstance(eval_prediction_sha256, str) and eval_prediction_sha256 and prediction_path.exists(): actual_sha256 = _sha256_file(prediction_path) if actual_sha256 != eval_prediction_sha256: issues.append( "recipe-run.json outputs.prediction_file sha256 does not match score eval-run prediction_file_sha256: " f"recorded={eval_prediction_sha256} actual={actual_sha256}" ) if isinstance(records, list) and isinstance(prediction_value, str) and prediction_value: matching_records = [ record for record in records if isinstance(record, dict) and _same_path(record.get("output_path"), Path(prediction_value)) ] if not matching_records: issues.append("recipe-run.json materialization records do not include outputs.prediction_file") elif isinstance(eval_prediction_sha256, str) and eval_prediction_sha256: for index, record in enumerate(matching_records): if record.get("output_sha256") != eval_prediction_sha256: issues.append( "recipe-run.json materialization prediction output_sha256 does not match score eval-run " f"prediction_file_sha256 for matching record {index}: " f"recorded={record.get('output_sha256')} score={eval_prediction_sha256}" ) return issues def _check_recipe_output_audit_consistency(run_dir: Path, recipe_run: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: if eval_run.get("eval_mode") != "page_level_ocr": return [] output_audit = recipe_run.get("output_audit") if output_audit is None: return [] issues: list[str] = [] if not isinstance(output_audit, dict): return ["recipe-run.json output_audit is not an object"] outputs = recipe_run.get("outputs") if isinstance(outputs, dict): output_audit_dir = outputs.get("output_audit_dir") if not isinstance(output_audit_dir, str) or not output_audit_dir: issues.append("recipe-run.json outputs.output_audit_dir is missing for page output audit") elif not _same_path(output_audit_dir, run_dir / "output-audit"): issues.append("recipe-run.json outputs.output_audit_dir does not point at root output-audit") elif not Path(output_audit_dir).is_dir(): issues.append(f"recipe-run.json output path does not exist for output_audit_dir: {output_audit_dir}") output_audit_path = run_dir / "output-audit" / "output-audit-run.json" try: output_audit_payload = _read_json(output_audit_path) except (FileNotFoundError, ValueError) as exc: issues.append(str(exc)) else: if output_audit_payload != output_audit: issues.append("output-audit/output-audit-run.json does not match recipe-run.json output_audit") command_template = output_audit.get("command_template") command_template_file = run_dir / "output-audit" / "command-template.txt" if not isinstance(command_template, str) or not command_template.strip(): issues.append("recipe-run.json output_audit command_template is missing") elif not command_template_file.is_file(): issues.append("missing required artifact: output-audit/command-template.txt") else: if command_template_file.read_text(encoding="utf-8").strip() != command_template.strip(): issues.append("output-audit/command-template.txt does not match recorded output_audit command_template") records = output_audit.get("records") record_count = output_audit.get("record_count") if not isinstance(records, list): issues.append("recipe-run.json output_audit records is not a list") records = [] elif record_count != len(records): issues.append(f"recipe-run.json output_audit record_count mismatch: recorded={record_count} actual={len(records)}") if output_audit.get("passed") is not True: issues.append("recipe-run.json output_audit did not pass") for index, record in enumerate(records): if not isinstance(record, dict): issues.append(f"recipe-run.json output_audit records[{index}] is not an object") continue if record.get("returncode") != 0: issues.append(f"recipe-run.json output_audit records[{index}] returncode is not zero: {record.get('returncode')}") command = record.get("command") output_dir = record.get("output_dir") if not isinstance(command, list) or not command or not all(isinstance(part, str) and part for part in command): issues.append(f"recipe-run.json output_audit records[{index}] command is missing or invalid") elif isinstance(command_template, str) and command_template.strip(): if not isinstance(output_dir, str) or not output_dir: issues.append(f"recipe-run.json output_audit records[{index}] missing output_dir") else: try: expected_command = shlex.split( command_template.format( output_dir=output_dir, documents_dir=str(run_dir / "documents"), out_dir=str(run_dir / "output-audit"), ) ) except (KeyError, ValueError) as exc: issues.append(f"recipe-run.json output_audit records[{index}] command_template could not be replayed: {exc}") else: if command != expected_command: issues.append(f"recipe-run.json output_audit records[{index}] command does not match output_audit command_template") for path_key in ("stdout_path", "stderr_path"): path_value = record.get(path_key) sha_key = path_key.replace("_path", "_sha256") sha_value = record.get(sha_key) if not isinstance(path_value, str) or not path_value: issues.append(f"recipe-run.json output_audit records[{index}] missing {path_key}") elif not Path(path_value).is_file(): issues.append(f"recipe-run.json output_audit records[{index}] {path_key} does not exist: {path_value}") elif isinstance(sha_value, str) and sha_value: actual_log_sha256 = _sha256_file(Path(path_value)) if actual_log_sha256 != sha_value: issues.append( f"recipe-run.json output_audit records[{index}] {sha_key} mismatch: " f"recorded={sha_value} actual={actual_log_sha256}" ) else: issues.append(f"recipe-run.json output_audit records[{index}] missing {sha_key}") audit_json = record.get("audit_json") if not isinstance(audit_json, dict): issues.append(f"recipe-run.json output_audit records[{index}] missing audit_json artifact") else: audit_json_path = audit_json.get("path") audit_json_sha256 = audit_json.get("sha256") if not isinstance(audit_json_path, str) or not audit_json_path: issues.append(f"recipe-run.json output_audit records[{index}] audit_json missing path") elif not Path(audit_json_path).is_file(): issues.append(f"recipe-run.json output_audit records[{index}] audit_json path does not exist: {audit_json_path}") elif isinstance(audit_json_sha256, str) and audit_json_sha256: actual_audit_json_sha256 = _sha256_file(Path(audit_json_path)) if actual_audit_json_sha256 != audit_json_sha256: issues.append( f"recipe-run.json output_audit records[{index}] audit_json sha256 mismatch: " f"recorded={audit_json_sha256} actual={actual_audit_json_sha256}" ) else: issues.append(f"recipe-run.json output_audit records[{index}] audit_json missing sha256") audit_markdown = record.get("audit_markdown") if audit_markdown is not None: if not isinstance(audit_markdown, dict): issues.append(f"recipe-run.json output_audit records[{index}] audit_markdown is not an object") else: audit_markdown_path = audit_markdown.get("path") audit_markdown_sha256 = audit_markdown.get("sha256") if not isinstance(audit_markdown_path, str) or not audit_markdown_path: issues.append(f"recipe-run.json output_audit records[{index}] audit_markdown missing path") elif not Path(audit_markdown_path).is_file(): issues.append(f"recipe-run.json output_audit records[{index}] audit_markdown path does not exist: {audit_markdown_path}") elif isinstance(audit_markdown_sha256, str) and audit_markdown_sha256: actual_audit_markdown_sha256 = _sha256_file(Path(audit_markdown_path)) if actual_audit_markdown_sha256 != audit_markdown_sha256: issues.append( f"recipe-run.json output_audit records[{index}] audit_markdown sha256 mismatch: " f"recorded={audit_markdown_sha256} actual={actual_audit_markdown_sha256}" ) else: issues.append(f"recipe-run.json output_audit records[{index}] audit_markdown missing sha256") report_path = run_dir / "output-audit" / "output-audit-run.md" if not report_path.is_file(): issues.append("missing required artifact: output-audit/output-audit-run.md") else: report_text = report_path.read_text(encoding="utf-8") expected_record_line = f"- records: {len(records)}" if expected_record_line not in report_text: issues.append(f"output-audit/output-audit-run.md missing or stale line: {expected_record_line}") for index, record in enumerate(records): if not isinstance(record, dict): continue sample_id = record.get("sample_id") output_dir = record.get("output_dir") returncode = record.get("returncode") if isinstance(sample_id, str) and isinstance(output_dir, str): expected_output_line = f"- {sample_id}: returncode={returncode} output_dir=`{output_dir}`" if expected_output_line not in report_text: issues.append(f"output-audit/output-audit-run.md missing or stale output line for record {index}") if isinstance(record.get("stdout_path"), str) and isinstance(record.get("stdout_sha256"), str): expected_stdout = f"stdout=`{record['stdout_path']}` sha256={record['stdout_sha256']}" if expected_stdout not in report_text: issues.append(f"output-audit/output-audit-run.md missing or stale stdout line for record {index}") if isinstance(record.get("stderr_path"), str) and isinstance(record.get("stderr_sha256"), str): expected_stderr = f"stderr=`{record['stderr_path']}` sha256={record['stderr_sha256']}" if expected_stderr not in report_text: issues.append(f"output-audit/output-audit-run.md missing or stale stderr line for record {index}") audit_json = record.get("audit_json") if isinstance(audit_json, dict) and isinstance(audit_json.get("path"), str) and isinstance(audit_json.get("sha256"), str): expected_audit_json = f"audit_json=`{audit_json['path']}` sha256={audit_json['sha256']}" if expected_audit_json not in report_text: issues.append(f"output-audit/output-audit-run.md missing or stale audit JSON line for record {index}") audit_markdown = record.get("audit_markdown") if ( isinstance(audit_markdown, dict) and isinstance(audit_markdown.get("path"), str) and isinstance(audit_markdown.get("sha256"), str) ): expected_audit_markdown = f"audit_markdown=`{audit_markdown['path']}` sha256={audit_markdown['sha256']}" if expected_audit_markdown not in report_text: issues.append(f"output-audit/output-audit-run.md missing or stale audit Markdown line for record {index}") return issues def _check_recipe_materialization_score_identity(recipe_run: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: materialization = recipe_run.get("materialization") if not isinstance(materialization, dict): return [] issues: list[str] = [] recipe_id = recipe_run.get("recipe_id") if isinstance(recipe_id, str) and recipe_id: system_name = materialization.get("system_name") if system_name != recipe_id: issues.append(f"recipe-run.json materialization system_name does not match recipe_id: recorded={system_name} recipe_id={recipe_id}") for field in ("model_name", "checkpoint", "config"): recorded = materialization.get(field) scored = eval_run.get(field) if recorded != scored: issues.append(f"recipe-run.json materialization {field} does not match score eval-run {field}: recorded={recorded} score={scored}") recorded = materialization.get("dictionary") scored = eval_run.get("dictionary") if recorded != scored: issues.append(f"recipe-run.json materialization dictionary does not match score eval-run dictionary: recorded={recorded} score={scored}") return issues def _check_recipe_score_artifact_identity(score_dir: Path, score: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] eval_run_name = "page-eval-run.json" if eval_run.get("eval_mode") == "page_level_ocr" else "eval-run.json" for key, expected_path in (("metrics_file", score_dir / "metrics.json"), ("eval_run_file", score_dir / eval_run_name)): value = score.get(key) if not isinstance(value, str) or not value: issues.append(f"recipe-run.json score missing {key}") elif not _same_path(value, expected_path): issues.append(f"recipe-run.json score {key} does not point at score/{expected_path.name}") hash_key = f"{key}_sha256" recorded_hash = score.get(hash_key) if not isinstance(recorded_hash, str) or not recorded_hash: issues.append(f"recipe-run.json score missing {hash_key}") elif expected_path.exists(): actual_hash = _sha256_file(expected_path) if actual_hash != recorded_hash: issues.append( f"recipe-run.json score {hash_key} does not match score/{expected_path.name}: " f"recorded={recorded_hash} actual={actual_hash}" ) return issues def _check_recipe_train_manifest_identity(recipe_run: dict[str, Any], eval_run: dict[str, Any]) -> list[str]: issues: list[str] = [] recorded = recipe_run.get("train_manifests") scored = eval_run.get("train_manifests") if recorded is None: recorded = [] if scored is None: scored = [] if not isinstance(recorded, list): return ["recipe-run.json train_manifests is not a list"] if not isinstance(scored, list): return ["eval-run.json train_manifests is not a list for recipe-root audit"] if len(recorded) != len(scored): return [f"recipe-run.json train_manifests count does not match eval-run.json: recorded={len(recorded)} score={len(scored)}"] for index, (recorded_item, scored_item) in enumerate(zip(recorded, scored, strict=True)): if not isinstance(recorded_item, dict): issues.append(f"recipe-run.json train_manifests[{index}] is not an object") continue if not isinstance(scored_item, dict): issues.append(f"eval-run.json train_manifests[{index}] is not an object for recipe-root audit") continue recorded_path = recorded_item.get("path") scored_path = scored_item.get("path") if not isinstance(recorded_path, str) or not recorded_path: issues.append(f"recipe-run.json train_manifests[{index}] missing path") elif not _same_path(recorded_path, Path(str(scored_path))): issues.append(f"recipe-run.json train_manifests[{index}] path does not match eval-run.json") recorded_sha = recorded_item.get("sha256") scored_sha = scored_item.get("sha256") if not isinstance(recorded_sha, str) or not recorded_sha: issues.append(f"recipe-run.json train_manifests[{index}] missing sha256") elif recorded_sha != scored_sha: issues.append(f"recipe-run.json train_manifests[{index}] sha256 does not match eval-run.json") return issues def _current_recipe_validation(registry_path: Path, root_path: Path, recipe_id: str) -> dict[str, Any]: validator = _load_tool("validate_eval_harness_recipes") validation = validator.validate_recipes(registry_path, root_path) for recipe_validation in validation.get("recipes", []): if isinstance(recipe_validation, dict) and recipe_validation.get("id") == recipe_id: return recipe_validation raise ValueError(f"recipe validator result missing selected recipe: {recipe_id}") def _check_recipe_artifact_consistency(run_dir: Path, recipe_run: dict[str, Any]) -> list[str]: issues: list[str] = [] outputs = recipe_run.get("outputs") if not isinstance(outputs, dict): return issues recipe_id = recipe_run.get("recipe_id") recipe_path_value = outputs.get("recipe_file") validation_path_value = outputs.get("recipe_validation_file") local_recipe: dict[str, Any] | None = None if isinstance(recipe_path_value, str) and recipe_path_value: try: payload = _read_json(Path(recipe_path_value)) except (FileNotFoundError, ValueError) as exc: issues.append(str(exc)) else: if isinstance(payload, dict): local_recipe = payload if isinstance(recipe_id, str) and recipe_id and payload.get("id") != recipe_id: issues.append( "recipe.json id does not match recipe-run.json recipe_id: " f"recipe.json={payload.get('id')} recipe-run.json={recipe_id}" ) else: issues.append("recipe.json must contain a JSON object") if isinstance(validation_path_value, str) and validation_path_value: try: validation_payload = _read_json(Path(validation_path_value)) except (FileNotFoundError, ValueError) as exc: issues.append(str(exc)) else: if validation_payload != recipe_run.get("recipe_validation"): issues.append("recipe-validation.json does not match recipe-run.json recipe_validation") registry_path_value = recipe_run.get("recipe_file") if isinstance(registry_path_value, str) and registry_path_value and isinstance(recipe_id, str) and recipe_id and local_recipe: registry_path = Path(registry_path_value) try: registry = _read_json(registry_path) except (FileNotFoundError, ValueError) as exc: issues.append(str(exc)) else: recipes = registry.get("recipes") if isinstance(registry, dict) else None if not isinstance(recipes, list): issues.append("recipe registry file must contain a recipes list") else: matches = [recipe for recipe in recipes if isinstance(recipe, dict) and recipe.get("id") == recipe_id] if not matches: issues.append(f"recipe registry file does not contain recipe_id: {recipe_id}") elif len(matches) > 1: issues.append(f"recipe registry file contains duplicate recipe_id: {recipe_id}") elif matches[0] != local_recipe: issues.append("recipe.json does not match selected recipe in recipe registry file") root_value = recipe_run.get("root") if isinstance(root_value, str) and root_value: try: current_validation = _current_recipe_validation(registry_path, Path(root_value), recipe_id) except Exception as exc: issues.append(f"recipe validation recomputation failed: {type(exc).__name__}: {exc}") else: if current_validation != recipe_run.get("recipe_validation"): issues.append("recipe-run.json recipe_validation does not match current recipe validator result") if isinstance(recipe_path_value, str) and recipe_path_value and not _same_path(recipe_path_value, run_dir / "recipe.json"): issues.append("recipe-run.json outputs.recipe_file does not point at root recipe.json") if isinstance(validation_path_value, str) and validation_path_value and not _same_path( validation_path_value, run_dir / "recipe-validation.json" ): issues.append("recipe-run.json outputs.recipe_validation_file does not point at root recipe-validation.json") return issues def _audit_recipe_root(run_dir: Path, score_dir: Path, eval_run: dict[str, Any], metrics: dict[str, Any], score_issues: list[str]) -> list[str]: if run_dir == score_dir: return [] issues = _check_required_artifacts(run_dir, RECIPE_ROOT_ARTIFACTS) if eval_run.get("eval_mode") == "recognizer_oracle_crop": issues.extend(_check_required_artifacts(run_dir, ("predictions.txt",))) try: recipe_run = _read_json(run_dir / "recipe-run.json") except (FileNotFoundError, ValueError) as exc: return [*issues, str(exc)] if not isinstance(recipe_run, dict): return [*issues, "recipe-run.json must contain a JSON object"] issues.extend(_check_required_fields(recipe_run, RECIPE_REQUIRED_FIELDS, "recipe-run.json")) issues.extend(_check_timestamp(recipe_run, "created_at", "recipe run")) issues.extend(_check_machine_metadata(recipe_run, "recipe run")) issues.extend(_check_recipe_context_identity(recipe_run)) if not isinstance(recipe_run.get("command"), str) or not recipe_run["command"]: issues.append("recipe run command is missing") issues.extend(_check_hash_field(recipe_run, "recipe_file", "recipe_file_sha256", "recipe registry file")) outputs = recipe_run.get("outputs") if not isinstance(outputs, dict): issues.append("recipe-run.json outputs is missing or not an object") else: required_outputs = [ "materialization_dir", "score_dir", "recipe_file", "recipe_validation_file", "audit_file", "claim_safety_report", ] if eval_run.get("eval_mode") == "page_level_ocr": required_outputs.append("documents_dir") else: required_outputs.append("prediction_file") for key in required_outputs: value = outputs.get(key) if not isinstance(value, str) or not value: issues.append(f"recipe-run.json outputs missing {key}") elif not Path(value).exists(): issues.append(f"recipe-run.json output path does not exist for {key}: {value}") if not _same_path(outputs.get("score_dir"), score_dir): issues.append("recipe-run.json outputs.score_dir does not match audited score directory") expected_root_paths = { "materialization_dir": run_dir / "materialize", "audit_file": run_dir / "audit.json", "claim_safety_report": run_dir / "claim-safety.md", } for key, expected_path in expected_root_paths.items(): if isinstance(outputs.get(key), str) and not _same_path(outputs.get(key), expected_path): issues.append(f"recipe-run.json outputs.{key} does not point at root {expected_path.name}") materialization = recipe_run.get("materialization") if not isinstance(materialization, dict): issues.append("recipe-run.json materialization is missing or not an object") else: issues.extend(_check_materialization_provenance(materialization)) issues.extend(_check_materialization_records(materialization, run_dir / "materialize")) issues.extend(_check_recipe_artifact_consistency(run_dir, recipe_run)) issues.extend(_check_recipe_train_manifest_identity(recipe_run, eval_run)) issues.extend(_check_recipe_output_lineage(run_dir, recipe_run, eval_run)) issues.extend(_check_recipe_output_audit_consistency(run_dir, recipe_run, eval_run)) issues.extend(_check_recipe_materialization_score_identity(recipe_run, eval_run)) recipe_audit = recipe_run.get("audit") if not isinstance(recipe_audit, dict): issues.append("recipe-run.json audit summary is missing or not an object") else: if bool(recipe_audit.get("audit_passed")) != (not score_issues): issues.append("recipe-run.json audit_passed does not match current score audit") if bool(recipe_audit.get("claim_safe")) != bool(eval_run.get("claim_safe")): issues.append("recipe-run.json audit claim_safe does not match eval run") if recipe_audit.get("issues") != score_issues: issues.append("recipe-run.json audit issues do not match current score audit") score = recipe_run.get("score") if not isinstance(score, dict): issues.append("recipe-run.json score summary is missing or not an object") else: if bool(score.get("claim_safe")) != bool(eval_run.get("claim_safe")): issues.append("recipe-run.json score claim_safe does not match eval run") warnings = eval_run.get("warnings") if isinstance(eval_run.get("warnings"), list) else [] if score.get("warnings") != warnings: issues.append("recipe-run.json score warnings do not match eval run") issues.extend(_check_recipe_score_artifact_identity(score_dir, score, eval_run)) try: root_audit = _read_json(run_dir / "audit.json") except (FileNotFoundError, ValueError) as exc: issues.append(str(exc)) else: if not isinstance(root_audit, dict): issues.append("audit.json must contain a JSON object") else: if bool(root_audit.get("audit_passed")) != (not score_issues): issues.append("audit.json audit_passed does not match current score audit") if bool(root_audit.get("claim_safe")) != bool(eval_run.get("claim_safe")): issues.append("audit.json claim_safe does not match eval run") if root_audit.get("warnings") != (eval_run.get("warnings") if isinstance(eval_run.get("warnings"), list) else []): issues.append("audit.json warnings do not match eval run") if root_audit.get("issues") != score_issues: issues.append("audit.json issues do not match current score audit") if not _same_path(root_audit.get("score_dir"), score_dir): issues.append("audit.json score_dir does not match audited score directory") claim_report = run_dir / "claim-safety.md" if claim_report.exists(): text = claim_report.read_text(encoding="utf-8") for marker in ("## What Was Scored", "## Ground Truth And Pack", "## Metrics", "## Warnings"): if marker not in text: issues.append(f"root claim-safety.md missing section: {marker}") issues.extend(_check_root_claim_safety_report_consistency(run_dir, recipe_run, eval_run, metrics)) return issues def _audit_page(score_dir: Path) -> tuple[list[str], dict[str, Any], dict[str, Any]]: issues = _check_required_artifacts(score_dir, PAGE_ARTIFACTS) eval_run = _read_json(score_dir / "page-eval-run.json") metrics = _read_json(score_dir / "metrics.json") if not isinstance(eval_run, dict) or not isinstance(metrics, dict): return ["page eval artifacts must contain JSON objects"], {}, {} issues.extend(_check_required_fields(eval_run, PAGE_REQUIRED_RUN_FIELDS, "page-eval-run.json")) issues.extend(_check_common(score_dir, eval_run, metrics)) issues.extend( _check_eval_run_identity_lists( eval_run, _csv_rows(score_dir / "page-metrics.csv"), run_label="page-eval-run.json", row_label="page-metrics.csv", ) ) issues.extend(_check_page_metadata_contradictions(score_dir, metrics, eval_run)) issues.extend(_check_page_metadata_coverage(metrics, eval_run)) issues.extend(_check_page_claim_blockers(eval_run, metrics)) issues.extend(_check_page_output_audit(score_dir, metrics, eval_run)) if eval_run.get("eval_mode") != "page_level_ocr": issues.append(f"unexpected page eval_mode: {eval_run.get('eval_mode')}") for path_field, hash_field, label in ( ("eval_manifest", "eval_manifest_sha256", "page eval manifest"), ("summary_file", "summary_file_sha256", "page summary file"), ("script_inventory_file", "script_inventory_file_sha256", "page script inventory file"), ("spotcheck_calibration_file", "spotcheck_calibration_file_sha256", "page spotcheck calibration file"), ): issues.extend(_check_hash_field(eval_run, path_field, hash_field, label)) issues.extend(_check_summary_metadata_consistency(eval_run, "page")) issues.extend(_check_spotcheck_calibration_summary(eval_run, metrics, "page")) issues.extend(_check_artifact_identity(eval_run, ("checkpoint", "config", "dictionary"), "page")) issues.extend(_check_train_manifest_hashes(eval_run)) issues.extend(_check_page_leakage_report(metrics, eval_run)) issues.extend(_check_page_output_duplicates(score_dir)) issues.extend(_check_page_missing_artifact(score_dir, eval_run)) issues.extend(_check_page_metric_identity_rows(score_dir, eval_run)) issues.extend(_check_page_pack_content_digest(eval_run, metrics)) issues.extend(_check_page_document_source_path_mismatches(metrics, eval_run)) issues.extend(_check_page_row_hashes(score_dir)) issues.extend(_check_page_manifest_count_metrics(metrics, eval_run)) issues.extend(_check_page_line_alignment_recompute(score_dir, metrics, eval_run)) issues.extend(_check_page_reference_artifacts(score_dir, eval_run)) issues.extend(_check_page_text_pairs_artifact(score_dir, eval_run)) issues.extend(_check_page_confusions(score_dir)) issues.extend(_check_page_text_metrics_recompute(score_dir, metrics, eval_run)) issues.extend( _check_normalization_summary( metrics.get("normalization_summary"), _expected_page_normalization_summary(score_dir), "page metrics", ) ) issues.extend(_check_page_uncertainty(score_dir, metrics)) issues.extend(_check_page_script_inventory(score_dir, metrics, eval_run)) issues.extend(_check_page_dictionary_coverage(score_dir, metrics, eval_run)) issues.extend(_check_page_worst_artifacts(score_dir)) issues.extend(_check_ratio(metrics, "text_codepoint_edits", "text_codepoint_ref_len", "text_cer_codepoint_micro")) issues.extend(_check_ratio(metrics, "text_grapheme_edits", "text_grapheme_ref_len", "text_cer_grapheme_micro")) if metrics.get("scored_pages") != _artifact_rows(score_dir / "page-metrics.csv"): issues.append("page-metrics.csv row count does not match scored_pages") if int(metrics.get("missing_documents", 0)) + int(metrics.get("missing_references", 0)) != _artifact_rows(score_dir / "missing.tsv"): issues.append("missing.tsv row count does not match missing document/reference counts") issues.extend(_check_page_reference_coverage(score_dir, metrics)) coverage = metrics.get("reference_coverage") if not isinstance(coverage, dict): issues.append("page reference coverage is missing") elif int(coverage.get("line_bboxes", 0)) <= 0: issues.append("page eval has no line-bbox reference coverage") metadata_coverage = metrics.get("metadata_coverage") if not isinstance(metadata_coverage, dict): issues.append("page metadata coverage is missing") gt_statuses = eval_run.get("gt_statuses") if isinstance(gt_statuses, list) and "machine_converted_spot_checked" in gt_statuses: calibration = eval_run.get("spotcheck_calibration") if not isinstance(calibration, dict): issues.append("page spot-checked machine GT is missing calibration summary") elif not calibration.get("claim_evidence_eligible"): issues.append("page spot-check calibration is not claim evidence eligible") metric_means = metrics.get("metric_means") if not isinstance(metric_means, dict): issues.append("page metric means are missing") else: for key in ("line_detection_f1", "line_mean_iou", "reading_order_pair_accuracy"): if key not in metric_means: issues.append(f"page metric_means missing {key}") issues.extend(_check_page_slice_metrics(score_dir, metrics.get("slices"))) leakage = metrics.get("leakage") if not isinstance(leakage, dict) or not leakage.get("checked"): issues.append("page train/dev leakage check was not run") return issues, eval_run, metrics def audit_run(run_dir: Path) -> dict[str, Any]: score_dir = _score_dir(run_dir) mode = _mode(score_dir) if mode == "recognizer_oracle_crop": issues, eval_run, metrics = _audit_recognizer(score_dir) elif mode == "page_level_ocr": issues, eval_run, metrics = _audit_page(score_dir) else: raise ValueError(f"unsupported eval mode: {mode}") issues.extend(_audit_recipe_root(run_dir, score_dir, eval_run, metrics, issues)) claim_safe = bool(eval_run.get("claim_safe")) if isinstance(eval_run, dict) else False return { "run_dir": str(run_dir), "score_dir": str(score_dir), "eval_mode": mode, "audit_passed": not issues, "claim_safe": claim_safe, "warnings": eval_run.get("warnings", []) if isinstance(eval_run, dict) else [], "issues": issues, "summary": { "model_name": eval_run.get("model_name") if isinstance(eval_run, dict) else None, "eval_pack_version": eval_run.get("eval_pack_version") if isinstance(eval_run, dict) else None, "normalization_profile": eval_run.get("normalization_profile") if isinstance(eval_run, dict) else None, "n_lines": metrics.get("n_lines") if isinstance(metrics, dict) else None, "scored_pages": metrics.get("scored_pages") if isinstance(metrics, dict) else None, }, } def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--out", type=Path) parser.add_argument("--require-audit-pass", action="store_true") parser.add_argument("--require-claim-safe", action="store_true") args = parser.parse_args(argv) result = audit_run(args.run_dir) if args.out: args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print( f"eval_mode={result['eval_mode']} audit_passed={result['audit_passed']} " f"claim_safe={result['claim_safe']} issues={len(result['issues'])}" ) if args.require_audit_pass and not result["audit_passed"]: raise SystemExit(1) if args.require_claim_safe and not result["claim_safe"]: raise SystemExit(1) return 0 if __name__ == "__main__": raise SystemExit(main())
#!/usr/bin/env python3
"""Audit a completed OCR eval output directory for review readiness."""
from __future__ import annotations
import argparse
import csv
import hashlib
import html
import importlib.util
import json
import random
import shlex
import sys
import unicodedata
from datetime import datetime
from pathlib import Path
from typing import Any
import regex
from rapidfuzz.distance import Levenshtein
from ocrtech.metrics import bbox_iou
from ocrtech.references import load_reference
from ocrtech.schemas import BBox, Document, TextLine
TOOLS_DIR = Path(__file__).resolve().parent
SUPPORTED_NORMALIZATION_PROFILE = "NFC_preserve_whitespace"
UNCERTAINTY_BOOTSTRAP_SAMPLES = 200
UNCERTAINTY_BOOTSTRAP_SEED = 1729
MIN_CLAIM_SCORING_UNITS = 30
RECOGNIZER_ARTIFACTS = (
"eval-run.json",
"metrics.json",
"metrics.md",
"predictions.tsv",
"unmatched.tsv",
"worst-lines.csv",
"sample-review.csv",
"sample-review.html",
"confusions.json",
"command.txt",
"claim-safety.md",
)
PAGE_ARTIFACTS = (
"page-eval-run.json",
"metrics.json",
"metrics.md",
"page-metrics.csv",
"reference-artifacts.tsv",
"text-pairs.tsv",
"missing.tsv",
"worst-pages.csv",
"confusions.json",
"sample-review.csv",
"sample-review.html",
"line-alignment.csv",
"command.txt",
"claim-safety.md",
)
RECIPE_ROOT_ARTIFACTS = (
"recipe-run.json",
"recipe.json",
"recipe-validation.json",
"audit.json",
"claim-safety.md",
"materialize/output-command-run.json",
"materialize/output-command-run.md",
"materialize/command-template.txt",
)
RECOGNIZER_REQUIRED_RUN_FIELDS = (
"created_at",
"model_name",
"checkpoint",
"config",
"dictionary",
"artifact_identity",
"label_file",
"label_file_sha256",
"prediction_file",
"prediction_file_sha256",
"data_dir",
"script_inventory_file",
"script_inventory_file_sha256",
"eval_pack_version",
"pack_content_sha256",
"frozen_eval_pack",
"unseen_for_selection",
"selected_checkpoint_on_test",
"normalization_profile",
"eval_mode",
"languages",
"scripts",
"domains",
"gt_statuses",
"splits",
"script_inventory",
"claim_safe",
"warnings",