from __future__ import annotations import csv import importlib.util import json import sys from argparse import Namespace from pathlib import Path from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest from ocrtech.schemas import BBox, Block, Document, Page, Table, TableCell, TextLine def _load_tool(name: str): path = Path(__file__).resolve().parents[1] / "tools" / f"{name}.py" spec = importlib.util.spec_from_file_location(name, path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _write_recognizer_fixture(tmp_path: Path) -> Namespace: scorer = _load_tool("run_recognizer_eval") pack = tmp_path / "pack" images = pack / "images" images.mkdir(parents=True) labels: list[str] = [] predictions: list[str] = [] manifest_rows: list[str] = [] digest_lines: list[object] = [] for index in range(1, 31): image_name = f"line-{index}.png" image_path = images / image_name image_path.write_bytes(f"eval-image-{index}".encode("utf-8")) image_sha256 = scorer._hash_file(image_path) text_sha256 = scorer._hash_text("abc") metadata = { "language": "fixture-lang", "script": "FixtureScript", "domain": "real_print", "gt_status": "human_verified", "split": "eval", "source_document": "fixture.pdf", "page": 1, "bbox": [1, index, 30, index + 20], "claim_evidence_eligible": True, "reviewed_by": "fixture-reviewer", } labels.append(f"images/{image_name}\tabc") predictions.append(f"{image_name}\tabc\t1.0") manifest_rows.append( json.dumps( { "sample_id": f"eval-{index}", "image_path": f"images/{image_name}", "sha256": image_sha256, "text_sha256": text_sha256, "sample_sha256": scorer._sample_hash(image_sha256, text_sha256), "metadata": metadata, } ) ) digest_lines.append( scorer.EvalLine( sample_id=f"eval-{index}", image_path=f"images/{image_name}", image_sha256=image_sha256, gt_text="abc", prediction="abc", prediction_score="1.0", metadata=metadata, ) ) (pack / "eval.txt").write_text("\n".join(labels) + "\n", encoding="utf-8") (pack / "manifest.jsonl").write_text("\n".join(manifest_rows) + "\n", encoding="utf-8") (pack / "summary.json").write_text( json.dumps( { "dataset": "fixture", "eval_pack_version": "fixture-v1", "pack_content_sha256": scorer._pack_content_digest( digest_lines, Namespace( language="fixture-lang", script="FixtureScript", domain="real_print", gt_status="human_verified", split="eval", claim_eligible=True, ), ), "eval_pack": {"lines": 30}, } ) + "\n", encoding="utf-8", ) (pack / "script-inventory.json").write_text( json.dumps({"required_codepoints": ["U+0061", "U+0062", "U+0063"], "required_graphemes": ["a", "b", "c"]}) + "\n", encoding="utf-8", ) (tmp_path / "preds.txt").write_text("\n".join(predictions) + "\n", encoding="utf-8") (tmp_path / "train.jsonl").write_text( json.dumps({"sample_id": "train-1", "image_path": "train.png", "text": "zzz", "sha256": "not-eval"}) + "\n", encoding="utf-8", ) (tmp_path / "checkpoint.pdparams").write_bytes(b"checkpoint") (tmp_path / "config.yml").write_text("model: fixture\n", encoding="utf-8") (tmp_path / "dict.txt").write_text("a\nb\nc\n", encoding="utf-8") return Namespace( label_file=pack / "eval.txt", data_dir=pack, prediction_file=tmp_path / "preds.txt", out=tmp_path / "recognizer-score", manifest_file=pack / "manifest.jsonl", summary_file=pack / "summary.json", script_inventory_file=pack / "script-inventory.json", spotcheck_calibration_file=None, train_manifest=[tmp_path / "train.jsonl"], model_name="fixture-model", checkpoint=str(tmp_path / "checkpoint.pdparams"), config=str(tmp_path / "config.yml"), dictionary=str(tmp_path / "dict.txt"), language="fixture-lang", script="FixtureScript", domain="real_print", gt_status="human_verified", split="eval", eval_pack_version="fixture-v1", frozen_eval_pack=True, unseen_for_selection=True, selected_checkpoint_on_test=False, claim_eligible=True, known_leakage="", min_match_rate=1.0, review_sample_size=5, require_full_match=True, require_claim_safe=True, force=False, ) def _refresh_recognizer_pack_digest(args: Namespace) -> None: scorer = _load_tool("run_recognizer_eval") label_by_path = {} for line in args.label_file.read_text(encoding="utf-8").splitlines(): rel_path, text = line.split("\t", 1) label_by_path[rel_path] = text rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] digest_lines: list[object] = [] for row in rows: rel_path = str(row.get("image_path") or row.get("image") or "") text = label_by_path[rel_path] image_sha256 = scorer._hash_file(args.data_dir / rel_path) text_sha256 = scorer._hash_text(text) row["sha256"] = image_sha256 row["text_sha256"] = text_sha256 row["sample_sha256"] = scorer._sample_hash(image_sha256, text_sha256) digest_lines.append( scorer.EvalLine( sample_id=str(row.get("sample_id") or row.get("id") or ""), image_path=rel_path, image_sha256=image_sha256, gt_text=text, prediction="", prediction_score="", metadata=dict(row.get("metadata") or {}), ) ) args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") summary = json.loads(args.summary_file.read_text(encoding="utf-8")) summary["pack_content_sha256"] = scorer._pack_content_digest(digest_lines, args) args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8") def _refresh_page_pack_digest(args: Namespace) -> None: scorer = _load_tool("run_page_ocr_eval") manifest = scorer.load_manifest(args.eval_manifest) manifest_dir = args.eval_manifest.parent digest_rows = [] for entry in manifest: input_path = scorer._resolve_manifest_path(entry.image_path, manifest_dir) reference_path = scorer._reference_path_for(entry, input_path, manifest_dir) reference = scorer._reference_for(entry, input_path, manifest_dir) reference_payload = scorer._reference_payload_for(entry, input_path, manifest_dir) reference_fields = scorer._reference_fields(reference) if scorer._reference_lines(reference_payload): reference_fields.append("line_bboxes") digest_rows.append( scorer.PageEvalRow( sample_id=entry.sample_id, input_path=str(input_path), input_sha256=sha256_file(input_path), document_path="", document_sha256="", document_source_path="", language=scorer._entry_language(entry, args.language), script=scorer._entry_script(entry, args.script), domain=scorer._entry_domain(entry, args.domain), gt_status=scorer._entry_gt_status(entry, args.gt_status), split=scorer._entry_split(entry), claim_eligible=scorer._entry_claim_eligible(entry, args.claim_eligible), source_document=scorer._metadata_value_for_keys(entry.metadata or {}, scorer.SOURCE_DOCUMENT_KEYS), source_page=scorer._metadata_value_for_keys(entry.metadata or {}, scorer.PAGE_KEYS), source_bbox=scorer._metadata_value_for_keys(entry.metadata or {}, scorer.BBOX_KEYS), reference_path=str(reference_path) if reference_path else "", reference_sha256=sha256_file(reference_path) if reference_path else "", reference_status=str((reference.metadata or {}).get("reference_status") or "") if reference else "", reference_claim_eligible=str((reference.metadata or {}).get("claim_evidence_eligible")) if reference and (reference.metadata or {}).get("claim_evidence_eligible") is not None else "", reference_text=reference.text if reference and reference.text is not None else "", reference_fields=reference_fields, ) ) summary = json.loads(args.summary_file.read_text(encoding="utf-8")) summary["pack_content_sha256"] = scorer._page_pack_content_digest(digest_rows) args.summary_file.write_text(json.dumps(summary, ensure_ascii=False) + "\n", encoding="utf-8") def _write_page_fixture(tmp_path: Path) -> Namespace: manifest_entries: list[ManifestEntry] = [] for index in range(1, 31): image = tmp_path / ("page.png" if index == 1 else f"page-{index}.png") image.write_bytes(f"page-image-{index}".encode("utf-8")) reference = tmp_path / ("page.ref.json" if index == 1 else f"page-{index}.ref.json") reference.write_text( json.dumps( { "text": "abc", "reading_order": [{"id": "l1", "text": "abc", "bbox": [0, 0, 50, 20], "page_index": 0}], "tables": [[["abc"]]], "metadata": {"claim_evidence_eligible": True}, } ) + "\n", encoding="utf-8", ) manifest_entries.append( ManifestEntry( sample_id=f"page-{index}", dataset="fixture-pages", split="eval", image_path=str(image), text="abc", sha256=sha256_file(image), metadata={ "reference_path": str(reference), "language": "fixture-lang", "script": "FixtureScript", "domain": "real_print", "gt_status": "human_verified", "source_document": "fixture-pages.pdf", "page": index, "bbox": [0, 0, 50, 50], "claim_evidence_eligible": True, "reviewed_by": "fixture-reviewer", }, ) ) write_manifest(manifest_entries, tmp_path / "manifest.jsonl") write_manifest( [ ManifestEntry( sample_id="train-1", dataset="fixture-pages", split="train", image_path="train.png", text="zzz", sha256="not-eval", ) ], tmp_path / "train.jsonl", ) table = Table( table_id="table-1", page_index=0, bbox=BBox(0, 0, 50, 20), cells=[TableCell(row=0, col=0, text="abc")], ) document = Document( source_path=str(image), pages=[ Page( page_index=0, text_lines=[TextLine(text="abc", bbox=BBox(0, 0, 50, 20), page_index=0, line_id="l1")], blocks=[ Block( block_id="b1", block_type="table", page_index=0, bbox=BBox(0, 0, 50, 20), order=0, text="abc", line_ids=["l1"], table_id=table.table_id, ) ], ) ], tables=[table], ) for index in range(1, 31): doc_dir = tmp_path / "documents" / f"page-{index}" doc_dir.mkdir(parents=True) page_document = Document( source_path=str(tmp_path / ("page.png" if index == 1 else f"page-{index}.png")), pages=document.pages, tables=document.tables, ) (doc_dir / "document.json").write_text(json.dumps(page_document.to_dict(), ensure_ascii=False) + "\n", encoding="utf-8") (tmp_path / "summary.json").write_text( json.dumps({"dataset": "fixture-pages", "eval_pack_version": "page-v1", "eval_pack": {"pages": 30}}) + "\n", encoding="utf-8", ) (tmp_path / "script-inventory.json").write_text( json.dumps({"required_codepoints": ["U+0061", "U+0062", "U+0063"], "required_graphemes": ["a", "b", "c"]}) + "\n", encoding="utf-8", ) (tmp_path / "checkpoint.pdparams").write_bytes(b"checkpoint") (tmp_path / "config.yml").write_text("model: fixture\n", encoding="utf-8") (tmp_path / "dict.txt").write_text("abc\n", encoding="utf-8") args = Namespace( eval_manifest=tmp_path / "manifest.jsonl", documents_dir=tmp_path / "documents", ocr_output_kind="sidecar_fixture", out=tmp_path / "page-score", summary_file=tmp_path / "summary.json", script_inventory_file=tmp_path / "script-inventory.json", spotcheck_calibration_file=None, train_manifest=[tmp_path / "train.jsonl"], system_name="fixture-page", model_name="fixture-model", checkpoint=str(tmp_path / "checkpoint.pdparams"), config=str(tmp_path / "config.yml"), dictionary=str(tmp_path / "dict.txt"), language="fixture-lang", script="FixtureScript", domain="real_print", gt_status="human_verified", eval_pack_version="page-v1", frozen_eval_pack=True, unseen_for_selection=True, selected_checkpoint_on_test=False, claim_eligible=True, line_iou_threshold=0.5, review_sample_size=5, require_claim_safe=True, force=False, ) _refresh_page_pack_digest(args) return args def _write_page_output_audits(documents_dir: Path, *, rows: int = 30, passed: bool = True) -> None: for index in range(1, rows + 1): output_dir = documents_dir / f"page-{index}" payload = { "passed": passed, "issues": [] if passed else ["fixture failure"], "output_dir": str(output_dir), } (output_dir / "limbu-output-audit.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") def _write_recipe_root_artifacts(tmp_path: Path, audit_tool, score_dir: Path, prediction_file: Path) -> None: root = tmp_path / "recipe-run" materialize = root / "materialize" materialize.mkdir(parents=True, exist_ok=True) recipes = root / "recipes.json" eval_run = json.loads((score_dir / "eval-run.json").read_text(encoding="utf-8")) recipe = { "id": "fixture-recipe", "task": "recognizer_oracle_crop", "language": eval_run["languages"][0], "script": eval_run["scripts"][0], "model_family": eval_run["model_name"], "runner": { "type": "external_command", "command_template": "python predict.py --labels {label_file} --data {data_dir} --out {prediction_file}", "requires_operator_resolution": False, "checkpoint": eval_run["checkpoint"], "config": eval_run["config"], "dictionary": eval_run["dictionary"], }, "eval_pack": { "label_file": eval_run["label_file"], "data_dir": eval_run["data_dir"], "normalization_profile": eval_run["normalization_profile"], "manifest_file": eval_run["manifest_file"], "summary_file": eval_run["summary_file"], "script_inventory_file": eval_run["script_inventory_file"], "dataset": "fixture", "domain": eval_run["domains"][0], "gt_status": eval_run["gt_statuses"][0], "split": eval_run["splits"][0], "eval_pack_version": eval_run["eval_pack_version"], "frozen_eval_pack": eval_run["frozen_eval_pack"], "unseen_for_selection": eval_run["unseen_for_selection"], "claim_eligible": eval_run["claim_eligible_default"], }, "leakage_sources": [{"manifest_file": item["path"], "split": "train"} for item in eval_run["train_manifests"]], "claim_blockers": [], } recipes.write_text(json.dumps({"recipes": [recipe]}) + "\n", encoding="utf-8") validator = _load_tool("validate_eval_harness_recipes") recipe_validation = validator.validate_recipes(recipes, tmp_path)["recipes"][0] (root / "recipe.json").write_text(json.dumps(recipe) + "\n", encoding="utf-8") (root / "recipe-validation.json").write_text(json.dumps(recipe_validation) + "\n", encoding="utf-8") root_prediction = root / "predictions.txt" root_prediction.write_text(prediction_file.read_text(encoding="utf-8"), encoding="utf-8") stdout_path = materialize / "stdout.txt" stderr_path = materialize / "stderr.txt" stdout_path.write_text("", encoding="utf-8") stderr_path.write_text("", encoding="utf-8") materialization = { "created_at": "2026-06-20T00:00:00+00:00", "mode": "recognizer", "system_name": "fixture-recipe", "model_name": eval_run["model_name"], "model_config": None, "checkpoint": eval_run["checkpoint"], "config": eval_run["config"], "dictionary": eval_run["dictionary"], "command_template": "fixture", "timeout_seconds": None, "label_file": eval_run["label_file"], "data_dir": eval_run["data_dir"], "prediction_file": str(root_prediction), "record_count": 1, "records": [ { "sample_id": "__recognizer_eval__", "command": ["fixture"], "stdout_path": str(stdout_path), "stdout_sha256": sha256_file(stdout_path), "stderr_path": str(stderr_path), "stderr_sha256": sha256_file(stderr_path), "returncode": 0, "output_path": str(root_prediction), "output_sha256": sha256_file(root_prediction), "output_size_bytes": root_prediction.stat().st_size, } ], "machine": {"platform": "fixture", "python": "fixture"}, "invocation": "fixture", } (materialize / "output-command-run.json").write_text(json.dumps(materialization) + "\n", encoding="utf-8") (materialize / "output-command-run.md").write_text( "\n".join( [ "# Eval Output Command Run", "", "- records: 1", "", "## Outputs", "", ( f"- __recognizer_eval__: `{root_prediction}` sha256={sha256_file(root_prediction)}; " f"stdout `{stdout_path}` sha256={sha256_file(stdout_path)}; " f"stderr `{stderr_path}` sha256={sha256_file(stderr_path)}" ), ] ) + "\n", encoding="utf-8", ) (materialize / "command-template.txt").write_text("fixture\n", encoding="utf-8") score_audit = audit_tool.audit_run(score_dir) (root / "audit.json").write_text(json.dumps(score_audit) + "\n", encoding="utf-8") recipe_run = { "created_at": "2026-06-20T00:00:00+00:00", "recipe_id": "fixture-recipe", "recipe_file": str(recipes), "recipe_file_sha256": sha256_file(recipes), "root": str(tmp_path), "outputs": { "prediction_file": str(root_prediction), "materialization_dir": str(materialize), "score_dir": str(score_dir), "recipe_file": str(root / "recipe.json"), "recipe_validation_file": str(root / "recipe-validation.json"), "audit_file": str(root / "audit.json"), "claim_safety_report": str(root / "claim-safety.md"), }, "train_manifests": eval_run["train_manifests"], "recipe_validation": recipe_validation, "materialization": materialization, "audit": { "audit_passed": score_audit["audit_passed"], "claim_safe": score_audit["claim_safe"], "issues": score_audit["issues"], }, "score": { "claim_safe": eval_run["claim_safe"], "warnings": eval_run["warnings"], "metrics_file": str(score_dir / "metrics.json"), "metrics_file_sha256": sha256_file(score_dir / "metrics.json"), "eval_run_file": str(score_dir / "eval-run.json"), "eval_run_file_sha256": sha256_file(score_dir / "eval-run.json"), }, "machine": {"platform": "fixture", "python": "fixture"}, "command": "fixture", } (root / "recipe-run.json").write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") metrics = json.loads((score_dir / "metrics.json").read_text(encoding="utf-8")) (root / "claim-safety.md").write_text( "\n".join( [ "# Eval Claim Safety Report", "", "- recipe: fixture-recipe", f"- task: {recipe['task']}", f"- audit passed: {score_audit['audit_passed']}", f"- claim safe: {eval_run['claim_safe']}", f"- eval mode: {eval_run['eval_mode']}", f"- score created at: {eval_run['created_at']}", f"- score machine platform: {eval_run['machine']['platform']}", f"- score machine python: {eval_run['machine']['python']}", f"- score command: {eval_run['command']}", "", "## What Was Scored", "", f"- model: {eval_run['model_name']}", f"- checkpoint: {eval_run['checkpoint'] or '(not recorded)'}", f"- config: {eval_run['config'] or '(not recorded)'}", f"- dictionary: {eval_run['dictionary'] or '(not recorded)'}", f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}", f"- normalization profile: {eval_run['normalization_profile']}", f"- label file: {eval_run['label_file']}", f"- prediction file: {eval_run['prediction_file']}", f"- scored lines: {metrics['n_lines']}", f"- unmatched labels: {metrics['unmatched_labels']}", f"- unmatched predictions: {metrics['unmatched_predictions']}", "", "## Ground Truth And Pack", "", f"- eval pack version: {eval_run['eval_pack_version'] or '(missing)'}", f"- frozen eval pack: {eval_run['frozen_eval_pack']}", f"- unseen for selection: {eval_run['unseen_for_selection']}", f"- selected checkpoint on test: {eval_run['selected_checkpoint_on_test']}", f"- languages: {eval_run['languages']}", f"- scripts: {eval_run['scripts']}", f"- domains: {eval_run['domains']}", f"- GT statuses: {eval_run['gt_statuses']}", f"- splits: {eval_run['splits']}", f"- script inventory file: {metrics['script_inventory']['inventory_file']}", f"- observed codepoints/graphemes: {metrics['script_inventory']['observed_codepoint_count']}/{metrics['script_inventory']['observed_grapheme_count']}", f"- missing required codepoints: {metrics['script_inventory']['missing_required_codepoints']}", f"- missing required graphemes: {metrics['script_inventory']['missing_required_graphemes']}", f"- metadata contradictions: {metrics['metadata_contradictions']}", f"- provenance coverage: {metrics['provenance_coverage']}", f"- normalizer counts: {metrics.get('normalizer_counts', {})}", f"- normalization changed GT/prediction/any: {metrics['normalization_summary']['gt_changed']}/{metrics['normalization_summary']['prediction_changed']}/{metrics['normalization_summary']['any_changed']} of {metrics['normalization_summary']['total']}", f"- dictionary coverage checked: {metrics['dictionary_coverage']['checked']}", f"- dictionary GT codepoints/dictionary codepoints: {metrics['dictionary_coverage']['gt_codepoint_count']}/{metrics['dictionary_coverage']['dictionary_codepoint_count']}", f"- dictionary missing GT codepoints: {metrics['dictionary_coverage']['missing_gt_codepoints']}", f"- leakage checked: {metrics['leakage']['checked']}", "", "## Metrics", "", f"- codepoint CER micro: {metrics['cer_codepoint_micro']:.6f}", f"- grapheme CER micro: {metrics['cer_grapheme_micro']:.6f}", f"- exact-line accuracy: {metrics['exact_line_accuracy']:.6f}", f"- codepoint edits/ref: {metrics['codepoint_edits']}/{metrics['codepoint_ref_len']}", f"- grapheme edits/ref: {metrics['grapheme_edits']}/{metrics['grapheme_ref_len']}", f"- zero-length GT lines: {metrics['zero_length_gt_lines']}", f"- uncertainty method: {metrics['uncertainty']['method']}", f"- uncertainty scoring units: {metrics['uncertainty']['effective_sample_size']}/{metrics['uncertainty']['min_claim_scoring_units']}", f"- grapheme CER 95% CI: {metrics['uncertainty']['metrics']['cer_grapheme_micro']['low']:.6f}..{metrics['uncertainty']['metrics']['cer_grapheme_micro']['high']:.6f}", "", "## Slices", "", f"- {metrics['slices'][0]['language']} / {metrics['slices'][0]['script']} / {metrics['slices'][0]['domain']} / {metrics['slices'][0]['gt_status']} / {metrics['slices'][0]['split']}: " f"n={metrics['slices'][0]['n_lines']} graph_CER={metrics['slices'][0]['cer_grapheme_micro']:.6f} " f"cp_CER={metrics['slices'][0]['cer_codepoint_micro']:.6f} exact={metrics['slices'][0]['exact_line_accuracy']:.6f}", "", "## Review Artifacts", "", "- audit JSON: audit.json", "- eval run JSON: score/eval-run.json", "- metrics JSON: score/metrics.json", "- metrics Markdown: score/metrics.md", "- command text: score/command.txt", "- predictions TSV: score/predictions.tsv", "- unmatched TSV: score/unmatched.tsv", "- worst lines: score/worst-lines.csv", "- review CSV: score/sample-review.csv", "- review HTML: score/sample-review.html", "- confusions JSON: score/confusions.json", "", "## Warnings", "", "- none", ] ) + "\n", encoding="utf-8", ) def test_audit_eval_run_accepts_claim_safe_recognizer_score(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) result = audit_tool.audit_run(args.out) assert result["eval_mode"] == "recognizer_oracle_crop" assert result["audit_passed"] is True assert result["claim_safe"] is True assert result["issues"] == [] def test_audit_eval_run_accepts_claim_safe_page_score(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) result = audit_tool.audit_run(args.out) assert result["eval_mode"] == "page_level_ocr" assert result["audit_passed"] is True assert result["claim_safe"] is True assert result["issues"] == [] def test_audit_eval_run_replays_page_output_audit_artifacts(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.ocr_output_kind = "live_detector" _write_page_output_audits(args.documents_dir) scorer.run_eval(args) clean = audit_tool.audit_run(args.out) assert clean["audit_passed"] is True (args.documents_dir / "page-1" / "limbu-output-audit.json").unlink() result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page_output_audit does not match current page output audit artifacts" in issue for issue in result["issues"]) assert any("replayed page output audits are not all passing: 29/30" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_recognizer_run_identity_lists(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["languages"] = ["wrong-lang"] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("eval-run.json languages does not match predictions.tsv language values" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_page_run_identity_lists(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["scripts"] = ["WrongScript"] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page-eval-run.json scripts does not match page-metrics.csv script values" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_recognizer_normalization_profile(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["normalization_profile"] = "NFKC_strip_whitespace" eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("normalization_profile does not match supported metric recomputation profile" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_page_normalization_profile(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["normalization_profile"] = "NFKC_strip_whitespace" eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("normalization_profile does not match supported metric recomputation profile" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_incomplete_recognizer_machine_metadata(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["machine"] = {"platform": ""} eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "eval run machine.platform is missing" in result["issues"] assert "eval run machine.python is missing" in result["issues"] def test_audit_eval_run_rejects_incomplete_page_machine_metadata(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["machine"] = {"python": ""} eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "eval run machine.platform is missing" in result["issues"] assert "eval run machine.python is missing" in result["issues"] def test_audit_eval_run_rejects_page_spotchecked_gt_without_calibration(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()] for row in rows: row["metadata"]["gt_status"] = "machine_converted_spot_checked" args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") _refresh_page_pack_digest(args) scorer.run_eval(args) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page spot-checked machine GT is missing calibration summary" in result["issues"] def test_audit_eval_run_recomputes_recognizer_spotcheck_calibration_summary(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] for row in rows: row["metadata"]["gt_status"] = "machine_converted_spot_checked" args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") args.gt_status = "machine_converted_spot_checked" _refresh_recognizer_pack_digest(args) calibration = tmp_path / "calibration.json" calibration.write_text( json.dumps( { "claim_evidence_eligible": True, "recommended_pack_gt_status": "machine_converted_spot_checked", "language": "fixture-lang", "script": "FixtureScript", "domain": "real_print", "source_gt_status": "machine_converted", "counts": {"usable_rows": 1}, "machine_label_metrics": {"cer_grapheme_micro": 0.0}, "eligibility_reasons": [], } ) + "\n", encoding="utf-8", ) args.spotcheck_calibration_file = calibration scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run["spotcheck_calibration"]["claim_evidence_eligible"] = False eval_run["spotcheck_calibration"]["language"] = "wrong-language" metrics["spotcheck_calibration"]["counts"] = {"usable_rows": 0} eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer eval-run spotcheck_calibration does not match calibration file" in issue for issue in result["issues"]) assert any("recognizer metrics spotcheck_calibration does not match calibration file" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_spotcheck_calibration_summary(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()] for row in rows: row["metadata"]["gt_status"] = "machine_converted_spot_checked" args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") args.gt_status = "machine_converted_spot_checked" _refresh_page_pack_digest(args) calibration = tmp_path / "calibration.json" calibration.write_text( json.dumps( { "claim_evidence_eligible": True, "recommended_pack_gt_status": "machine_converted_spot_checked", "language": "fixture-lang", "script": "FixtureScript", "domain": "real_print", "source_gt_status": "machine_converted", "counts": {"usable_rows": 1}, "machine_label_metrics": {"cer_grapheme_micro": 0.0}, "eligibility_reasons": [], } ) + "\n", encoding="utf-8", ) args.spotcheck_calibration_file = calibration scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run["spotcheck_calibration"]["claim_evidence_eligible"] = False eval_run["spotcheck_calibration"]["language"] = "wrong-language" metrics["spotcheck_calibration"]["counts"] = {"usable_rows": 0} eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page eval-run spotcheck_calibration does not match calibration file" in issue for issue in result["issues"]) assert any("page metrics spotcheck_calibration does not match calibration file" in issue for issue in result["issues"]) def test_audit_eval_run_reports_missing_review_artifact(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) (args.out / "sample-review.html").unlink() result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "missing required artifact: sample-review.html" in result["issues"] def test_audit_eval_run_recomputes_recognizer_worst_review_artifacts(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) worst_path = args.out / "worst-lines.csv" with worst_path.open("r", encoding="utf-8", newline="") as handle: worst_rows = list(csv.DictReader(handle)) fieldnames = list(worst_rows[0]) worst_rows[0]["sample_id"] = "not-the-worst" with worst_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(worst_rows) review_path = args.out / "sample-review.csv" with review_path.open("r", encoding="utf-8", newline="") as handle: review_rows = list(csv.DictReader(handle)) review_fieldnames = list(review_rows[0]) review_rows[0]["sample_id"] = "not-the-worst" with review_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=review_fieldnames) writer.writeheader() writer.writerows(review_rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("worst-lines.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"]) assert any("sample-review.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_recognizer_worst_fields(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) worst_path = args.out / "worst-lines.csv" with worst_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) fieldnames = list(rows[0]) rows[0]["language"] = "wrong-language" rows[0]["source_document"] = "wrong.pdf" rows[0]["image_sha256"] = "bad-hash" with worst_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "worst-lines.csv eval-1 language mismatch" in result["issues"] assert "worst-lines.csv eval-1 source_document mismatch" in result["issues"] assert "worst-lines.csv eval-1 image_sha256 mismatch" in result["issues"] def test_audit_eval_run_rejects_stale_recognizer_review_html(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) (args.out / "sample-review.html").write_text( "

Wrong review sample

\n", encoding="utf-8", ) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("sample-review.html missing recognizer review sample eval-1 fields" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_recognizer_review_csv_fields(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) review_path = args.out / "sample-review.csv" with review_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) fieldnames = list(rows[0]) rows[0]["gt_text"] = "stale-gt" rows[0]["prediction_score"] = "0.123" with review_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "sample-review.csv eval-1 gt_text mismatch" in result["issues"] assert "sample-review.csv eval-1 prediction_score mismatch" in result["issues"] def test_audit_eval_run_rejects_duplicate_recognizer_output_ids(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) predictions_path = args.out / "predictions.tsv" with predictions_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle, delimiter="\t")) fieldnames = list(rows[0]) rows.append(dict(rows[0])) with predictions_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t") writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "predictions.tsv duplicate sample_id values: ['eval-1']" in result["issues"] assert "predictions.tsv duplicate image_path values: ['images/line-1.png']" in result["issues"] def test_audit_eval_run_recomputes_recognizer_unmatched_rows(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_full_match = False args.require_claim_safe = False args.prediction_file.write_text("other-line.png\tzzz\t0.1\n", encoding="utf-8") scorer.run_eval(args) (args.out / "unmatched.tsv").write_text( "kind\timage_key\timage_path\timage_sha256\ttext\tprediction_score\n" "label_without_prediction\tline-1.png\timages/not-line-1.png\tbad-hash\tabc\t\n" "prediction_without_label\tother-line.png\tother-line.png\t\twrong-text\t0.9\n", encoding="utf-8", ) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("unmatched.tsv rows do not match recomputed label/prediction misses" in issue for issue in result["issues"]) def test_audit_eval_run_reports_missing_direct_claim_report(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) (args.out / "claim-safety.md").unlink() result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "missing required artifact: claim-safety.md" in result["issues"] def test_audit_eval_run_reports_missing_page_review_artifact(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) (args.out / "sample-review.html").unlink() result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "missing required artifact: sample-review.html" in result["issues"] def test_audit_eval_run_recomputes_page_worst_review_artifacts(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) worst_path = args.out / "worst-pages.csv" with worst_path.open("r", encoding="utf-8", newline="") as handle: worst_rows = list(csv.DictReader(handle)) fieldnames = list(worst_rows[0]) worst_rows[0]["sample_id"] = "not-the-worst-page" with worst_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(worst_rows) review_path = args.out / "sample-review.csv" with review_path.open("r", encoding="utf-8", newline="") as handle: review_rows = list(csv.DictReader(handle)) review_fieldnames = list(review_rows[0]) review_rows[0]["sample_id"] = "not-the-worst-page" with review_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=review_fieldnames) writer.writeheader() writer.writerows(review_rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("worst-pages.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"]) assert any("sample-review.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_page_worst_fields(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) worst_path = args.out / "worst-pages.csv" with worst_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) fieldnames = list(rows[0]) rows[0]["language"] = "wrong-language" rows[0]["source_document"] = "wrong.pdf" rows[0]["reference_sha256"] = "bad-hash" with worst_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "worst-pages.csv page-1 language mismatch" in result["issues"] assert "worst-pages.csv page-1 source_document mismatch" in result["issues"] assert "worst-pages.csv page-1 reference_sha256 mismatch" in result["issues"] def test_audit_eval_run_rejects_stale_page_review_html(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) (args.out / "sample-review.html").write_text( "

Wrong page review sample

\n", encoding="utf-8", ) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("sample-review.html missing page review sample page-1 fields" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_page_review_csv_fields(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) review_path = args.out / "sample-review.csv" with review_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) fieldnames = list(rows[0]) rows[0]["input_path"] = "stale-page.png" rows[0]["text_cer_grapheme"] = "0.5" with review_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "sample-review.csv page-1 input_path mismatch" in result["issues"] assert any("sample-review.csv page-1 text_cer_grapheme mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_duplicate_page_output_ids(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) page_metrics_path = args.out / "page-metrics.csv" with page_metrics_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) fieldnames = list(rows[0]) rows.append(dict(rows[0])) with page_metrics_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page-metrics.csv duplicate sample_id values: ['page-1']" in result["issues"] assert any("page-metrics.csv duplicate input_path values" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_duplicate_page_manifest_image_paths(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()] rows[1]["image_path"] = rows[0]["image_path"] args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page eval manifest duplicate image_path" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_page_manifest_image_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()] rows[0]["sha256"] = "0" * 64 args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page eval manifest page-1 sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_missing_rows(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False (args.documents_dir / "page-1" / "document.json").unlink() scorer.run_eval(args) (args.out / "missing.tsv").write_text("kind,sample_id\nmissing_document,not-page-1\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("missing.tsv rows do not match recomputed manifest/document/reference misses" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_page_metric_identity_fields(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) page_metrics_path = args.out / "page-metrics.csv" with page_metrics_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) assert rows fieldnames = list(rows[0]) rows[0]["claim_eligible"] = "False" rows[0]["source_document"] = "wrong.pdf" rows[0]["source_page"] = "99" rows[0]["source_bbox"] = "[9,9,9,9]" rows[0]["reference_fields"] = "text" with page_metrics_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any( "page-metrics.csv page-1 claim_eligible does not match eval manifest/reference" in issue for issue in result["issues"] ) assert any( "page-metrics.csv page-1 reference_fields does not match eval manifest/reference" in issue for issue in result["issues"] ) assert any( "page-metrics.csv page-1 source_document does not match eval manifest/reference" in issue for issue in result["issues"] ) assert any( "page-metrics.csv page-1 source_page does not match eval manifest/reference" in issue for issue in result["issues"] ) assert any( "page-metrics.csv page-1 source_bbox does not match eval manifest/reference" in issue for issue in result["issues"] ) def test_audit_eval_run_reports_malformed_direct_claim_report(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) (args.out / "claim-safety.md").write_text("# incomplete\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "claim-safety.md missing section: ## What Was Scored" in result["issues"] def test_audit_eval_run_rejects_stale_recognizer_claim_report(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) report_path = args.out / "claim-safety.md" report = report_path.read_text(encoding="utf-8") assert "- claim safe: True" in report assert "- dictionary missing GT codepoints: []" in report eval_run = json.loads((args.out / "eval-run.json").read_text(encoding="utf-8")) command_line = f"- command: {eval_run['command']}" artifact_line = f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}" assert command_line in report assert artifact_line in report assert "- sample-review.html" in report expected_slice = "- fixture-lang / FixtureScript / real_print / human_verified / eval: n=30 graph_CER=0.000000 cp_CER=0.000000 exact=1.000000" assert expected_slice in report report = report.replace("- claim safe: True", "- claim safe: False") report = report.replace(command_line, "- command: stale") report = report.replace(artifact_line, "- artifact identity: {}") report = report.replace("- sample-review.html", "- stale-review.html") report = report.replace("- dictionary missing GT codepoints: []", "- dictionary missing GT codepoints: ['U+1234']") report = report.replace(expected_slice, expected_slice.replace("n=30", "n=0")) report_path.write_text(report, encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "claim-safety.md missing or stale line: - claim safe: True" in result["issues"] assert f"claim-safety.md missing or stale line: {command_line}" in result["issues"] assert f"claim-safety.md missing or stale line: {artifact_line}" in result["issues"] assert "claim-safety.md missing or stale line: - sample-review.html" in result["issues"] assert "claim-safety.md missing or stale line: - dictionary missing GT codepoints: []" in result["issues"] assert f"claim-safety.md missing or stale line: {expected_slice}" in result["issues"] def test_audit_eval_run_rejects_stale_page_claim_report(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) report_path = args.out / "claim-safety.md" report = report_path.read_text(encoding="utf-8") assert "- scored pages: 30" in report assert "- normalization changed reference/prediction/any: 0/0/0 of 30" in report eval_run = json.loads((args.out / "page-eval-run.json").read_text(encoding="utf-8")) machine_line = f"- machine platform: {eval_run['machine']['platform']}" artifact_line = f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}" assert machine_line in report assert artifact_line in report assert "- line-alignment.csv" in report expected_slice = "- fixture-lang / FixtureScript / real_print / human_verified / eval: pages=30 text_graph_CER=0.000000 text_cp_CER=0.000000 exact=1.000000 line_F1=1.000000" assert expected_slice in report report = report.replace("- scored pages: 30", "- scored pages: 0") report = report.replace(machine_line, "- machine platform: stale") report = report.replace(artifact_line, "- artifact identity: {}") report = report.replace("- line-alignment.csv", "- stale-line-alignment.csv") report = report.replace( "- normalization changed reference/prediction/any: 0/0/0 of 30", "- normalization changed reference/prediction/any: 1/0/1 of 30", ) report = report.replace(expected_slice, expected_slice.replace("pages=30", "pages=0")) report_path.write_text(report, encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "claim-safety.md missing or stale line: - scored pages: 30" in result["issues"] assert f"claim-safety.md missing or stale line: {machine_line}" in result["issues"] assert f"claim-safety.md missing or stale line: {artifact_line}" in result["issues"] assert "claim-safety.md missing or stale line: - line-alignment.csv" in result["issues"] assert "claim-safety.md missing or stale line: - normalization changed reference/prediction/any: 0/0/0 of 30" in result["issues"] assert f"claim-safety.md missing or stale line: {expected_slice}" in result["issues"] def test_audit_eval_run_checks_page_recipe_root_claim_safety_report(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") recipe_tool = _load_tool("run_eval_recipe") args = _write_page_fixture(tmp_path) score = scorer.run_eval(args) root = tmp_path / "page-recipe-root" root.mkdir() recipe = {"id": "page-recipe", "task": "page_level_ocr"} (root / "recipe.json").write_text(json.dumps(recipe) + "\n", encoding="utf-8") recipe_run = {"recipe_id": "page-recipe", "audit": {"audit_passed": True}} report_path = root / "claim-safety.md" recipe_tool._write_claim_safety_report(report_path, "page-recipe", recipe, score, {"audit_passed": True}) issues = audit_tool._check_root_claim_safety_report_consistency(root, recipe_run, score["eval_run"], score["metrics"]) assert issues == [] report = report_path.read_text(encoding="utf-8") assert "- eval manifest: " + str(args.eval_manifest) in report assert "- scored pages: 30" in report assert "- text grapheme CER micro: 0.000000" in report assert "- normalization changed reference/prediction/any: 0/0/0 of 30" in report assert "- missing required codepoints: []" in report assert "- text grapheme CER 95% CI: 0.000000..0.000000" in report assert "- line alignment CSV: score/line-alignment.csv" in report assert "- confusions JSON: score/confusions.json" in report command_line = f"- score command: {score['eval_run']['command']}" artifact_line = f"- artifact identity: {json.dumps(score['eval_run'].get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}" assert command_line in report assert artifact_line in report expected_slice = "- fixture-lang / FixtureScript / real_print / human_verified / eval: pages=30 text_graph_CER=0.000000 text_cp_CER=0.000000 exact=1.000000 line_F1=1.000000" assert expected_slice in report report = report.replace("- scored pages: 30", "- scored pages: 0") report = report.replace(command_line, "- score command: stale") report = report.replace(artifact_line, "- artifact identity: {}") report = report.replace("- missing required codepoints: []", "- missing required codepoints: ['U+1234']") report = report.replace("- text grapheme CER 95% CI: 0.000000..0.000000", "- text grapheme CER 95% CI: 0.100000..0.200000") report = report.replace("- line alignment CSV: score/line-alignment.csv", "- line alignment CSV: stale.csv") report = report.replace( "- normalization changed reference/prediction/any: 0/0/0 of 30", "- normalization changed reference/prediction/any: 1/0/1 of 30", ) report = report.replace(expected_slice, expected_slice.replace("pages=30", "pages=0")) report_path.write_text(report, encoding="utf-8") stale_issues = audit_tool._check_root_claim_safety_report_consistency(root, recipe_run, score["eval_run"], score["metrics"]) assert "root claim-safety.md missing or stale line: - scored pages: 30" in stale_issues assert f"root claim-safety.md missing or stale line: {command_line}" in stale_issues assert f"root claim-safety.md missing or stale line: {artifact_line}" in stale_issues assert "root claim-safety.md missing or stale line: - missing required codepoints: []" in stale_issues assert "root claim-safety.md missing or stale line: - text grapheme CER 95% CI: 0.000000..0.000000" in stale_issues assert "root claim-safety.md missing or stale line: - line alignment CSV: score/line-alignment.csv" in stale_issues assert "root claim-safety.md missing or stale line: - normalization changed reference/prediction/any: 0/0/0 of 30" in stale_issues assert f"root claim-safety.md missing or stale line: {expected_slice}" in stale_issues def test_audit_eval_run_checks_page_recipe_root_document_lineage(tmp_path: Path) -> None: audit_tool = _load_tool("audit_eval_run") root = tmp_path / "page-recipe-root" materialize = root / "materialize" documents_dir = root / "documents" document_json = documents_dir / "page-1" / "document.json" materialize.mkdir(parents=True) document_json.parent.mkdir(parents=True) document_json.write_text('{"pages":[]}\n', encoding="utf-8") for rel_path in audit_tool.RECIPE_ROOT_ARTIFACTS: path = root / rel_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text("fixture\n", encoding="utf-8") assert not (root / "predictions.txt").exists() assert audit_tool._check_required_artifacts(root, audit_tool.RECIPE_ROOT_ARTIFACTS) == [] materialization = { "records": [ { "sample_id": "page-1", "output_path": str(document_json), "output_sha256": audit_tool._sha256_file(document_json), } ], "command_template": "fixture", } (materialize / "output-command-run.json").write_text(json.dumps(materialization) + "\n", encoding="utf-8") (materialize / "command-template.txt").write_text("fixture\n", encoding="utf-8") (materialize / "output-command-run.md").write_text( f"- records: 1\n- page-1: `{document_json}` sha256={audit_tool._sha256_file(document_json)}\n", encoding="utf-8", ) recipe_run = { "outputs": {"documents_dir": str(documents_dir)}, "materialization": materialization, } eval_run = {"eval_mode": "page_level_ocr", "documents_dir": str(documents_dir)} issues = audit_tool._check_recipe_output_lineage(root, recipe_run, eval_run) assert issues == [] recipe_run["outputs"]["documents_dir"] = str(root / "wrong-documents") stale_issues = audit_tool._check_recipe_output_lineage(root, recipe_run, eval_run) assert "recipe-run.json outputs.documents_dir does not match score eval-run documents_dir" in stale_issues assert any("output path does not exist for documents_dir" in issue for issue in stale_issues) recipe_run["outputs"]["documents_dir"] = str(documents_dir) outside = root / "outside-document.json" outside.write_text("{}\n", encoding="utf-8") materialization["records"][0]["output_path"] = str(outside) outside_issues = audit_tool._check_recipe_output_lineage(root, recipe_run, eval_run) assert "recipe-run.json materialization record 0 output_path is not under outputs.documents_dir" in outside_issues def test_audit_eval_run_checks_page_materialization_input_identity(tmp_path: Path) -> None: audit_tool = _load_tool("audit_eval_run") page_image = tmp_path / "page.png" page_image.write_bytes(b"page image") document_json = tmp_path / "documents" / "page-1" / "document.json" document_json.parent.mkdir(parents=True) document_json.write_text('{"pages":[]}\n', encoding="utf-8") stdout_path = tmp_path / "stdout.txt" stderr_path = tmp_path / "stderr.txt" stdout_path.write_text("", encoding="utf-8") stderr_path.write_text("", encoding="utf-8") materialize_dir = tmp_path / "materialize" payload = { "mode": "page", "command_template": "fixture {input_path} {sample_id} {output_dir} {document_json} {documents_dir} {out_dir}", "documents_dir": str(document_json.parent.parent), "record_count": 1, "records": [ { "sample_id": "page-1", "command": [ "fixture", str(page_image), "page-1", str(document_json.parent), str(document_json), str(document_json.parent.parent), str(materialize_dir), ], "stdout_path": str(stdout_path), "stdout_sha256": audit_tool._sha256_file(stdout_path), "stderr_path": str(stderr_path), "stderr_sha256": audit_tool._sha256_file(stderr_path), "returncode": 0, "input_path": str(page_image), "input_sha256": audit_tool._sha256_file(page_image), "output_path": str(document_json), "output_sha256": audit_tool._sha256_file(document_json), "output_size_bytes": document_json.stat().st_size, } ], } assert audit_tool._check_materialization_records(payload, materialize_dir) == [] payload["records"][0]["command"] = ["stale-command"] command_issues = audit_tool._check_materialization_records(payload, materialize_dir) assert "materialization records[0] command does not match materialization command_template" in command_issues payload["records"][0]["command"] = [ "fixture", str(page_image), "page-1", str(document_json.parent), str(document_json), str(document_json.parent.parent), str(materialize_dir), ] payload["records"][0]["input_sha256"] = "0" * 64 issues = audit_tool._check_materialization_records(payload) assert any("materialization records[0] input_sha256 mismatch" in issue for issue in issues) def test_audit_eval_run_rejects_stale_recognizer_metrics_report(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) report_path = args.out / "metrics.md" report = report_path.read_text(encoding="utf-8") assert "- codepoint CER: 0.000000" in report assert "- normalization changed GT/prediction/any: 0/0/0 of 30" in report report = report.replace("- codepoint CER: 0.000000", "- codepoint CER: 0.500000") report = report.replace( "- normalization changed GT/prediction/any: 0/0/0 of 30", "- normalization changed GT/prediction/any: 1/0/1 of 30", ) report_path.write_text(report, encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "metrics.md missing or stale line: - codepoint CER: 0.000000" in result["issues"] assert "metrics.md missing or stale line: - normalization changed GT/prediction/any: 0/0/0 of 30" in result["issues"] def test_audit_eval_run_rejects_stale_page_metrics_report(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) report_path = args.out / "metrics.md" report = report_path.read_text(encoding="utf-8") assert "- scored pages: 30" in report assert "- dictionary missing reference codepoints: []" in report report = report.replace("- scored pages: 30", "- scored pages: 0") report = report.replace("- dictionary missing reference codepoints: []", "- dictionary missing reference codepoints: ['U+1234']") report_path.write_text(report, encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "metrics.md missing or stale line: - scored pages: 30" in result["issues"] assert "metrics.md missing or stale line: - dictionary missing reference codepoints: []" in result["issues"] def test_audit_eval_run_reports_missing_timestamp(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run.pop("created_at") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "eval-run.json missing required field: created_at" in result["issues"] assert "eval run created_at timestamp is missing" in result["issues"] def test_audit_eval_run_reports_command_file_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) (args.out / "command.txt").write_text("different command\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "command.txt does not match eval run command" in result["issues"] def test_audit_eval_run_reports_recognizer_input_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) args.prediction_file.write_text("line-1.png\tchanged\t1.0\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer prediction file sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_reports_recognizer_image_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) image_path = args.data_dir / "images" / "line-1.png" image_path.write_bytes(image_path.read_bytes() + b"changed") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("predictions.tsv eval-1 image_sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_prediction_rows(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) predictions = args.out / "predictions.tsv" lines = predictions.read_text(encoding="utf-8").splitlines() lines[1] = lines[1].replace("\t0\t0\tTrue", "\t1\t1\tFalse") predictions.write_text("\n".join(lines) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("codepoint_edits mismatch" in issue for issue in result["issues"]) assert any("grapheme_edits mismatch" in issue for issue in result["issues"]) assert any("exact mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_recognizer_prediction_source_rows(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) predictions = args.out / "predictions.tsv" with predictions.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle, delimiter="\t")) assert rows fieldnames = list(rows[0]) rows[0]["prediction_score"] = "0.123" rows[0]["claim_eligible"] = "False" rows[0]["source_document"] = "wrong.pdf" with predictions.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t") writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "predictions.tsv eval-1 prediction_score does not match source files" in result["issues"] assert "predictions.tsv eval-1 claim_eligible does not match source files" in result["issues"] assert "predictions.tsv eval-1 source_document does not match source files" in result["issues"] def test_audit_eval_run_rejects_recognizer_manifest_image_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] rows[0]["sha256"] = "0" * 64 args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer manifest eval-1 sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_recognizer_manifest_text_and_sample_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] rows[0]["text_sha256"] = "0" * 64 rows[0]["sample_sha256"] = "1" * 64 args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer manifest eval-1 text_sha256 mismatch" in issue for issue in result["issues"]) assert any("recognizer manifest eval-1 sample_sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_pack_content_digest(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run["pack_content_sha256"] = "0" * 64 metrics["pack_content_sha256"] = "1" * 64 eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "eval-run.json pack_content_sha256 mismatch" in "\n".join(result["issues"]) assert "metrics.json pack_content_sha256 mismatch" in "\n".join(result["issues"]) def test_audit_eval_run_recomputes_page_pack_content_digest(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run["pack_content_sha256"] = "0" * 64 metrics["pack_content_sha256"] = "1" * 64 eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page-eval-run.json pack_content_sha256 mismatch" in "\n".join(result["issues"]) assert "metrics.json pack_content_sha256 mismatch" in "\n".join(result["issues"]) def test_audit_eval_run_recomputes_recognizer_aggregate_metrics(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["codepoint_edits"] = 1 metrics["cer_codepoint_micro"] = 1 / metrics["codepoint_ref_len"] metrics["grapheme_edits"] = 1 metrics["cer_grapheme_micro"] = 1 / metrics["grapheme_ref_len"] metrics["exact_line_accuracy"] = 0.0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("metrics codepoint_edits does not match recomputed predictions.tsv value" in issue for issue in result["issues"]) assert any("metrics exact_line_accuracy does not match recomputed predictions.tsv value" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_uncertainty(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["uncertainty"]["metrics"]["cer_grapheme_micro"]["high"] = 0.5 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer uncertainty cer_grapheme_micro.high mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_script_inventory(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" eval_run_path = args.out / "eval-run.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics["script_inventory"]["observed_codepoint_count"] = 999 eval_run["script_inventory"]["observed_grapheme_count"] = 999 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer metrics script_inventory observed_codepoint_count mismatch" in issue for issue in result["issues"]) assert any("recognizer eval-run script_inventory observed_grapheme_count mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_match_count_metrics(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) (args.data_dir / "images" / "line-31.png").write_bytes(b"eval-image-31") with args.label_file.open("a", encoding="utf-8") as handle: handle.write("images/line-31.png\tdef\n") with args.prediction_file.open("a", encoding="utf-8") as handle: handle.write("extra.png\tzzz\t0.5\n") args.require_full_match = False args.require_claim_safe = False scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["label_lines"] == 31 assert metrics["matched_lines"] == 30 assert metrics["unmatched_labels"] == 1 assert metrics["unmatched_predictions"] == 1 assert metrics["prediction_match_rate"] == 30 / 31 metrics["label_lines"] = 1 metrics["matched_lines"] = 2 metrics["n_lines"] = 2 metrics["unmatched_labels"] = 0 metrics["unmatched_predictions"] = 0 metrics["prediction_match_rate"] = 1.0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("metrics label_lines does not match recomputed recognizer source value" in issue for issue in result["issues"]) assert any("metrics matched_lines does not match recomputed recognizer source value" in issue for issue in result["issues"]) assert any("metrics unmatched_labels does not match recomputed recognizer source value" in issue for issue in result["issues"]) assert any("metrics unmatched_predictions does not match recomputed recognizer source value" in issue for issue in result["issues"]) assert any("metrics prediction_match_rate does not match recomputed recognizer source value" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_claim_safe_with_low_recognizer_match_rate(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) (args.data_dir / "images" / "line-31.png").write_bytes(b"extra-eval-image") with args.label_file.open("a", encoding="utf-8") as handle: handle.write("images/line-31.png\tdef\n") with args.prediction_file.open("a", encoding="utf-8") as handle: handle.write("extra.png\tzzz\t0.5\n") args.require_full_match = False args.require_claim_safe = False scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer claim_safe is true despite unmatched prediction/label rows" in result["issues"] assert "recognizer claim_safe is true despite low prediction match rate" in result["issues"] assert any("missing claim-safety warning for unmatched prediction/label rows" in issue for issue in result["issues"]) assert any("missing claim-safety warning for low prediction match rate" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_confusions(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) (args.out / "confusions.json").write_text('{"stale->artifact": 1}\n', encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("confusions.json does not match recomputed predictions.tsv confusions" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_confusions(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) (args.out / "confusions.json").write_text('{"stale->artifact": 1}\n', encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("confusions.json does not match recomputed text-pairs.tsv confusions" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_slice_metrics(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["slices"][0]["cer_grapheme_micro"] = 0.5 metrics["slices"][0]["exact_line_accuracy"] = 0.0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer slice" in issue and "cer_grapheme_micro mismatch" in issue for issue in result["issues"]) assert any("recognizer slice" in issue and "exact_line_accuracy mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_tampered_recognizer_claim_safe_blockers(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] metrics["metadata_contradictions"] = ["eval-1: synthetic/transliterated provenance is declared as real domain"] metrics["leakage"]["overlaps"]["sample_id"] = ["eval-1"] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer claim_safe is true despite metadata contradictions" in result["issues"] assert "recognizer claim_safe is true despite train/dev/eval leakage" in result["issues"] assert any("missing claim-safety warning for metadata contradictions" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_claim_safe_with_undersized_recognizer_slice(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False manifest_rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] manifest_rows[0]["metadata"]["language"] = "other-lang" args.manifest_file.write_text("\n".join(json.dumps(row) for row in manifest_rows) + "\n", encoding="utf-8") scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer claim_safe is true despite mixed languages" in result["issues"] assert "recognizer claim_safe is true despite slice sample size too small" in result["issues"] assert any("missing claim-safety warning for slice sample size too small" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_tampered_recognizer_unrun_leakage_check(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False args.train_manifest = [] scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer claim_safe is true despite unrun leakage check" in result["issues"] assert any("missing claim-safety warning for unrun leakage check" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_claim_safe_with_missing_recognizer_model_identity(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["model_name"] = "" eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer claim_safe is true despite missing model identity" in result["issues"] assert any("missing claim-safety warning for missing model identity" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_tampered_recognizer_missing_manifest_summary_blockers(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False args.manifest_file = None args.summary_file = None scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer claim_safe is true despite missing eval manifest" in result["issues"] assert "recognizer claim_safe is true despite missing eval pack summary/datasheet" in result["issues"] assert any("missing claim-safety warning for missing eval manifest" in issue for issue in result["issues"]) assert any("missing claim-safety warning for missing eval pack summary/datasheet" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_tampered_recognizer_row_claim_eligibility_blocker(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] rows[0]["metadata"]["claim_evidence_eligible"] = False args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer claim_safe is true despite row-level claim ineligible values" in result["issues"] assert any("missing claim-safety warning for row-level claim ineligible values" in issue for issue in result["issues"]) def test_audit_eval_run_reports_recognizer_artifact_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) Path(args.checkpoint).write_bytes(b"changed") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer artifact_identity.checkpoint sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_recognizer_artifact_identity_field_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["checkpoint"] = str(tmp_path / "other-checkpoint.pdparams") eval_run["dictionary"] = str(tmp_path / "other-dict.txt") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer artifact_identity.checkpoint.value does not match eval-run checkpoint" in issue for issue in result["issues"]) assert any("recognizer artifact_identity.dictionary.value does not match eval-run dictionary" in issue for issue in result["issues"]) def test_audit_eval_run_reports_page_manifest_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) args.eval_manifest.write_text(args.eval_manifest.read_text(encoding="utf-8") + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page eval manifest sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_page_artifact_identity_field_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["config"] = str(tmp_path / "other-config.yml") eval_run["dictionary"] = str(tmp_path / "other-dict.txt") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page artifact_identity.config.value does not match eval-run config" in issue for issue in result["issues"]) assert any("page artifact_identity.dictionary.value does not match eval-run dictionary" in issue for issue in result["issues"]) def test_audit_eval_run_reports_page_train_manifest_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) args.train_manifest[0].write_text(args.train_manifest[0].read_text(encoding="utf-8") + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("train_manifests[0] sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_leakage_report(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False eval_image = tmp_path / "pack" / "images" / "line-1.png" args.train_manifest[0].write_text( json.dumps( { "sample_id": "eval-1", "image_path": "images/line-1.png", "text": "abc", "sha256": sha256_file(eval_image), } ) + "\n", encoding="utf-8", ) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["leakage"]["overlaps"] = {key: [] for key in metrics["leakage"]["overlaps"]} metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer leakage overlaps.sample_id mismatch" in issue for issue in result["issues"]) assert any("recognizer leakage overlaps.image_sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_leakage_report(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False image = tmp_path / "page.png" write_manifest( [ ManifestEntry( sample_id="page-1", dataset="fixture-pages", split="train", image_path=str(image), text="abc", sha256=sha256_file(image), ) ], args.train_manifest[0], ) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["leakage"]["overlaps"] = {key: [] for key in metrics["leakage"]["overlaps"]} metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page leakage overlaps.sample_id mismatch" in issue for issue in result["issues"]) assert any("page leakage overlaps.image_sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_source_sample_id_leakage(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False train_image = tmp_path / "train-crop.png" train_image.write_bytes(b"page-derived-crop") args.train_manifest[0].write_text( json.dumps( { "sample_id": "page-1::line-1", "dataset": "fixture-page-derived-crops", "split": "train", "image_path": str(train_image), "image_sha256": sha256_file(train_image), "text": "शीर्षक", "text_sha256": scorer._text_hash("शीर्षक"), "source_sample_id": "page-1", }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["leakage"]["overlaps"]["source_sample_id"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page leakage overlaps.source_sample_id mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_metadata_contradictions(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] rows[0]["metadata"]["label_provenance"] = "machine_converted_legacy_font" args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["metadata_contradictions"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer metadata_contradictions do not match source metadata" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_metadata_contradictions(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False reference_payload = json.loads((tmp_path / "page.ref.json").read_text(encoding="utf-8")) reference_payload["metadata"]["source_type"] = "synthetic" (tmp_path / "page.ref.json").write_text(json.dumps(reference_payload) + "\n", encoding="utf-8") scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["metadata_contradictions"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page metadata_contradictions do not match source metadata" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_provenance_coverage(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()] rows[0]["metadata"].pop("source_document") rows[0]["metadata"].pop("bbox") rows[0]["metadata"].pop("reviewed_by") args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["provenance_coverage"]["missing_source_document"] = 0 metrics["provenance_coverage"]["missing_bbox"] = 0 metrics["provenance_coverage"]["missing_human_review_evidence"] = 0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("recognizer provenance_coverage does not match source metadata" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_metadata_coverage(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()] rows[0]["metadata"].pop("source_document") rows[0]["metadata"].pop("page") rows[0]["metadata"].pop("bbox") rows[0]["metadata"].pop("claim_evidence_eligible") rows[0]["metadata"].pop("reviewed_by") rows[0]["metadata"].pop("language") args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") _refresh_page_pack_digest(args) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["metadata_coverage"]["missing_source_document"] = 0 metrics["metadata_coverage"]["missing_page"] = 0 metrics["metadata_coverage"]["missing_bbox"] = 0 metrics["metadata_coverage"]["missing_claim_eligibility"] = 0 metrics["metadata_coverage"]["missing_human_review_evidence"] = 0 metrics["metadata_coverage"]["missing_language"] = 0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page metadata_coverage does not match eval manifest" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_reference_coverage(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["reference_coverage"] = {key: 0 for key in metrics["reference_coverage"]} metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page reference_coverage does not match page-metrics.csv" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_manifest_count_metrics(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()] rows[0]["metadata"]["claim_evidence_eligible"] = False args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") _refresh_page_pack_digest(args) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["claim_eligible_rows"] = metrics["manifest_samples"] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page metrics claim_eligible_rows does not match manifest/doc/reference sources" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_document_source_path_mismatches(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False document_path = args.documents_dir / "page-1" / "document.json" payload = json.loads(document_path.read_text(encoding="utf-8")) payload["source_path"] = str(tmp_path / "wrong-page.png") document_path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["document_source_path_mismatch_count"] == 1 metrics["document_source_path_mismatch_count"] = 0 metrics["document_source_path_mismatches"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page document_source_path_mismatches do not match scored documents" in issue for issue in result["issues"]) assert any("page document_source_path_mismatch_count does not match scored documents" in issue for issue in result["issues"]) def test_audit_eval_run_reports_page_document_hash_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) image_path = tmp_path / "page.png" image_path.write_bytes(image_path.read_bytes() + b"changed") document_path = args.documents_dir / "page-1" / "document.json" document_path.write_text(document_path.read_text(encoding="utf-8") + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page-metrics.csv page-1 input_sha256 mismatch" in issue for issue in result["issues"]) assert any("page-metrics.csv page-1 document_sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_line_alignment_rows(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) page_metrics_path = args.out / "page-metrics.csv" with page_metrics_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) assert rows fieldnames = list(rows[0]) rows[0]["line_match_count"] = "0" rows[0]["line_detection_f1"] = "0.0" with page_metrics_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page-metrics.csv page-1 line_match_count mismatch" in issue for issue in result["issues"]) assert any("page-metrics.csv page-1 line_detection_f1 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_line_alignment_row_contents(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) alignment_path = args.out / "line-alignment.csv" with alignment_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) assert rows fieldnames = list(rows[0]) rows[0]["reference_line_id"] = "wrong-ref" rows[0]["predicted_bbox"] = "[9,9,9,9]" rows[0]["predicted_text"] = "wrong text" with alignment_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("line-alignment.csv page-1 row 1 reference_line_id mismatch" in issue for issue in result["issues"]) assert any("line-alignment.csv page-1 row 1 predicted_bbox mismatch" in issue for issue in result["issues"]) assert any("line-alignment.csv page-1 row 1 predicted_text mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_metric_means(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["metric_means"]["line_detection_f1"] = 0.5 metrics["metric_means"]["reading_order_pair_accuracy"] = 0.5 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("metrics metric_means.line_detection_f1 does not match page-metrics.csv mean" in issue for issue in result["issues"]) assert any("metrics metric_means.reading_order_pair_accuracy does not match page-metrics.csv mean" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_text_metric_totals(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["text_codepoint_edits"] = 1 metrics["text_cer_codepoint_micro"] = 1 / metrics["text_codepoint_ref_len"] metrics["text_grapheme_edits"] = 1 metrics["text_cer_grapheme_micro"] = 1 / metrics["text_grapheme_ref_len"] metrics["text_exact_pages"] = 0 metrics["text_exact_accuracy"] = 0.0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("metrics text_codepoint_edits does not match page-metrics.csv sum" in issue for issue in result["issues"]) assert any("metrics text_cer_grapheme_micro does not match page-metrics.csv sum" in issue for issue in result["issues"]) assert any("metrics text_exact_pages does not match page-metrics.csv sum" in issue for issue in result["issues"]) assert any("metrics text_exact_accuracy does not match page-metrics.csv sum" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_uncertainty(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["uncertainty"]["metrics"]["text_cer_grapheme_micro"]["high"] = 0.5 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page uncertainty text_cer_grapheme_micro.high mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_script_inventory(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" eval_run_path = args.out / "page-eval-run.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics["script_inventory"]["observed_codepoint_count"] = 999 eval_run["script_inventory"]["observed_grapheme_count"] = 999 metrics_path.write_text(json.dumps(metrics, ensure_ascii=False) + "\n", encoding="utf-8") eval_run_path.write_text(json.dumps(eval_run, ensure_ascii=False) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page metrics script_inventory observed_codepoint_count mismatch" in issue for issue in result["issues"]) assert any("page eval-run script_inventory observed_grapheme_count mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_text_row_cer(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) page_metrics_path = args.out / "page-metrics.csv" with page_metrics_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) assert rows fieldnames = list(rows[0]) rows[0]["text_codepoint_edits"] = "1" rows[0]["text_cer_codepoint"] = "0.5" rows[0]["text_exact"] = "False" with page_metrics_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page-metrics.csv page-1 text_codepoint_edits mismatch" in issue for issue in result["issues"]) assert any("page-metrics.csv page-1 text_cer_codepoint mismatch" in issue for issue in result["issues"]) assert any("page-metrics.csv page-1 text_exact mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_page_text_pairs(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) text_pairs_path = args.out / "text-pairs.tsv" with text_pairs_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle, delimiter="\t")) assert rows fieldnames = list(rows[0]) rows[0]["reference_text"] = "wrong reference" rows[0]["text_codepoint_edits"] = "0" rows[0]["text_exact"] = "True" with text_pairs_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t") writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("text-pairs.tsv page-1 reference_text_sha256 mismatch" in issue for issue in result["issues"]) assert any("text-pairs.tsv page-1 text_codepoint_edits mismatch" in issue for issue in result["issues"]) assert any("text-pairs.tsv page-1 reference_text does not match reference source text" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_page_reference_artifacts(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) references_path = args.out / "reference-artifacts.tsv" with references_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle, delimiter="\t")) assert rows fieldnames = list(rows[0]) rows[0]["reference_path"] = str(tmp_path / "wrong.ref.json") rows[0]["reference_sha256"] = "0" * 64 rows[0]["reference_status"] = "unverified" with references_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t") writer.writeheader() writer.writerows(rows) result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("reference-artifacts.tsv page-1 reference_path does not match reference source" in issue for issue in result["issues"]) assert any("reference-artifacts.tsv page-1 reference_sha256 does not match reference source" in issue for issue in result["issues"]) assert any("reference-artifacts.tsv page-1 reference_status does not match reference source" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_text_rows_from_document_reference(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) page_metrics_path = args.out / "page-metrics.csv" with page_metrics_path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) assert rows fieldnames = list(rows[0]) rows[0]["text_codepoint_edits"] = "1" rows[0]["text_grapheme_edits"] = "1" rows[0]["text_cer_codepoint"] = "0.3333333333333333" rows[0]["text_cer_grapheme"] = "0.3333333333333333" with page_metrics_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["text_codepoint_edits"] = 1 metrics["text_grapheme_edits"] = 1 metrics["text_cer_codepoint_micro"] = 1 / metrics["text_codepoint_ref_len"] metrics["text_cer_grapheme_micro"] = 1 / metrics["text_grapheme_ref_len"] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page-metrics.csv page-1 text_codepoint_edits mismatch" in issue for issue in result["issues"]) assert any("metrics text_codepoint_edits does not match page-metrics.csv sum" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_slice_metrics(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) metrics["slices"][0]["text_cer_grapheme_micro"] = 0.5 metrics["slices"][0]["text_exact_accuracy"] = 0.5 metrics["slices"][0]["metric_means"]["line_detection_f1"] = 0.5 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any("page slice" in issue and "text_cer_grapheme_micro mismatch" in issue for issue in result["issues"]) assert any("page slice" in issue and "text_exact_accuracy mismatch" in issue for issue in result["issues"]) assert any("page slice" in issue and "metric_means.line_detection_f1 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_tampered_page_claim_safe_blockers(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] metrics["metadata_contradictions"] = ["page-1: machine/synthetic provenance is declared as human_verified GT"] metrics["leakage"]["overlaps"]["sample_id"] = ["page-1"] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page claim_safe is true despite metadata contradictions" in result["issues"] assert "page claim_safe is true despite train/dev/eval leakage" in result["issues"] assert any("missing claim-safety warning for train/dev/eval leakage" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_claim_safe_with_undersized_page_slice(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False manifest_rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()] manifest_rows[0]["metadata"]["language"] = "other-lang" args.eval_manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in manifest_rows) + "\n", encoding="utf-8") _refresh_page_pack_digest(args) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page claim_safe is true despite mixed languages" in result["issues"] assert "page claim_safe is true despite slice sample size too small" in result["issues"] assert any("missing claim-safety warning for slice sample size too small" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_tampered_page_unrun_leakage_check(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False args.train_manifest = [] scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page claim_safe is true despite unrun leakage check" in result["issues"] assert any("missing claim-safety warning for unrun leakage check" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_claim_safe_with_missing_page_model_identity(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["system_name"] = "" eval_run["model_name"] = "" eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page claim_safe is true despite missing system identity" in result["issues"] assert "page claim_safe is true despite missing model identity" in result["issues"] assert any("missing claim-safety warning for missing system identity" in issue for issue in result["issues"]) assert any("missing claim-safety warning for missing model identity" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_tampered_page_missing_summary_blocker(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False args.summary_file = None scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page claim_safe is true despite missing eval pack summary/datasheet" in result["issues"] assert any("missing claim-safety warning for missing eval pack summary/datasheet" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_claim_safe_with_normalizer_defaults(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False summary = json.loads(args.summary_file.read_text(encoding="utf-8")) summary["counts"] = {"defaulted_domain": 1} args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8") scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["claim_safe"] = True eval_run["warnings"] = [] eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer normalizer default count missing warning: defaulted_domain" in result["issues"] assert "recognizer claim_safe is true despite normalizer-defaulted metadata" in result["issues"] def test_audit_eval_run_recomputes_normalizer_counts_from_recognizer_summary(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False summary = json.loads(args.summary_file.read_text(encoding="utf-8")) summary["counts"] = {"labels": 30, "defaulted_domain": 1} args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8") scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) eval_run["normalizer_counts"] = {} metrics["normalizer_counts"] = {} eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "metrics.json normalizer_counts do not match recognizer summary counts" in result["issues"] assert "eval-run.json normalizer_counts do not match recognizer summary counts" in result["issues"] def test_audit_eval_run_rejects_recognizer_summary_metadata_contradictions(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) summary = json.loads(args.summary_file.read_text(encoding="utf-8")) summary.update( { "language": "wrong-lang", "script": "WrongScript", "domain": "synthetic", "gt_status": "machine_converted", "split": "train", "eval_pack_version": "wrong-v1", "frozen_eval_pack": False, "unseen_for_selection": False, "claim_eligible_default": False, "eval_pack": {"task": "page_level_ocr"}, } ) args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8") eval_run_path = args.out / "eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["summary_file_sha256"] = sha256_file(args.summary_file) eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer summary language mismatch: summary='wrong-lang' run=['fixture-lang']" in result["issues"] assert "recognizer summary script mismatch: summary='WrongScript' run=['FixtureScript']" in result["issues"] assert "recognizer summary domain mismatch: summary='synthetic' run=['real_print']" in result["issues"] assert "recognizer summary gt_status mismatch: summary='machine_converted' run=['human_verified']" in result["issues"] assert "recognizer summary split mismatch: summary='train' run=['eval']" in result["issues"] assert "recognizer summary eval_pack_version mismatch: summary='wrong-v1' run='fixture-v1'" in result["issues"] assert "recognizer summary frozen_eval_pack mismatch: summary=False run=True" in result["issues"] assert "recognizer summary unseen_for_selection mismatch: summary=False run=True" in result["issues"] assert "recognizer summary claim_eligible_default mismatch: summary=False run=True" in result["issues"] assert "recognizer summary task mismatch: summary='page_level_ocr' run='recognizer_oracle_crop'" in result["issues"] def test_audit_eval_run_rejects_page_summary_metadata_contradictions(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) scorer.run_eval(args) summary = json.loads(args.summary_file.read_text(encoding="utf-8")) summary.update( { "language": "wrong-lang", "script": "WrongScript", "domain": "synthetic", "gt_status": "machine_converted", "split": "train", "eval_pack_version": "wrong-v1", "frozen_eval_pack": False, "unseen_for_selection": False, "claim_eligible_default": False, "eval_pack": {"task": "recognizer_oracle_crop"}, } ) args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8") eval_run_path = args.out / "page-eval-run.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) eval_run["summary_file_sha256"] = sha256_file(args.summary_file) eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page summary language mismatch: summary='wrong-lang' run=['fixture-lang']" in result["issues"] assert "page summary script mismatch: summary='WrongScript' run=['FixtureScript']" in result["issues"] assert "page summary domain mismatch: summary='synthetic' run=['real_print']" in result["issues"] assert "page summary gt_status mismatch: summary='machine_converted' run=['human_verified']" in result["issues"] assert "page summary split mismatch: summary='train' run=['eval']" in result["issues"] assert "page summary eval_pack_version mismatch: summary='wrong-v1' run='page-v1'" in result["issues"] assert "page summary frozen_eval_pack mismatch: summary=False run=True" in result["issues"] assert "page summary unseen_for_selection mismatch: summary=False run=True" in result["issues"] assert "page summary claim_eligible_default mismatch: summary=False run=True" in result["issues"] assert "page summary task mismatch: summary='recognizer_oracle_crop' run='page_level_ocr'" in result["issues"] def test_audit_eval_run_recomputes_recognizer_dictionary_coverage(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False Path(args.dictionary).write_text("a\nb\n", encoding="utf-8") scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["dictionary_coverage"]["missing_gt_codepoints"] == ["U+0063"] metrics["dictionary_coverage"]["missing_gt_codepoints"] = [] metrics["dictionary_coverage"]["missing_gt_codepoint_count"] = 0 eval_run["claim_safe"] = True eval_run["warnings"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any( "recognizer metrics dictionary_coverage missing_gt_codepoints mismatch" in issue and "U+0063" in issue for issue in result["issues"] ) assert "recognizer claim_safe is true despite GT codepoints missing from dictionary" in result["issues"] assert any("recognizer missing claim-safety warning for GT codepoints missing from dictionary" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_page_dictionary_coverage(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False Path(args.dictionary).write_text("a\nb\n", encoding="utf-8") scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["dictionary_coverage"]["missing_gt_codepoints"] == ["U+0063"] metrics["dictionary_coverage"]["missing_gt_codepoints"] = [] metrics["dictionary_coverage"]["missing_gt_codepoint_count"] = 0 eval_run["claim_safe"] = True eval_run["warnings"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert any( "page metrics dictionary_coverage missing_gt_codepoints mismatch" in issue and "U+0063" in issue for issue in result["issues"] ) assert "page claim_safe is true despite GT codepoints missing from dictionary" in result["issues"] assert any("page missing claim-safety warning for GT codepoints missing from dictionary" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_zero_length_recognizer_gt_rows(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False label_lines = args.label_file.read_text(encoding="utf-8").splitlines() label_lines[0] = "images/line-1.png\t" args.label_file.write_text("\n".join(label_lines) + "\n", encoding="utf-8") prediction_lines = args.prediction_file.read_text(encoding="utf-8").splitlines() prediction_lines[0] = "line-1.png\t\t1.0" args.prediction_file.write_text("\n".join(prediction_lines) + "\n", encoding="utf-8") _refresh_recognizer_pack_digest(args) scorer.run_eval(args) eval_run_path = args.out / "eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["zero_length_gt_lines"] == 1 metrics["zero_length_gt_lines"] = 0 eval_run["claim_safe"] = True eval_run["warnings"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "metrics zero_length_gt_lines does not match recomputed predictions.tsv value: recorded=0 recomputed=1" in result["issues"] assert "recognizer claim_safe is true despite zero-length GT rows" in result["issues"] assert any("recognizer missing claim-safety warning for zero-length GT rows" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_zero_length_page_text_references(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False reference = tmp_path / "page.ref.json" payload = json.loads(reference.read_text(encoding="utf-8")) payload["text"] = "" reference.write_text(json.dumps(payload) + "\n", encoding="utf-8") scorer.run_eval(args) eval_run_path = args.out / "page-eval-run.json" metrics_path = args.out / "metrics.json" eval_run = json.loads(eval_run_path.read_text(encoding="utf-8")) metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["zero_length_text_references"] == 1 metrics["zero_length_text_references"] = 0 eval_run["claim_safe"] = True eval_run["warnings"] = [] metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "metrics zero_length_text_references does not match page-metrics.csv sum: recorded=0 recomputed=1" in result["issues"] assert "page claim_safe is true despite zero-length text references" in result["issues"] assert any("page missing claim-safety warning for zero-length text references" in issue for issue in result["issues"]) def test_audit_eval_run_recomputes_recognizer_normalization_summary(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.require_claim_safe = False Path(args.dictionary).write_text("a\nb\nc\ná\n", encoding="utf-8") label_lines = args.label_file.read_text(encoding="utf-8").splitlines() label_lines[0] = "images/line-1.png\ta\u0301bc" args.label_file.write_text("\n".join(label_lines) + "\n", encoding="utf-8") prediction_lines = args.prediction_file.read_text(encoding="utf-8").splitlines() prediction_lines[0] = "line-1.png\ta\u0301bc\t1.0" args.prediction_file.write_text("\n".join(prediction_lines) + "\n", encoding="utf-8") _refresh_recognizer_pack_digest(args) scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["normalization_summary"]["gt_changed"] == 1 assert metrics["normalization_summary"]["prediction_changed"] == 1 metrics["normalization_summary"]["gt_changed"] = 0 metrics["normalization_summary"]["prediction_changed"] = 0 metrics["normalization_summary"]["any_changed"] = 0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "recognizer metrics normalization_summary gt_changed mismatch: recorded=0 recomputed=1" in result["issues"] assert "recognizer metrics normalization_summary prediction_changed mismatch: recorded=0 recomputed=1" in result["issues"] assert "recognizer metrics normalization_summary any_changed mismatch: recorded=0 recomputed=1" in result["issues"] def test_audit_eval_run_recomputes_page_normalization_summary(tmp_path: Path) -> None: scorer = _load_tool("run_page_ocr_eval") audit_tool = _load_tool("audit_eval_run") args = _write_page_fixture(tmp_path) args.require_claim_safe = False Path(args.dictionary).write_text("a\nb\nc\ná\n", encoding="utf-8") document_path = tmp_path / "documents" / "page-1" / "document.json" payload = json.loads(document_path.read_text(encoding="utf-8")) payload["pages"][0]["text_lines"][0]["text"] = "a\u0301bc" payload["pages"][0]["blocks"][0]["text"] = "a\u0301bc" document_path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") scorer.run_eval(args) metrics_path = args.out / "metrics.json" metrics = json.loads(metrics_path.read_text(encoding="utf-8")) assert metrics["normalization_summary"]["prediction_changed"] == 1 metrics["normalization_summary"]["prediction_changed"] = 0 metrics["normalization_summary"]["any_changed"] = 0 metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8") result = audit_tool.audit_run(args.out) assert result["audit_passed"] is False assert "page metrics normalization_summary prediction_changed mismatch: recorded=0 recomputed=1" in result["issues"] assert "page metrics normalization_summary any_changed mismatch: recorded=0 recomputed=1" in result["issues"] def test_audit_eval_run_accepts_complete_recipe_run_root(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["score_dir"].endswith("recipe-run/score") assert result["audit_passed"] is True def test_audit_eval_run_rejects_stale_recipe_materialization_file(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) materialization_path = tmp_path / "recipe-run" / "materialize" / "output-command-run.json" materialization = json.loads(materialization_path.read_text(encoding="utf-8")) materialization["records"][0]["output_sha256"] = "0" * 64 materialization_path.write_text(json.dumps(materialization) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "materialize/output-command-run.json does not match recipe-run.json materialization" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_validation_artifact(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) validation_path = tmp_path / "recipe-run" / "recipe-validation.json" validation = json.loads(validation_path.read_text(encoding="utf-8")) validation["claim_ready"] = False validation_path.write_text(json.dumps(validation) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-validation.json does not match recipe-run.json recipe_validation" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_validation_against_validator(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) validation_path = tmp_path / "recipe-run" / "recipe-validation.json" recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" validation = json.loads(validation_path.read_text(encoding="utf-8")) validation["claim_ready"] = False validation["claim_blockers"] = ["stale blocker"] validation_path.write_text(json.dumps(validation) + "\n", encoding="utf-8") recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["recipe_validation"] = validation recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json recipe_validation does not match current recipe validator result" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_artifact_against_registry(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_path = tmp_path / "recipe-run" / "recipe.json" recipe = json.loads(recipe_path.read_text(encoding="utf-8")) recipe["task"] = "stale-task" recipe_path.write_text(json.dumps(recipe) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe.json does not match selected recipe in recipe registry file" in result["issues"] def test_audit_eval_run_rejects_missing_recipe_run_provenance(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["machine"] = {"platform": ""} recipe_run["command"] = "" recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe run machine.platform is missing" in result["issues"] assert "recipe run machine.python is missing" in result["issues"] assert "recipe run command is missing" in result["issues"] def test_audit_eval_run_rejects_missing_recipe_run_root(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["root"] = str(tmp_path / "missing-root") recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert any("recipe-run.json root does not exist or is not a directory" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_recipe_registry_outside_declared_root(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) declared_root = tmp_path / "declared-root" declared_root.mkdir() recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["root"] = str(declared_root) recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json recipe_file is not under declared root" in result["issues"] def test_audit_eval_run_rejects_recipe_outputs_pointing_outside_root_artifacts(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) stale_dir = tmp_path / "stale" stale_dir.mkdir() stale_audit = stale_dir / "audit.json" stale_claim = stale_dir / "claim-safety.md" stale_audit.write_text((tmp_path / "recipe-run" / "audit.json").read_text(encoding="utf-8"), encoding="utf-8") stale_claim.write_text((tmp_path / "recipe-run" / "claim-safety.md").read_text(encoding="utf-8"), encoding="utf-8") recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["outputs"]["materialization_dir"] = str(stale_dir) recipe_run["outputs"]["audit_file"] = str(stale_audit) recipe_run["outputs"]["claim_safety_report"] = str(stale_claim) recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json outputs.materialization_dir does not point at root materialize" in result["issues"] assert "recipe-run.json outputs.audit_file does not point at root audit.json" in result["issues"] assert "recipe-run.json outputs.claim_safety_report does not point at root claim-safety.md" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_run_audit_issues(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["audit"]["issues"] = ["stale issue"] recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json audit issues do not match current score audit" in result["issues"] def test_audit_eval_run_rejects_stale_root_audit_summary(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) audit_path = tmp_path / "recipe-run" / "audit.json" audit = json.loads(audit_path.read_text(encoding="utf-8")) audit["issues"] = ["stale issue"] audit["warnings"] = ["stale warning"] audit["score_dir"] = str(tmp_path / "stale-score") audit_path.write_text(json.dumps(audit) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "audit.json warnings do not match eval run" in result["issues"] assert "audit.json issues do not match current score audit" in result["issues"] assert "audit.json score_dir does not match audited score directory" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_score_artifact_hashes(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["score"]["metrics_file_sha256"] = "0" * 64 recipe_run["score"]["eval_run_file_sha256"] = "1" * 64 recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert any("recipe-run.json score metrics_file_sha256 does not match score/metrics.json" in issue for issue in result["issues"]) assert any("recipe-run.json score eval_run_file_sha256 does not match score/eval-run.json" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_recipe_score_artifacts_outside_score_dir(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) stale_dir = tmp_path / "stale-score" stale_dir.mkdir() stale_metrics = stale_dir / "metrics.json" stale_eval_run = stale_dir / "eval-run.json" stale_metrics.write_text((args.out / "metrics.json").read_text(encoding="utf-8"), encoding="utf-8") stale_eval_run.write_text((args.out / "eval-run.json").read_text(encoding="utf-8"), encoding="utf-8") recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["score"]["metrics_file"] = str(stale_metrics) recipe_run["score"]["eval_run_file"] = str(stale_eval_run) recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json score metrics_file does not point at score/metrics.json" in result["issues"] assert "recipe-run.json score eval_run_file does not point at score/eval-run.json" in result["issues"] def test_audit_eval_run_rejects_recipe_train_manifest_identity_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) other_manifest = tmp_path / "other-train.jsonl" other_manifest.write_text("", encoding="utf-8") recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["train_manifests"][0]["path"] = str(other_manifest) recipe_run["train_manifests"][0]["sha256"] = "0" * 64 recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json train_manifests[0] path does not match eval-run.json" in result["issues"] assert "recipe-run.json train_manifests[0] sha256 does not match eval-run.json" in result["issues"] def test_audit_eval_run_rejects_recipe_train_manifest_count_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) scorer.run_eval(args) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True, exist_ok=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["train_manifests"] = [] recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json train_manifests count does not match eval-run.json: recorded=0 score=1" in result["issues"] def test_audit_eval_run_rejects_missing_recipe_materialization_provenance(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["materialization"].pop("created_at") recipe_run["materialization"]["machine"] = {"platform": ""} recipe_run["materialization"].pop("invocation") recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "materialization run created_at timestamp is missing" in result["issues"] assert "materialization machine.platform is missing" in result["issues"] assert "materialization machine.python is missing" in result["issues"] assert "materialization missing required field: invocation" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_materialization_record_count(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["materialization"]["record_count"] = 2 recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "materialization record_count does not match records length: recorded=2 actual=1" in result["issues"] def test_audit_eval_run_rejects_recipe_materialization_score_identity_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["materialization"]["model_name"] = "stale-model" recipe_run["materialization"]["checkpoint"] = "stale-checkpoint" recipe_run["materialization"]["dictionary"] = "stale-dictionary" recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert any("recipe-run.json materialization model_name does not match score eval-run model_name" in issue for issue in result["issues"]) assert any("recipe-run.json materialization checkpoint does not match score eval-run checkpoint" in issue for issue in result["issues"]) assert any("recipe-run.json materialization dictionary does not match score eval-run dictionary" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_recipe_command_template_file(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) (tmp_path / "recipe-run" / "materialize" / "command-template.txt").write_text("stale command\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "materialize/command-template.txt does not match recorded materialization command_template" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_materialization_record_command(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) materialization_path = tmp_path / "recipe-run" / "materialize" / "output-command-run.json" recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" materialization = json.loads(materialization_path.read_text(encoding="utf-8")) materialization["records"][0]["command"] = ["stale-command"] recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["materialization"] = materialization materialization_path.write_text(json.dumps(materialization) + "\n", encoding="utf-8") recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "materialization records[0] command does not match materialization command_template" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_materialization_log_hash(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) (tmp_path / "recipe-run" / "materialize" / "stdout.txt").write_text("stale stdout\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert any("materialization records[0] stdout_sha256 mismatch" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_stale_recipe_materialization_report(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) (tmp_path / "recipe-run" / "materialize" / "output-command-run.md").write_text("# stale\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "materialize/output-command-run.md missing or stale line: - records: 1" in result["issues"] assert any("materialize/output-command-run.md missing or stale output line" in issue for issue in result["issues"]) def test_audit_eval_run_rejects_recipe_prediction_path_mismatch(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) other_prediction = tmp_path / "recipe-run" / "other-predictions.txt" other_prediction.write_text(root_prediction.read_text(encoding="utf-8"), encoding="utf-8") recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json" recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8")) recipe_run["outputs"]["prediction_file"] = str(other_prediction) recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "recipe-run.json outputs.prediction_file does not match score eval-run prediction_file" in result["issues"] assert "recipe-run.json materialization records do not include outputs.prediction_file" in result["issues"] def test_audit_eval_run_rejects_stale_recipe_root_claim_report(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) root_prediction = tmp_path / "recipe-run" / "predictions.txt" root_prediction.parent.mkdir(parents=True) root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8") args.prediction_file = root_prediction args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) _write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file) report_path = tmp_path / "recipe-run" / "claim-safety.md" report = report_path.read_text(encoding="utf-8") assert "- claim safe: True" in report report_path.write_text(report.replace("- claim safe: True", "- claim safe: False"), encoding="utf-8") result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "root claim-safety.md missing or stale line: - claim safe: True" in result["issues"] def test_audit_eval_run_rejects_incomplete_recipe_run_root(tmp_path: Path) -> None: scorer = _load_tool("run_recognizer_eval") audit_tool = _load_tool("audit_eval_run") args = _write_recognizer_fixture(tmp_path) args.out = tmp_path / "recipe-run" / "score" scorer.run_eval(args) result = audit_tool.audit_run(tmp_path / "recipe-run") assert result["audit_passed"] is False assert "missing required artifact: recipe-run.json" in result["issues"] assert "missing required artifact: claim-safety.md" in result["issues"]