from __future__ import annotations
import csv
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest
from ocrtech.schemas import BBox, Block, Document, Page, Table, TableCell, TextLine
def _load_tool(name: str):
path = Path(__file__).resolve().parents[1] / "tools" / f"{name}.py"
spec = importlib.util.spec_from_file_location(name, path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _write_recognizer_fixture(tmp_path: Path) -> Namespace:
scorer = _load_tool("run_recognizer_eval")
pack = tmp_path / "pack"
images = pack / "images"
images.mkdir(parents=True)
labels: list[str] = []
predictions: list[str] = []
manifest_rows: list[str] = []
digest_lines: list[object] = []
for index in range(1, 31):
image_name = f"line-{index}.png"
image_path = images / image_name
image_path.write_bytes(f"eval-image-{index}".encode("utf-8"))
image_sha256 = scorer._hash_file(image_path)
text_sha256 = scorer._hash_text("abc")
metadata = {
"language": "fixture-lang",
"script": "FixtureScript",
"domain": "real_print",
"gt_status": "human_verified",
"split": "eval",
"source_document": "fixture.pdf",
"page": 1,
"bbox": [1, index, 30, index + 20],
"claim_evidence_eligible": True,
"reviewed_by": "fixture-reviewer",
}
labels.append(f"images/{image_name}\tabc")
predictions.append(f"{image_name}\tabc\t1.0")
manifest_rows.append(
json.dumps(
{
"sample_id": f"eval-{index}",
"image_path": f"images/{image_name}",
"sha256": image_sha256,
"text_sha256": text_sha256,
"sample_sha256": scorer._sample_hash(image_sha256, text_sha256),
"metadata": metadata,
}
)
)
digest_lines.append(
scorer.EvalLine(
sample_id=f"eval-{index}",
image_path=f"images/{image_name}",
image_sha256=image_sha256,
gt_text="abc",
prediction="abc",
prediction_score="1.0",
metadata=metadata,
)
)
(pack / "eval.txt").write_text("\n".join(labels) + "\n", encoding="utf-8")
(pack / "manifest.jsonl").write_text("\n".join(manifest_rows) + "\n", encoding="utf-8")
(pack / "summary.json").write_text(
json.dumps(
{
"dataset": "fixture",
"eval_pack_version": "fixture-v1",
"pack_content_sha256": scorer._pack_content_digest(
digest_lines,
Namespace(
language="fixture-lang",
script="FixtureScript",
domain="real_print",
gt_status="human_verified",
split="eval",
claim_eligible=True,
),
),
"eval_pack": {"lines": 30},
}
)
+ "\n",
encoding="utf-8",
)
(pack / "script-inventory.json").write_text(
json.dumps({"required_codepoints": ["U+0061", "U+0062", "U+0063"], "required_graphemes": ["a", "b", "c"]}) + "\n",
encoding="utf-8",
)
(tmp_path / "preds.txt").write_text("\n".join(predictions) + "\n", encoding="utf-8")
(tmp_path / "train.jsonl").write_text(
json.dumps({"sample_id": "train-1", "image_path": "train.png", "text": "zzz", "sha256": "not-eval"}) + "\n",
encoding="utf-8",
)
(tmp_path / "checkpoint.pdparams").write_bytes(b"checkpoint")
(tmp_path / "config.yml").write_text("model: fixture\n", encoding="utf-8")
(tmp_path / "dict.txt").write_text("a\nb\nc\n", encoding="utf-8")
return Namespace(
label_file=pack / "eval.txt",
data_dir=pack,
prediction_file=tmp_path / "preds.txt",
out=tmp_path / "recognizer-score",
manifest_file=pack / "manifest.jsonl",
summary_file=pack / "summary.json",
script_inventory_file=pack / "script-inventory.json",
spotcheck_calibration_file=None,
train_manifest=[tmp_path / "train.jsonl"],
model_name="fixture-model",
checkpoint=str(tmp_path / "checkpoint.pdparams"),
config=str(tmp_path / "config.yml"),
dictionary=str(tmp_path / "dict.txt"),
language="fixture-lang",
script="FixtureScript",
domain="real_print",
gt_status="human_verified",
split="eval",
eval_pack_version="fixture-v1",
frozen_eval_pack=True,
unseen_for_selection=True,
selected_checkpoint_on_test=False,
claim_eligible=True,
known_leakage="",
min_match_rate=1.0,
review_sample_size=5,
require_full_match=True,
require_claim_safe=True,
force=False,
)
def _refresh_recognizer_pack_digest(args: Namespace) -> None:
scorer = _load_tool("run_recognizer_eval")
label_by_path = {}
for line in args.label_file.read_text(encoding="utf-8").splitlines():
rel_path, text = line.split("\t", 1)
label_by_path[rel_path] = text
rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
digest_lines: list[object] = []
for row in rows:
rel_path = str(row.get("image_path") or row.get("image") or "")
text = label_by_path[rel_path]
image_sha256 = scorer._hash_file(args.data_dir / rel_path)
text_sha256 = scorer._hash_text(text)
row["sha256"] = image_sha256
row["text_sha256"] = text_sha256
row["sample_sha256"] = scorer._sample_hash(image_sha256, text_sha256)
digest_lines.append(
scorer.EvalLine(
sample_id=str(row.get("sample_id") or row.get("id") or ""),
image_path=rel_path,
image_sha256=image_sha256,
gt_text=text,
prediction="",
prediction_score="",
metadata=dict(row.get("metadata") or {}),
)
)
args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
summary = json.loads(args.summary_file.read_text(encoding="utf-8"))
summary["pack_content_sha256"] = scorer._pack_content_digest(digest_lines, args)
args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8")
def _refresh_page_pack_digest(args: Namespace) -> None:
scorer = _load_tool("run_page_ocr_eval")
manifest = scorer.load_manifest(args.eval_manifest)
manifest_dir = args.eval_manifest.parent
digest_rows = []
for entry in manifest:
input_path = scorer._resolve_manifest_path(entry.image_path, manifest_dir)
reference_path = scorer._reference_path_for(entry, input_path, manifest_dir)
reference = scorer._reference_for(entry, input_path, manifest_dir)
reference_payload = scorer._reference_payload_for(entry, input_path, manifest_dir)
reference_fields = scorer._reference_fields(reference)
if scorer._reference_lines(reference_payload):
reference_fields.append("line_bboxes")
digest_rows.append(
scorer.PageEvalRow(
sample_id=entry.sample_id,
input_path=str(input_path),
input_sha256=sha256_file(input_path),
document_path="",
document_sha256="",
document_source_path="",
language=scorer._entry_language(entry, args.language),
script=scorer._entry_script(entry, args.script),
domain=scorer._entry_domain(entry, args.domain),
gt_status=scorer._entry_gt_status(entry, args.gt_status),
split=scorer._entry_split(entry),
claim_eligible=scorer._entry_claim_eligible(entry, args.claim_eligible),
source_document=scorer._metadata_value_for_keys(entry.metadata or {}, scorer.SOURCE_DOCUMENT_KEYS),
source_page=scorer._metadata_value_for_keys(entry.metadata or {}, scorer.PAGE_KEYS),
source_bbox=scorer._metadata_value_for_keys(entry.metadata or {}, scorer.BBOX_KEYS),
reference_path=str(reference_path) if reference_path else "",
reference_sha256=sha256_file(reference_path) if reference_path else "",
reference_status=str((reference.metadata or {}).get("reference_status") or "") if reference else "",
reference_claim_eligible=str((reference.metadata or {}).get("claim_evidence_eligible"))
if reference and (reference.metadata or {}).get("claim_evidence_eligible") is not None
else "",
reference_text=reference.text if reference and reference.text is not None else "",
reference_fields=reference_fields,
)
)
summary = json.loads(args.summary_file.read_text(encoding="utf-8"))
summary["pack_content_sha256"] = scorer._page_pack_content_digest(digest_rows)
args.summary_file.write_text(json.dumps(summary, ensure_ascii=False) + "\n", encoding="utf-8")
def _write_page_fixture(tmp_path: Path) -> Namespace:
manifest_entries: list[ManifestEntry] = []
for index in range(1, 31):
image = tmp_path / ("page.png" if index == 1 else f"page-{index}.png")
image.write_bytes(f"page-image-{index}".encode("utf-8"))
reference = tmp_path / ("page.ref.json" if index == 1 else f"page-{index}.ref.json")
reference.write_text(
json.dumps(
{
"text": "abc",
"reading_order": [{"id": "l1", "text": "abc", "bbox": [0, 0, 50, 20], "page_index": 0}],
"tables": [[["abc"]]],
"metadata": {"claim_evidence_eligible": True},
}
)
+ "\n",
encoding="utf-8",
)
manifest_entries.append(
ManifestEntry(
sample_id=f"page-{index}",
dataset="fixture-pages",
split="eval",
image_path=str(image),
text="abc",
sha256=sha256_file(image),
metadata={
"reference_path": str(reference),
"language": "fixture-lang",
"script": "FixtureScript",
"domain": "real_print",
"gt_status": "human_verified",
"source_document": "fixture-pages.pdf",
"page": index,
"bbox": [0, 0, 50, 50],
"claim_evidence_eligible": True,
"reviewed_by": "fixture-reviewer",
},
)
)
write_manifest(manifest_entries, tmp_path / "manifest.jsonl")
write_manifest(
[
ManifestEntry(
sample_id="train-1",
dataset="fixture-pages",
split="train",
image_path="train.png",
text="zzz",
sha256="not-eval",
)
],
tmp_path / "train.jsonl",
)
table = Table(
table_id="table-1",
page_index=0,
bbox=BBox(0, 0, 50, 20),
cells=[TableCell(row=0, col=0, text="abc")],
)
document = Document(
source_path=str(image),
pages=[
Page(
page_index=0,
text_lines=[TextLine(text="abc", bbox=BBox(0, 0, 50, 20), page_index=0, line_id="l1")],
blocks=[
Block(
block_id="b1",
block_type="table",
page_index=0,
bbox=BBox(0, 0, 50, 20),
order=0,
text="abc",
line_ids=["l1"],
table_id=table.table_id,
)
],
)
],
tables=[table],
)
for index in range(1, 31):
doc_dir = tmp_path / "documents" / f"page-{index}"
doc_dir.mkdir(parents=True)
page_document = Document(
source_path=str(tmp_path / ("page.png" if index == 1 else f"page-{index}.png")),
pages=document.pages,
tables=document.tables,
)
(doc_dir / "document.json").write_text(json.dumps(page_document.to_dict(), ensure_ascii=False) + "\n", encoding="utf-8")
(tmp_path / "summary.json").write_text(
json.dumps({"dataset": "fixture-pages", "eval_pack_version": "page-v1", "eval_pack": {"pages": 30}}) + "\n",
encoding="utf-8",
)
(tmp_path / "script-inventory.json").write_text(
json.dumps({"required_codepoints": ["U+0061", "U+0062", "U+0063"], "required_graphemes": ["a", "b", "c"]}) + "\n",
encoding="utf-8",
)
(tmp_path / "checkpoint.pdparams").write_bytes(b"checkpoint")
(tmp_path / "config.yml").write_text("model: fixture\n", encoding="utf-8")
(tmp_path / "dict.txt").write_text("abc\n", encoding="utf-8")
args = Namespace(
eval_manifest=tmp_path / "manifest.jsonl",
documents_dir=tmp_path / "documents",
ocr_output_kind="sidecar_fixture",
out=tmp_path / "page-score",
summary_file=tmp_path / "summary.json",
script_inventory_file=tmp_path / "script-inventory.json",
spotcheck_calibration_file=None,
train_manifest=[tmp_path / "train.jsonl"],
system_name="fixture-page",
model_name="fixture-model",
checkpoint=str(tmp_path / "checkpoint.pdparams"),
config=str(tmp_path / "config.yml"),
dictionary=str(tmp_path / "dict.txt"),
language="fixture-lang",
script="FixtureScript",
domain="real_print",
gt_status="human_verified",
eval_pack_version="page-v1",
frozen_eval_pack=True,
unseen_for_selection=True,
selected_checkpoint_on_test=False,
claim_eligible=True,
line_iou_threshold=0.5,
review_sample_size=5,
require_claim_safe=True,
force=False,
)
_refresh_page_pack_digest(args)
return args
def _write_page_output_audits(documents_dir: Path, *, rows: int = 30, passed: bool = True) -> None:
for index in range(1, rows + 1):
output_dir = documents_dir / f"page-{index}"
payload = {
"passed": passed,
"issues": [] if passed else ["fixture failure"],
"output_dir": str(output_dir),
}
(output_dir / "limbu-output-audit.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
def _write_recipe_root_artifacts(tmp_path: Path, audit_tool, score_dir: Path, prediction_file: Path) -> None:
root = tmp_path / "recipe-run"
materialize = root / "materialize"
materialize.mkdir(parents=True, exist_ok=True)
recipes = root / "recipes.json"
eval_run = json.loads((score_dir / "eval-run.json").read_text(encoding="utf-8"))
recipe = {
"id": "fixture-recipe",
"task": "recognizer_oracle_crop",
"language": eval_run["languages"][0],
"script": eval_run["scripts"][0],
"model_family": eval_run["model_name"],
"runner": {
"type": "external_command",
"command_template": "python predict.py --labels {label_file} --data {data_dir} --out {prediction_file}",
"requires_operator_resolution": False,
"checkpoint": eval_run["checkpoint"],
"config": eval_run["config"],
"dictionary": eval_run["dictionary"],
},
"eval_pack": {
"label_file": eval_run["label_file"],
"data_dir": eval_run["data_dir"],
"normalization_profile": eval_run["normalization_profile"],
"manifest_file": eval_run["manifest_file"],
"summary_file": eval_run["summary_file"],
"script_inventory_file": eval_run["script_inventory_file"],
"dataset": "fixture",
"domain": eval_run["domains"][0],
"gt_status": eval_run["gt_statuses"][0],
"split": eval_run["splits"][0],
"eval_pack_version": eval_run["eval_pack_version"],
"frozen_eval_pack": eval_run["frozen_eval_pack"],
"unseen_for_selection": eval_run["unseen_for_selection"],
"claim_eligible": eval_run["claim_eligible_default"],
},
"leakage_sources": [{"manifest_file": item["path"], "split": "train"} for item in eval_run["train_manifests"]],
"claim_blockers": [],
}
recipes.write_text(json.dumps({"recipes": [recipe]}) + "\n", encoding="utf-8")
validator = _load_tool("validate_eval_harness_recipes")
recipe_validation = validator.validate_recipes(recipes, tmp_path)["recipes"][0]
(root / "recipe.json").write_text(json.dumps(recipe) + "\n", encoding="utf-8")
(root / "recipe-validation.json").write_text(json.dumps(recipe_validation) + "\n", encoding="utf-8")
root_prediction = root / "predictions.txt"
root_prediction.write_text(prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
stdout_path = materialize / "stdout.txt"
stderr_path = materialize / "stderr.txt"
stdout_path.write_text("", encoding="utf-8")
stderr_path.write_text("", encoding="utf-8")
materialization = {
"created_at": "2026-06-20T00:00:00+00:00",
"mode": "recognizer",
"system_name": "fixture-recipe",
"model_name": eval_run["model_name"],
"model_config": None,
"checkpoint": eval_run["checkpoint"],
"config": eval_run["config"],
"dictionary": eval_run["dictionary"],
"command_template": "fixture",
"timeout_seconds": None,
"label_file": eval_run["label_file"],
"data_dir": eval_run["data_dir"],
"prediction_file": str(root_prediction),
"record_count": 1,
"records": [
{
"sample_id": "__recognizer_eval__",
"command": ["fixture"],
"stdout_path": str(stdout_path),
"stdout_sha256": sha256_file(stdout_path),
"stderr_path": str(stderr_path),
"stderr_sha256": sha256_file(stderr_path),
"returncode": 0,
"output_path": str(root_prediction),
"output_sha256": sha256_file(root_prediction),
"output_size_bytes": root_prediction.stat().st_size,
}
],
"machine": {"platform": "fixture", "python": "fixture"},
"invocation": "fixture",
}
(materialize / "output-command-run.json").write_text(json.dumps(materialization) + "\n", encoding="utf-8")
(materialize / "output-command-run.md").write_text(
"\n".join(
[
"# Eval Output Command Run",
"",
"- records: 1",
"",
"## Outputs",
"",
(
f"- __recognizer_eval__: `{root_prediction}` sha256={sha256_file(root_prediction)}; "
f"stdout `{stdout_path}` sha256={sha256_file(stdout_path)}; "
f"stderr `{stderr_path}` sha256={sha256_file(stderr_path)}"
),
]
)
+ "\n",
encoding="utf-8",
)
(materialize / "command-template.txt").write_text("fixture\n", encoding="utf-8")
score_audit = audit_tool.audit_run(score_dir)
(root / "audit.json").write_text(json.dumps(score_audit) + "\n", encoding="utf-8")
recipe_run = {
"created_at": "2026-06-20T00:00:00+00:00",
"recipe_id": "fixture-recipe",
"recipe_file": str(recipes),
"recipe_file_sha256": sha256_file(recipes),
"root": str(tmp_path),
"outputs": {
"prediction_file": str(root_prediction),
"materialization_dir": str(materialize),
"score_dir": str(score_dir),
"recipe_file": str(root / "recipe.json"),
"recipe_validation_file": str(root / "recipe-validation.json"),
"audit_file": str(root / "audit.json"),
"claim_safety_report": str(root / "claim-safety.md"),
},
"train_manifests": eval_run["train_manifests"],
"recipe_validation": recipe_validation,
"materialization": materialization,
"audit": {
"audit_passed": score_audit["audit_passed"],
"claim_safe": score_audit["claim_safe"],
"issues": score_audit["issues"],
},
"score": {
"claim_safe": eval_run["claim_safe"],
"warnings": eval_run["warnings"],
"metrics_file": str(score_dir / "metrics.json"),
"metrics_file_sha256": sha256_file(score_dir / "metrics.json"),
"eval_run_file": str(score_dir / "eval-run.json"),
"eval_run_file_sha256": sha256_file(score_dir / "eval-run.json"),
},
"machine": {"platform": "fixture", "python": "fixture"},
"command": "fixture",
}
(root / "recipe-run.json").write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
metrics = json.loads((score_dir / "metrics.json").read_text(encoding="utf-8"))
(root / "claim-safety.md").write_text(
"\n".join(
[
"# Eval Claim Safety Report",
"",
"- recipe: fixture-recipe",
f"- task: {recipe['task']}",
f"- audit passed: {score_audit['audit_passed']}",
f"- claim safe: {eval_run['claim_safe']}",
f"- eval mode: {eval_run['eval_mode']}",
f"- score created at: {eval_run['created_at']}",
f"- score machine platform: {eval_run['machine']['platform']}",
f"- score machine python: {eval_run['machine']['python']}",
f"- score command: {eval_run['command']}",
"",
"## What Was Scored",
"",
f"- model: {eval_run['model_name']}",
f"- checkpoint: {eval_run['checkpoint'] or '(not recorded)'}",
f"- config: {eval_run['config'] or '(not recorded)'}",
f"- dictionary: {eval_run['dictionary'] or '(not recorded)'}",
f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}",
f"- normalization profile: {eval_run['normalization_profile']}",
f"- label file: {eval_run['label_file']}",
f"- prediction file: {eval_run['prediction_file']}",
f"- scored lines: {metrics['n_lines']}",
f"- unmatched labels: {metrics['unmatched_labels']}",
f"- unmatched predictions: {metrics['unmatched_predictions']}",
"",
"## Ground Truth And Pack",
"",
f"- eval pack version: {eval_run['eval_pack_version'] or '(missing)'}",
f"- frozen eval pack: {eval_run['frozen_eval_pack']}",
f"- unseen for selection: {eval_run['unseen_for_selection']}",
f"- selected checkpoint on test: {eval_run['selected_checkpoint_on_test']}",
f"- languages: {eval_run['languages']}",
f"- scripts: {eval_run['scripts']}",
f"- domains: {eval_run['domains']}",
f"- GT statuses: {eval_run['gt_statuses']}",
f"- splits: {eval_run['splits']}",
f"- script inventory file: {metrics['script_inventory']['inventory_file']}",
f"- observed codepoints/graphemes: {metrics['script_inventory']['observed_codepoint_count']}/{metrics['script_inventory']['observed_grapheme_count']}",
f"- missing required codepoints: {metrics['script_inventory']['missing_required_codepoints']}",
f"- missing required graphemes: {metrics['script_inventory']['missing_required_graphemes']}",
f"- metadata contradictions: {metrics['metadata_contradictions']}",
f"- provenance coverage: {metrics['provenance_coverage']}",
f"- normalizer counts: {metrics.get('normalizer_counts', {})}",
f"- normalization changed GT/prediction/any: {metrics['normalization_summary']['gt_changed']}/{metrics['normalization_summary']['prediction_changed']}/{metrics['normalization_summary']['any_changed']} of {metrics['normalization_summary']['total']}",
f"- dictionary coverage checked: {metrics['dictionary_coverage']['checked']}",
f"- dictionary GT codepoints/dictionary codepoints: {metrics['dictionary_coverage']['gt_codepoint_count']}/{metrics['dictionary_coverage']['dictionary_codepoint_count']}",
f"- dictionary missing GT codepoints: {metrics['dictionary_coverage']['missing_gt_codepoints']}",
f"- leakage checked: {metrics['leakage']['checked']}",
"",
"## Metrics",
"",
f"- codepoint CER micro: {metrics['cer_codepoint_micro']:.6f}",
f"- grapheme CER micro: {metrics['cer_grapheme_micro']:.6f}",
f"- exact-line accuracy: {metrics['exact_line_accuracy']:.6f}",
f"- codepoint edits/ref: {metrics['codepoint_edits']}/{metrics['codepoint_ref_len']}",
f"- grapheme edits/ref: {metrics['grapheme_edits']}/{metrics['grapheme_ref_len']}",
f"- zero-length GT lines: {metrics['zero_length_gt_lines']}",
f"- uncertainty method: {metrics['uncertainty']['method']}",
f"- uncertainty scoring units: {metrics['uncertainty']['effective_sample_size']}/{metrics['uncertainty']['min_claim_scoring_units']}",
f"- grapheme CER 95% CI: {metrics['uncertainty']['metrics']['cer_grapheme_micro']['low']:.6f}..{metrics['uncertainty']['metrics']['cer_grapheme_micro']['high']:.6f}",
"",
"## Slices",
"",
f"- {metrics['slices'][0]['language']} / {metrics['slices'][0]['script']} / {metrics['slices'][0]['domain']} / {metrics['slices'][0]['gt_status']} / {metrics['slices'][0]['split']}: "
f"n={metrics['slices'][0]['n_lines']} graph_CER={metrics['slices'][0]['cer_grapheme_micro']:.6f} "
f"cp_CER={metrics['slices'][0]['cer_codepoint_micro']:.6f} exact={metrics['slices'][0]['exact_line_accuracy']:.6f}",
"",
"## Review Artifacts",
"",
"- audit JSON: audit.json",
"- eval run JSON: score/eval-run.json",
"- metrics JSON: score/metrics.json",
"- metrics Markdown: score/metrics.md",
"- command text: score/command.txt",
"- predictions TSV: score/predictions.tsv",
"- unmatched TSV: score/unmatched.tsv",
"- worst lines: score/worst-lines.csv",
"- review CSV: score/sample-review.csv",
"- review HTML: score/sample-review.html",
"- confusions JSON: score/confusions.json",
"",
"## Warnings",
"",
"- none",
]
)
+ "\n",
encoding="utf-8",
)
def test_audit_eval_run_accepts_claim_safe_recognizer_score(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
result = audit_tool.audit_run(args.out)
assert result["eval_mode"] == "recognizer_oracle_crop"
assert result["audit_passed"] is True
assert result["claim_safe"] is True
assert result["issues"] == []
def test_audit_eval_run_accepts_claim_safe_page_score(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
result = audit_tool.audit_run(args.out)
assert result["eval_mode"] == "page_level_ocr"
assert result["audit_passed"] is True
assert result["claim_safe"] is True
assert result["issues"] == []
def test_audit_eval_run_replays_page_output_audit_artifacts(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.ocr_output_kind = "live_detector"
_write_page_output_audits(args.documents_dir)
scorer.run_eval(args)
clean = audit_tool.audit_run(args.out)
assert clean["audit_passed"] is True
(args.documents_dir / "page-1" / "limbu-output-audit.json").unlink()
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page_output_audit does not match current page output audit artifacts" in issue for issue in result["issues"])
assert any("replayed page output audits are not all passing: 29/30" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_recognizer_run_identity_lists(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["languages"] = ["wrong-lang"]
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("eval-run.json languages does not match predictions.tsv language values" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_page_run_identity_lists(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["scripts"] = ["WrongScript"]
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page-eval-run.json scripts does not match page-metrics.csv script values" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_recognizer_normalization_profile(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["normalization_profile"] = "NFKC_strip_whitespace"
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("normalization_profile does not match supported metric recomputation profile" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_page_normalization_profile(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["normalization_profile"] = "NFKC_strip_whitespace"
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("normalization_profile does not match supported metric recomputation profile" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_incomplete_recognizer_machine_metadata(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["machine"] = {"platform": ""}
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "eval run machine.platform is missing" in result["issues"]
assert "eval run machine.python is missing" in result["issues"]
def test_audit_eval_run_rejects_incomplete_page_machine_metadata(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["machine"] = {"python": ""}
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "eval run machine.platform is missing" in result["issues"]
assert "eval run machine.python is missing" in result["issues"]
def test_audit_eval_run_rejects_page_spotchecked_gt_without_calibration(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()]
for row in rows:
row["metadata"]["gt_status"] = "machine_converted_spot_checked"
args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
_refresh_page_pack_digest(args)
scorer.run_eval(args)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page spot-checked machine GT is missing calibration summary" in result["issues"]
def test_audit_eval_run_recomputes_recognizer_spotcheck_calibration_summary(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
for row in rows:
row["metadata"]["gt_status"] = "machine_converted_spot_checked"
args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
args.gt_status = "machine_converted_spot_checked"
_refresh_recognizer_pack_digest(args)
calibration = tmp_path / "calibration.json"
calibration.write_text(
json.dumps(
{
"claim_evidence_eligible": True,
"recommended_pack_gt_status": "machine_converted_spot_checked",
"language": "fixture-lang",
"script": "FixtureScript",
"domain": "real_print",
"source_gt_status": "machine_converted",
"counts": {"usable_rows": 1},
"machine_label_metrics": {"cer_grapheme_micro": 0.0},
"eligibility_reasons": [],
}
)
+ "\n",
encoding="utf-8",
)
args.spotcheck_calibration_file = calibration
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run["spotcheck_calibration"]["claim_evidence_eligible"] = False
eval_run["spotcheck_calibration"]["language"] = "wrong-language"
metrics["spotcheck_calibration"]["counts"] = {"usable_rows": 0}
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer eval-run spotcheck_calibration does not match calibration file" in issue for issue in result["issues"])
assert any("recognizer metrics spotcheck_calibration does not match calibration file" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_spotcheck_calibration_summary(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()]
for row in rows:
row["metadata"]["gt_status"] = "machine_converted_spot_checked"
args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
args.gt_status = "machine_converted_spot_checked"
_refresh_page_pack_digest(args)
calibration = tmp_path / "calibration.json"
calibration.write_text(
json.dumps(
{
"claim_evidence_eligible": True,
"recommended_pack_gt_status": "machine_converted_spot_checked",
"language": "fixture-lang",
"script": "FixtureScript",
"domain": "real_print",
"source_gt_status": "machine_converted",
"counts": {"usable_rows": 1},
"machine_label_metrics": {"cer_grapheme_micro": 0.0},
"eligibility_reasons": [],
}
)
+ "\n",
encoding="utf-8",
)
args.spotcheck_calibration_file = calibration
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run["spotcheck_calibration"]["claim_evidence_eligible"] = False
eval_run["spotcheck_calibration"]["language"] = "wrong-language"
metrics["spotcheck_calibration"]["counts"] = {"usable_rows": 0}
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page eval-run spotcheck_calibration does not match calibration file" in issue for issue in result["issues"])
assert any("page metrics spotcheck_calibration does not match calibration file" in issue for issue in result["issues"])
def test_audit_eval_run_reports_missing_review_artifact(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "sample-review.html").unlink()
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "missing required artifact: sample-review.html" in result["issues"]
def test_audit_eval_run_recomputes_recognizer_worst_review_artifacts(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
worst_path = args.out / "worst-lines.csv"
with worst_path.open("r", encoding="utf-8", newline="") as handle:
worst_rows = list(csv.DictReader(handle))
fieldnames = list(worst_rows[0])
worst_rows[0]["sample_id"] = "not-the-worst"
with worst_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(worst_rows)
review_path = args.out / "sample-review.csv"
with review_path.open("r", encoding="utf-8", newline="") as handle:
review_rows = list(csv.DictReader(handle))
review_fieldnames = list(review_rows[0])
review_rows[0]["sample_id"] = "not-the-worst"
with review_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=review_fieldnames)
writer.writeheader()
writer.writerows(review_rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("worst-lines.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"])
assert any("sample-review.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_recognizer_worst_fields(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
worst_path = args.out / "worst-lines.csv"
with worst_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
fieldnames = list(rows[0])
rows[0]["language"] = "wrong-language"
rows[0]["source_document"] = "wrong.pdf"
rows[0]["image_sha256"] = "bad-hash"
with worst_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "worst-lines.csv eval-1 language mismatch" in result["issues"]
assert "worst-lines.csv eval-1 source_document mismatch" in result["issues"]
assert "worst-lines.csv eval-1 image_sha256 mismatch" in result["issues"]
def test_audit_eval_run_rejects_stale_recognizer_review_html(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "sample-review.html").write_text(
"
Wrong review sample
\n",
encoding="utf-8",
)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("sample-review.html missing recognizer review sample eval-1 fields" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_recognizer_review_csv_fields(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
review_path = args.out / "sample-review.csv"
with review_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
fieldnames = list(rows[0])
rows[0]["gt_text"] = "stale-gt"
rows[0]["prediction_score"] = "0.123"
with review_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "sample-review.csv eval-1 gt_text mismatch" in result["issues"]
assert "sample-review.csv eval-1 prediction_score mismatch" in result["issues"]
def test_audit_eval_run_rejects_duplicate_recognizer_output_ids(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
predictions_path = args.out / "predictions.tsv"
with predictions_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle, delimiter="\t"))
fieldnames = list(rows[0])
rows.append(dict(rows[0]))
with predictions_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t")
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "predictions.tsv duplicate sample_id values: ['eval-1']" in result["issues"]
assert "predictions.tsv duplicate image_path values: ['images/line-1.png']" in result["issues"]
def test_audit_eval_run_recomputes_recognizer_unmatched_rows(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_full_match = False
args.require_claim_safe = False
args.prediction_file.write_text("other-line.png\tzzz\t0.1\n", encoding="utf-8")
scorer.run_eval(args)
(args.out / "unmatched.tsv").write_text(
"kind\timage_key\timage_path\timage_sha256\ttext\tprediction_score\n"
"label_without_prediction\tline-1.png\timages/not-line-1.png\tbad-hash\tabc\t\n"
"prediction_without_label\tother-line.png\tother-line.png\t\twrong-text\t0.9\n",
encoding="utf-8",
)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("unmatched.tsv rows do not match recomputed label/prediction misses" in issue for issue in result["issues"])
def test_audit_eval_run_reports_missing_direct_claim_report(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "claim-safety.md").unlink()
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "missing required artifact: claim-safety.md" in result["issues"]
def test_audit_eval_run_reports_missing_page_review_artifact(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "sample-review.html").unlink()
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "missing required artifact: sample-review.html" in result["issues"]
def test_audit_eval_run_recomputes_page_worst_review_artifacts(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
worst_path = args.out / "worst-pages.csv"
with worst_path.open("r", encoding="utf-8", newline="") as handle:
worst_rows = list(csv.DictReader(handle))
fieldnames = list(worst_rows[0])
worst_rows[0]["sample_id"] = "not-the-worst-page"
with worst_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(worst_rows)
review_path = args.out / "sample-review.csv"
with review_path.open("r", encoding="utf-8", newline="") as handle:
review_rows = list(csv.DictReader(handle))
review_fieldnames = list(review_rows[0])
review_rows[0]["sample_id"] = "not-the-worst-page"
with review_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=review_fieldnames)
writer.writeheader()
writer.writerows(review_rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("worst-pages.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"])
assert any("sample-review.csv sample order does not match recomputed worst examples" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_page_worst_fields(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
worst_path = args.out / "worst-pages.csv"
with worst_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
fieldnames = list(rows[0])
rows[0]["language"] = "wrong-language"
rows[0]["source_document"] = "wrong.pdf"
rows[0]["reference_sha256"] = "bad-hash"
with worst_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "worst-pages.csv page-1 language mismatch" in result["issues"]
assert "worst-pages.csv page-1 source_document mismatch" in result["issues"]
assert "worst-pages.csv page-1 reference_sha256 mismatch" in result["issues"]
def test_audit_eval_run_rejects_stale_page_review_html(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "sample-review.html").write_text(
"Wrong page review sample
\n",
encoding="utf-8",
)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("sample-review.html missing page review sample page-1 fields" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_page_review_csv_fields(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
review_path = args.out / "sample-review.csv"
with review_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
fieldnames = list(rows[0])
rows[0]["input_path"] = "stale-page.png"
rows[0]["text_cer_grapheme"] = "0.5"
with review_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "sample-review.csv page-1 input_path mismatch" in result["issues"]
assert any("sample-review.csv page-1 text_cer_grapheme mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_duplicate_page_output_ids(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
page_metrics_path = args.out / "page-metrics.csv"
with page_metrics_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
fieldnames = list(rows[0])
rows.append(dict(rows[0]))
with page_metrics_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page-metrics.csv duplicate sample_id values: ['page-1']" in result["issues"]
assert any("page-metrics.csv duplicate input_path values" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_duplicate_page_manifest_image_paths(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()]
rows[1]["image_path"] = rows[0]["image_path"]
args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page eval manifest duplicate image_path" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_page_manifest_image_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()]
rows[0]["sha256"] = "0" * 64
args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page eval manifest page-1 sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_missing_rows(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
(args.documents_dir / "page-1" / "document.json").unlink()
scorer.run_eval(args)
(args.out / "missing.tsv").write_text("kind,sample_id\nmissing_document,not-page-1\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("missing.tsv rows do not match recomputed manifest/document/reference misses" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_page_metric_identity_fields(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
page_metrics_path = args.out / "page-metrics.csv"
with page_metrics_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
assert rows
fieldnames = list(rows[0])
rows[0]["claim_eligible"] = "False"
rows[0]["source_document"] = "wrong.pdf"
rows[0]["source_page"] = "99"
rows[0]["source_bbox"] = "[9,9,9,9]"
rows[0]["reference_fields"] = "text"
with page_metrics_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any(
"page-metrics.csv page-1 claim_eligible does not match eval manifest/reference" in issue
for issue in result["issues"]
)
assert any(
"page-metrics.csv page-1 reference_fields does not match eval manifest/reference" in issue
for issue in result["issues"]
)
assert any(
"page-metrics.csv page-1 source_document does not match eval manifest/reference" in issue
for issue in result["issues"]
)
assert any(
"page-metrics.csv page-1 source_page does not match eval manifest/reference" in issue
for issue in result["issues"]
)
assert any(
"page-metrics.csv page-1 source_bbox does not match eval manifest/reference" in issue
for issue in result["issues"]
)
def test_audit_eval_run_reports_malformed_direct_claim_report(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "claim-safety.md").write_text("# incomplete\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "claim-safety.md missing section: ## What Was Scored" in result["issues"]
def test_audit_eval_run_rejects_stale_recognizer_claim_report(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
report_path = args.out / "claim-safety.md"
report = report_path.read_text(encoding="utf-8")
assert "- claim safe: True" in report
assert "- dictionary missing GT codepoints: []" in report
eval_run = json.loads((args.out / "eval-run.json").read_text(encoding="utf-8"))
command_line = f"- command: {eval_run['command']}"
artifact_line = f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}"
assert command_line in report
assert artifact_line in report
assert "- sample-review.html" in report
expected_slice = "- fixture-lang / FixtureScript / real_print / human_verified / eval: n=30 graph_CER=0.000000 cp_CER=0.000000 exact=1.000000"
assert expected_slice in report
report = report.replace("- claim safe: True", "- claim safe: False")
report = report.replace(command_line, "- command: stale")
report = report.replace(artifact_line, "- artifact identity: {}")
report = report.replace("- sample-review.html", "- stale-review.html")
report = report.replace("- dictionary missing GT codepoints: []", "- dictionary missing GT codepoints: ['U+1234']")
report = report.replace(expected_slice, expected_slice.replace("n=30", "n=0"))
report_path.write_text(report, encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "claim-safety.md missing or stale line: - claim safe: True" in result["issues"]
assert f"claim-safety.md missing or stale line: {command_line}" in result["issues"]
assert f"claim-safety.md missing or stale line: {artifact_line}" in result["issues"]
assert "claim-safety.md missing or stale line: - sample-review.html" in result["issues"]
assert "claim-safety.md missing or stale line: - dictionary missing GT codepoints: []" in result["issues"]
assert f"claim-safety.md missing or stale line: {expected_slice}" in result["issues"]
def test_audit_eval_run_rejects_stale_page_claim_report(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
report_path = args.out / "claim-safety.md"
report = report_path.read_text(encoding="utf-8")
assert "- scored pages: 30" in report
assert "- normalization changed reference/prediction/any: 0/0/0 of 30" in report
eval_run = json.loads((args.out / "page-eval-run.json").read_text(encoding="utf-8"))
machine_line = f"- machine platform: {eval_run['machine']['platform']}"
artifact_line = f"- artifact identity: {json.dumps(eval_run.get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}"
assert machine_line in report
assert artifact_line in report
assert "- line-alignment.csv" in report
expected_slice = "- fixture-lang / FixtureScript / real_print / human_verified / eval: pages=30 text_graph_CER=0.000000 text_cp_CER=0.000000 exact=1.000000 line_F1=1.000000"
assert expected_slice in report
report = report.replace("- scored pages: 30", "- scored pages: 0")
report = report.replace(machine_line, "- machine platform: stale")
report = report.replace(artifact_line, "- artifact identity: {}")
report = report.replace("- line-alignment.csv", "- stale-line-alignment.csv")
report = report.replace(
"- normalization changed reference/prediction/any: 0/0/0 of 30",
"- normalization changed reference/prediction/any: 1/0/1 of 30",
)
report = report.replace(expected_slice, expected_slice.replace("pages=30", "pages=0"))
report_path.write_text(report, encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "claim-safety.md missing or stale line: - scored pages: 30" in result["issues"]
assert f"claim-safety.md missing or stale line: {machine_line}" in result["issues"]
assert f"claim-safety.md missing or stale line: {artifact_line}" in result["issues"]
assert "claim-safety.md missing or stale line: - line-alignment.csv" in result["issues"]
assert "claim-safety.md missing or stale line: - normalization changed reference/prediction/any: 0/0/0 of 30" in result["issues"]
assert f"claim-safety.md missing or stale line: {expected_slice}" in result["issues"]
def test_audit_eval_run_checks_page_recipe_root_claim_safety_report(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
recipe_tool = _load_tool("run_eval_recipe")
args = _write_page_fixture(tmp_path)
score = scorer.run_eval(args)
root = tmp_path / "page-recipe-root"
root.mkdir()
recipe = {"id": "page-recipe", "task": "page_level_ocr"}
(root / "recipe.json").write_text(json.dumps(recipe) + "\n", encoding="utf-8")
recipe_run = {"recipe_id": "page-recipe", "audit": {"audit_passed": True}}
report_path = root / "claim-safety.md"
recipe_tool._write_claim_safety_report(report_path, "page-recipe", recipe, score, {"audit_passed": True})
issues = audit_tool._check_root_claim_safety_report_consistency(root, recipe_run, score["eval_run"], score["metrics"])
assert issues == []
report = report_path.read_text(encoding="utf-8")
assert "- eval manifest: " + str(args.eval_manifest) in report
assert "- scored pages: 30" in report
assert "- text grapheme CER micro: 0.000000" in report
assert "- normalization changed reference/prediction/any: 0/0/0 of 30" in report
assert "- missing required codepoints: []" in report
assert "- text grapheme CER 95% CI: 0.000000..0.000000" in report
assert "- line alignment CSV: score/line-alignment.csv" in report
assert "- confusions JSON: score/confusions.json" in report
command_line = f"- score command: {score['eval_run']['command']}"
artifact_line = f"- artifact identity: {json.dumps(score['eval_run'].get('artifact_identity', {}), ensure_ascii=False, sort_keys=True)}"
assert command_line in report
assert artifact_line in report
expected_slice = "- fixture-lang / FixtureScript / real_print / human_verified / eval: pages=30 text_graph_CER=0.000000 text_cp_CER=0.000000 exact=1.000000 line_F1=1.000000"
assert expected_slice in report
report = report.replace("- scored pages: 30", "- scored pages: 0")
report = report.replace(command_line, "- score command: stale")
report = report.replace(artifact_line, "- artifact identity: {}")
report = report.replace("- missing required codepoints: []", "- missing required codepoints: ['U+1234']")
report = report.replace("- text grapheme CER 95% CI: 0.000000..0.000000", "- text grapheme CER 95% CI: 0.100000..0.200000")
report = report.replace("- line alignment CSV: score/line-alignment.csv", "- line alignment CSV: stale.csv")
report = report.replace(
"- normalization changed reference/prediction/any: 0/0/0 of 30",
"- normalization changed reference/prediction/any: 1/0/1 of 30",
)
report = report.replace(expected_slice, expected_slice.replace("pages=30", "pages=0"))
report_path.write_text(report, encoding="utf-8")
stale_issues = audit_tool._check_root_claim_safety_report_consistency(root, recipe_run, score["eval_run"], score["metrics"])
assert "root claim-safety.md missing or stale line: - scored pages: 30" in stale_issues
assert f"root claim-safety.md missing or stale line: {command_line}" in stale_issues
assert f"root claim-safety.md missing or stale line: {artifact_line}" in stale_issues
assert "root claim-safety.md missing or stale line: - missing required codepoints: []" in stale_issues
assert "root claim-safety.md missing or stale line: - text grapheme CER 95% CI: 0.000000..0.000000" in stale_issues
assert "root claim-safety.md missing or stale line: - line alignment CSV: score/line-alignment.csv" in stale_issues
assert "root claim-safety.md missing or stale line: - normalization changed reference/prediction/any: 0/0/0 of 30" in stale_issues
assert f"root claim-safety.md missing or stale line: {expected_slice}" in stale_issues
def test_audit_eval_run_checks_page_recipe_root_document_lineage(tmp_path: Path) -> None:
audit_tool = _load_tool("audit_eval_run")
root = tmp_path / "page-recipe-root"
materialize = root / "materialize"
documents_dir = root / "documents"
document_json = documents_dir / "page-1" / "document.json"
materialize.mkdir(parents=True)
document_json.parent.mkdir(parents=True)
document_json.write_text('{"pages":[]}\n', encoding="utf-8")
for rel_path in audit_tool.RECIPE_ROOT_ARTIFACTS:
path = root / rel_path
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("fixture\n", encoding="utf-8")
assert not (root / "predictions.txt").exists()
assert audit_tool._check_required_artifacts(root, audit_tool.RECIPE_ROOT_ARTIFACTS) == []
materialization = {
"records": [
{
"sample_id": "page-1",
"output_path": str(document_json),
"output_sha256": audit_tool._sha256_file(document_json),
}
],
"command_template": "fixture",
}
(materialize / "output-command-run.json").write_text(json.dumps(materialization) + "\n", encoding="utf-8")
(materialize / "command-template.txt").write_text("fixture\n", encoding="utf-8")
(materialize / "output-command-run.md").write_text(
f"- records: 1\n- page-1: `{document_json}` sha256={audit_tool._sha256_file(document_json)}\n",
encoding="utf-8",
)
recipe_run = {
"outputs": {"documents_dir": str(documents_dir)},
"materialization": materialization,
}
eval_run = {"eval_mode": "page_level_ocr", "documents_dir": str(documents_dir)}
issues = audit_tool._check_recipe_output_lineage(root, recipe_run, eval_run)
assert issues == []
recipe_run["outputs"]["documents_dir"] = str(root / "wrong-documents")
stale_issues = audit_tool._check_recipe_output_lineage(root, recipe_run, eval_run)
assert "recipe-run.json outputs.documents_dir does not match score eval-run documents_dir" in stale_issues
assert any("output path does not exist for documents_dir" in issue for issue in stale_issues)
recipe_run["outputs"]["documents_dir"] = str(documents_dir)
outside = root / "outside-document.json"
outside.write_text("{}\n", encoding="utf-8")
materialization["records"][0]["output_path"] = str(outside)
outside_issues = audit_tool._check_recipe_output_lineage(root, recipe_run, eval_run)
assert "recipe-run.json materialization record 0 output_path is not under outputs.documents_dir" in outside_issues
def test_audit_eval_run_checks_page_materialization_input_identity(tmp_path: Path) -> None:
audit_tool = _load_tool("audit_eval_run")
page_image = tmp_path / "page.png"
page_image.write_bytes(b"page image")
document_json = tmp_path / "documents" / "page-1" / "document.json"
document_json.parent.mkdir(parents=True)
document_json.write_text('{"pages":[]}\n', encoding="utf-8")
stdout_path = tmp_path / "stdout.txt"
stderr_path = tmp_path / "stderr.txt"
stdout_path.write_text("", encoding="utf-8")
stderr_path.write_text("", encoding="utf-8")
materialize_dir = tmp_path / "materialize"
payload = {
"mode": "page",
"command_template": "fixture {input_path} {sample_id} {output_dir} {document_json} {documents_dir} {out_dir}",
"documents_dir": str(document_json.parent.parent),
"record_count": 1,
"records": [
{
"sample_id": "page-1",
"command": [
"fixture",
str(page_image),
"page-1",
str(document_json.parent),
str(document_json),
str(document_json.parent.parent),
str(materialize_dir),
],
"stdout_path": str(stdout_path),
"stdout_sha256": audit_tool._sha256_file(stdout_path),
"stderr_path": str(stderr_path),
"stderr_sha256": audit_tool._sha256_file(stderr_path),
"returncode": 0,
"input_path": str(page_image),
"input_sha256": audit_tool._sha256_file(page_image),
"output_path": str(document_json),
"output_sha256": audit_tool._sha256_file(document_json),
"output_size_bytes": document_json.stat().st_size,
}
],
}
assert audit_tool._check_materialization_records(payload, materialize_dir) == []
payload["records"][0]["command"] = ["stale-command"]
command_issues = audit_tool._check_materialization_records(payload, materialize_dir)
assert "materialization records[0] command does not match materialization command_template" in command_issues
payload["records"][0]["command"] = [
"fixture",
str(page_image),
"page-1",
str(document_json.parent),
str(document_json),
str(document_json.parent.parent),
str(materialize_dir),
]
payload["records"][0]["input_sha256"] = "0" * 64
issues = audit_tool._check_materialization_records(payload)
assert any("materialization records[0] input_sha256 mismatch" in issue for issue in issues)
def test_audit_eval_run_rejects_stale_recognizer_metrics_report(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
report_path = args.out / "metrics.md"
report = report_path.read_text(encoding="utf-8")
assert "- codepoint CER: 0.000000" in report
assert "- normalization changed GT/prediction/any: 0/0/0 of 30" in report
report = report.replace("- codepoint CER: 0.000000", "- codepoint CER: 0.500000")
report = report.replace(
"- normalization changed GT/prediction/any: 0/0/0 of 30",
"- normalization changed GT/prediction/any: 1/0/1 of 30",
)
report_path.write_text(report, encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "metrics.md missing or stale line: - codepoint CER: 0.000000" in result["issues"]
assert "metrics.md missing or stale line: - normalization changed GT/prediction/any: 0/0/0 of 30" in result["issues"]
def test_audit_eval_run_rejects_stale_page_metrics_report(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
report_path = args.out / "metrics.md"
report = report_path.read_text(encoding="utf-8")
assert "- scored pages: 30" in report
assert "- dictionary missing reference codepoints: []" in report
report = report.replace("- scored pages: 30", "- scored pages: 0")
report = report.replace("- dictionary missing reference codepoints: []", "- dictionary missing reference codepoints: ['U+1234']")
report_path.write_text(report, encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "metrics.md missing or stale line: - scored pages: 30" in result["issues"]
assert "metrics.md missing or stale line: - dictionary missing reference codepoints: []" in result["issues"]
def test_audit_eval_run_reports_missing_timestamp(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run.pop("created_at")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "eval-run.json missing required field: created_at" in result["issues"]
assert "eval run created_at timestamp is missing" in result["issues"]
def test_audit_eval_run_reports_command_file_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "command.txt").write_text("different command\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "command.txt does not match eval run command" in result["issues"]
def test_audit_eval_run_reports_recognizer_input_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
args.prediction_file.write_text("line-1.png\tchanged\t1.0\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer prediction file sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_reports_recognizer_image_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
image_path = args.data_dir / "images" / "line-1.png"
image_path.write_bytes(image_path.read_bytes() + b"changed")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("predictions.tsv eval-1 image_sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_prediction_rows(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
predictions = args.out / "predictions.tsv"
lines = predictions.read_text(encoding="utf-8").splitlines()
lines[1] = lines[1].replace("\t0\t0\tTrue", "\t1\t1\tFalse")
predictions.write_text("\n".join(lines) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("codepoint_edits mismatch" in issue for issue in result["issues"])
assert any("grapheme_edits mismatch" in issue for issue in result["issues"])
assert any("exact mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_recognizer_prediction_source_rows(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
predictions = args.out / "predictions.tsv"
with predictions.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle, delimiter="\t"))
assert rows
fieldnames = list(rows[0])
rows[0]["prediction_score"] = "0.123"
rows[0]["claim_eligible"] = "False"
rows[0]["source_document"] = "wrong.pdf"
with predictions.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t")
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "predictions.tsv eval-1 prediction_score does not match source files" in result["issues"]
assert "predictions.tsv eval-1 claim_eligible does not match source files" in result["issues"]
assert "predictions.tsv eval-1 source_document does not match source files" in result["issues"]
def test_audit_eval_run_rejects_recognizer_manifest_image_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
rows[0]["sha256"] = "0" * 64
args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer manifest eval-1 sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_recognizer_manifest_text_and_sample_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
rows[0]["text_sha256"] = "0" * 64
rows[0]["sample_sha256"] = "1" * 64
args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer manifest eval-1 text_sha256 mismatch" in issue for issue in result["issues"])
assert any("recognizer manifest eval-1 sample_sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_pack_content_digest(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run["pack_content_sha256"] = "0" * 64
metrics["pack_content_sha256"] = "1" * 64
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "eval-run.json pack_content_sha256 mismatch" in "\n".join(result["issues"])
assert "metrics.json pack_content_sha256 mismatch" in "\n".join(result["issues"])
def test_audit_eval_run_recomputes_page_pack_content_digest(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run["pack_content_sha256"] = "0" * 64
metrics["pack_content_sha256"] = "1" * 64
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page-eval-run.json pack_content_sha256 mismatch" in "\n".join(result["issues"])
assert "metrics.json pack_content_sha256 mismatch" in "\n".join(result["issues"])
def test_audit_eval_run_recomputes_recognizer_aggregate_metrics(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["codepoint_edits"] = 1
metrics["cer_codepoint_micro"] = 1 / metrics["codepoint_ref_len"]
metrics["grapheme_edits"] = 1
metrics["cer_grapheme_micro"] = 1 / metrics["grapheme_ref_len"]
metrics["exact_line_accuracy"] = 0.0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("metrics codepoint_edits does not match recomputed predictions.tsv value" in issue for issue in result["issues"])
assert any("metrics exact_line_accuracy does not match recomputed predictions.tsv value" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_uncertainty(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["uncertainty"]["metrics"]["cer_grapheme_micro"]["high"] = 0.5
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer uncertainty cer_grapheme_micro.high mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_script_inventory(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
eval_run_path = args.out / "eval-run.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics["script_inventory"]["observed_codepoint_count"] = 999
eval_run["script_inventory"]["observed_grapheme_count"] = 999
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer metrics script_inventory observed_codepoint_count mismatch" in issue for issue in result["issues"])
assert any("recognizer eval-run script_inventory observed_grapheme_count mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_match_count_metrics(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
(args.data_dir / "images" / "line-31.png").write_bytes(b"eval-image-31")
with args.label_file.open("a", encoding="utf-8") as handle:
handle.write("images/line-31.png\tdef\n")
with args.prediction_file.open("a", encoding="utf-8") as handle:
handle.write("extra.png\tzzz\t0.5\n")
args.require_full_match = False
args.require_claim_safe = False
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["label_lines"] == 31
assert metrics["matched_lines"] == 30
assert metrics["unmatched_labels"] == 1
assert metrics["unmatched_predictions"] == 1
assert metrics["prediction_match_rate"] == 30 / 31
metrics["label_lines"] = 1
metrics["matched_lines"] = 2
metrics["n_lines"] = 2
metrics["unmatched_labels"] = 0
metrics["unmatched_predictions"] = 0
metrics["prediction_match_rate"] = 1.0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("metrics label_lines does not match recomputed recognizer source value" in issue for issue in result["issues"])
assert any("metrics matched_lines does not match recomputed recognizer source value" in issue for issue in result["issues"])
assert any("metrics unmatched_labels does not match recomputed recognizer source value" in issue for issue in result["issues"])
assert any("metrics unmatched_predictions does not match recomputed recognizer source value" in issue for issue in result["issues"])
assert any("metrics prediction_match_rate does not match recomputed recognizer source value" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_claim_safe_with_low_recognizer_match_rate(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
(args.data_dir / "images" / "line-31.png").write_bytes(b"extra-eval-image")
with args.label_file.open("a", encoding="utf-8") as handle:
handle.write("images/line-31.png\tdef\n")
with args.prediction_file.open("a", encoding="utf-8") as handle:
handle.write("extra.png\tzzz\t0.5\n")
args.require_full_match = False
args.require_claim_safe = False
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer claim_safe is true despite unmatched prediction/label rows" in result["issues"]
assert "recognizer claim_safe is true despite low prediction match rate" in result["issues"]
assert any("missing claim-safety warning for unmatched prediction/label rows" in issue for issue in result["issues"])
assert any("missing claim-safety warning for low prediction match rate" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_confusions(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "confusions.json").write_text('{"stale->artifact": 1}\n', encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("confusions.json does not match recomputed predictions.tsv confusions" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_confusions(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
(args.out / "confusions.json").write_text('{"stale->artifact": 1}\n', encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("confusions.json does not match recomputed text-pairs.tsv confusions" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_slice_metrics(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["slices"][0]["cer_grapheme_micro"] = 0.5
metrics["slices"][0]["exact_line_accuracy"] = 0.0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer slice" in issue and "cer_grapheme_micro mismatch" in issue for issue in result["issues"])
assert any("recognizer slice" in issue and "exact_line_accuracy mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_tampered_recognizer_claim_safe_blockers(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
metrics["metadata_contradictions"] = ["eval-1: synthetic/transliterated provenance is declared as real domain"]
metrics["leakage"]["overlaps"]["sample_id"] = ["eval-1"]
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer claim_safe is true despite metadata contradictions" in result["issues"]
assert "recognizer claim_safe is true despite train/dev/eval leakage" in result["issues"]
assert any("missing claim-safety warning for metadata contradictions" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_claim_safe_with_undersized_recognizer_slice(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
manifest_rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
manifest_rows[0]["metadata"]["language"] = "other-lang"
args.manifest_file.write_text("\n".join(json.dumps(row) for row in manifest_rows) + "\n", encoding="utf-8")
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer claim_safe is true despite mixed languages" in result["issues"]
assert "recognizer claim_safe is true despite slice sample size too small" in result["issues"]
assert any("missing claim-safety warning for slice sample size too small" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_tampered_recognizer_unrun_leakage_check(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
args.train_manifest = []
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer claim_safe is true despite unrun leakage check" in result["issues"]
assert any("missing claim-safety warning for unrun leakage check" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_claim_safe_with_missing_recognizer_model_identity(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["model_name"] = ""
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer claim_safe is true despite missing model identity" in result["issues"]
assert any("missing claim-safety warning for missing model identity" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_tampered_recognizer_missing_manifest_summary_blockers(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
args.manifest_file = None
args.summary_file = None
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer claim_safe is true despite missing eval manifest" in result["issues"]
assert "recognizer claim_safe is true despite missing eval pack summary/datasheet" in result["issues"]
assert any("missing claim-safety warning for missing eval manifest" in issue for issue in result["issues"])
assert any("missing claim-safety warning for missing eval pack summary/datasheet" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_tampered_recognizer_row_claim_eligibility_blocker(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
rows[0]["metadata"]["claim_evidence_eligible"] = False
args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer claim_safe is true despite row-level claim ineligible values" in result["issues"]
assert any("missing claim-safety warning for row-level claim ineligible values" in issue for issue in result["issues"])
def test_audit_eval_run_reports_recognizer_artifact_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
Path(args.checkpoint).write_bytes(b"changed")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer artifact_identity.checkpoint sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_recognizer_artifact_identity_field_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["checkpoint"] = str(tmp_path / "other-checkpoint.pdparams")
eval_run["dictionary"] = str(tmp_path / "other-dict.txt")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer artifact_identity.checkpoint.value does not match eval-run checkpoint" in issue for issue in result["issues"])
assert any("recognizer artifact_identity.dictionary.value does not match eval-run dictionary" in issue for issue in result["issues"])
def test_audit_eval_run_reports_page_manifest_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
args.eval_manifest.write_text(args.eval_manifest.read_text(encoding="utf-8") + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page eval manifest sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_page_artifact_identity_field_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["config"] = str(tmp_path / "other-config.yml")
eval_run["dictionary"] = str(tmp_path / "other-dict.txt")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page artifact_identity.config.value does not match eval-run config" in issue for issue in result["issues"])
assert any("page artifact_identity.dictionary.value does not match eval-run dictionary" in issue for issue in result["issues"])
def test_audit_eval_run_reports_page_train_manifest_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
args.train_manifest[0].write_text(args.train_manifest[0].read_text(encoding="utf-8") + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("train_manifests[0] sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_leakage_report(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
eval_image = tmp_path / "pack" / "images" / "line-1.png"
args.train_manifest[0].write_text(
json.dumps(
{
"sample_id": "eval-1",
"image_path": "images/line-1.png",
"text": "abc",
"sha256": sha256_file(eval_image),
}
)
+ "\n",
encoding="utf-8",
)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["leakage"]["overlaps"] = {key: [] for key in metrics["leakage"]["overlaps"]}
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer leakage overlaps.sample_id mismatch" in issue for issue in result["issues"])
assert any("recognizer leakage overlaps.image_sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_leakage_report(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
image = tmp_path / "page.png"
write_manifest(
[
ManifestEntry(
sample_id="page-1",
dataset="fixture-pages",
split="train",
image_path=str(image),
text="abc",
sha256=sha256_file(image),
)
],
args.train_manifest[0],
)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["leakage"]["overlaps"] = {key: [] for key in metrics["leakage"]["overlaps"]}
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page leakage overlaps.sample_id mismatch" in issue for issue in result["issues"])
assert any("page leakage overlaps.image_sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_source_sample_id_leakage(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
train_image = tmp_path / "train-crop.png"
train_image.write_bytes(b"page-derived-crop")
args.train_manifest[0].write_text(
json.dumps(
{
"sample_id": "page-1::line-1",
"dataset": "fixture-page-derived-crops",
"split": "train",
"image_path": str(train_image),
"image_sha256": sha256_file(train_image),
"text": "शीर्षक",
"text_sha256": scorer._text_hash("शीर्षक"),
"source_sample_id": "page-1",
},
ensure_ascii=False,
)
+ "\n",
encoding="utf-8",
)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["leakage"]["overlaps"]["source_sample_id"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page leakage overlaps.source_sample_id mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_metadata_contradictions(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
rows[0]["metadata"]["label_provenance"] = "machine_converted_legacy_font"
args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["metadata_contradictions"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer metadata_contradictions do not match source metadata" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_metadata_contradictions(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
reference_payload = json.loads((tmp_path / "page.ref.json").read_text(encoding="utf-8"))
reference_payload["metadata"]["source_type"] = "synthetic"
(tmp_path / "page.ref.json").write_text(json.dumps(reference_payload) + "\n", encoding="utf-8")
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["metadata_contradictions"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page metadata_contradictions do not match source metadata" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_provenance_coverage(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
rows = [json.loads(line) for line in args.manifest_file.read_text(encoding="utf-8").splitlines()]
rows[0]["metadata"].pop("source_document")
rows[0]["metadata"].pop("bbox")
rows[0]["metadata"].pop("reviewed_by")
args.manifest_file.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["provenance_coverage"]["missing_source_document"] = 0
metrics["provenance_coverage"]["missing_bbox"] = 0
metrics["provenance_coverage"]["missing_human_review_evidence"] = 0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("recognizer provenance_coverage does not match source metadata" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_metadata_coverage(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()]
rows[0]["metadata"].pop("source_document")
rows[0]["metadata"].pop("page")
rows[0]["metadata"].pop("bbox")
rows[0]["metadata"].pop("claim_evidence_eligible")
rows[0]["metadata"].pop("reviewed_by")
rows[0]["metadata"].pop("language")
args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
_refresh_page_pack_digest(args)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["metadata_coverage"]["missing_source_document"] = 0
metrics["metadata_coverage"]["missing_page"] = 0
metrics["metadata_coverage"]["missing_bbox"] = 0
metrics["metadata_coverage"]["missing_claim_eligibility"] = 0
metrics["metadata_coverage"]["missing_human_review_evidence"] = 0
metrics["metadata_coverage"]["missing_language"] = 0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page metadata_coverage does not match eval manifest" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_reference_coverage(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["reference_coverage"] = {key: 0 for key in metrics["reference_coverage"]}
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page reference_coverage does not match page-metrics.csv" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_manifest_count_metrics(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()]
rows[0]["metadata"]["claim_evidence_eligible"] = False
args.eval_manifest.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
_refresh_page_pack_digest(args)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["claim_eligible_rows"] = metrics["manifest_samples"]
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page metrics claim_eligible_rows does not match manifest/doc/reference sources" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_document_source_path_mismatches(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
document_path = args.documents_dir / "page-1" / "document.json"
payload = json.loads(document_path.read_text(encoding="utf-8"))
payload["source_path"] = str(tmp_path / "wrong-page.png")
document_path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["document_source_path_mismatch_count"] == 1
metrics["document_source_path_mismatch_count"] = 0
metrics["document_source_path_mismatches"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page document_source_path_mismatches do not match scored documents" in issue for issue in result["issues"])
assert any("page document_source_path_mismatch_count does not match scored documents" in issue for issue in result["issues"])
def test_audit_eval_run_reports_page_document_hash_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
image_path = tmp_path / "page.png"
image_path.write_bytes(image_path.read_bytes() + b"changed")
document_path = args.documents_dir / "page-1" / "document.json"
document_path.write_text(document_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page-metrics.csv page-1 input_sha256 mismatch" in issue for issue in result["issues"])
assert any("page-metrics.csv page-1 document_sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_line_alignment_rows(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
page_metrics_path = args.out / "page-metrics.csv"
with page_metrics_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
assert rows
fieldnames = list(rows[0])
rows[0]["line_match_count"] = "0"
rows[0]["line_detection_f1"] = "0.0"
with page_metrics_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page-metrics.csv page-1 line_match_count mismatch" in issue for issue in result["issues"])
assert any("page-metrics.csv page-1 line_detection_f1 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_line_alignment_row_contents(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
alignment_path = args.out / "line-alignment.csv"
with alignment_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
assert rows
fieldnames = list(rows[0])
rows[0]["reference_line_id"] = "wrong-ref"
rows[0]["predicted_bbox"] = "[9,9,9,9]"
rows[0]["predicted_text"] = "wrong text"
with alignment_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("line-alignment.csv page-1 row 1 reference_line_id mismatch" in issue for issue in result["issues"])
assert any("line-alignment.csv page-1 row 1 predicted_bbox mismatch" in issue for issue in result["issues"])
assert any("line-alignment.csv page-1 row 1 predicted_text mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_metric_means(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["metric_means"]["line_detection_f1"] = 0.5
metrics["metric_means"]["reading_order_pair_accuracy"] = 0.5
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("metrics metric_means.line_detection_f1 does not match page-metrics.csv mean" in issue for issue in result["issues"])
assert any("metrics metric_means.reading_order_pair_accuracy does not match page-metrics.csv mean" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_text_metric_totals(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["text_codepoint_edits"] = 1
metrics["text_cer_codepoint_micro"] = 1 / metrics["text_codepoint_ref_len"]
metrics["text_grapheme_edits"] = 1
metrics["text_cer_grapheme_micro"] = 1 / metrics["text_grapheme_ref_len"]
metrics["text_exact_pages"] = 0
metrics["text_exact_accuracy"] = 0.0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("metrics text_codepoint_edits does not match page-metrics.csv sum" in issue for issue in result["issues"])
assert any("metrics text_cer_grapheme_micro does not match page-metrics.csv sum" in issue for issue in result["issues"])
assert any("metrics text_exact_pages does not match page-metrics.csv sum" in issue for issue in result["issues"])
assert any("metrics text_exact_accuracy does not match page-metrics.csv sum" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_uncertainty(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["uncertainty"]["metrics"]["text_cer_grapheme_micro"]["high"] = 0.5
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page uncertainty text_cer_grapheme_micro.high mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_script_inventory(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
eval_run_path = args.out / "page-eval-run.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics["script_inventory"]["observed_codepoint_count"] = 999
eval_run["script_inventory"]["observed_grapheme_count"] = 999
metrics_path.write_text(json.dumps(metrics, ensure_ascii=False) + "\n", encoding="utf-8")
eval_run_path.write_text(json.dumps(eval_run, ensure_ascii=False) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page metrics script_inventory observed_codepoint_count mismatch" in issue for issue in result["issues"])
assert any("page eval-run script_inventory observed_grapheme_count mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_text_row_cer(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
page_metrics_path = args.out / "page-metrics.csv"
with page_metrics_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
assert rows
fieldnames = list(rows[0])
rows[0]["text_codepoint_edits"] = "1"
rows[0]["text_cer_codepoint"] = "0.5"
rows[0]["text_exact"] = "False"
with page_metrics_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page-metrics.csv page-1 text_codepoint_edits mismatch" in issue for issue in result["issues"])
assert any("page-metrics.csv page-1 text_cer_codepoint mismatch" in issue for issue in result["issues"])
assert any("page-metrics.csv page-1 text_exact mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_page_text_pairs(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
text_pairs_path = args.out / "text-pairs.tsv"
with text_pairs_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle, delimiter="\t"))
assert rows
fieldnames = list(rows[0])
rows[0]["reference_text"] = "wrong reference"
rows[0]["text_codepoint_edits"] = "0"
rows[0]["text_exact"] = "True"
with text_pairs_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t")
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("text-pairs.tsv page-1 reference_text_sha256 mismatch" in issue for issue in result["issues"])
assert any("text-pairs.tsv page-1 text_codepoint_edits mismatch" in issue for issue in result["issues"])
assert any("text-pairs.tsv page-1 reference_text does not match reference source text" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_page_reference_artifacts(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
references_path = args.out / "reference-artifacts.tsv"
with references_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle, delimiter="\t"))
assert rows
fieldnames = list(rows[0])
rows[0]["reference_path"] = str(tmp_path / "wrong.ref.json")
rows[0]["reference_sha256"] = "0" * 64
rows[0]["reference_status"] = "unverified"
with references_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t")
writer.writeheader()
writer.writerows(rows)
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("reference-artifacts.tsv page-1 reference_path does not match reference source" in issue for issue in result["issues"])
assert any("reference-artifacts.tsv page-1 reference_sha256 does not match reference source" in issue for issue in result["issues"])
assert any("reference-artifacts.tsv page-1 reference_status does not match reference source" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_text_rows_from_document_reference(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
page_metrics_path = args.out / "page-metrics.csv"
with page_metrics_path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
assert rows
fieldnames = list(rows[0])
rows[0]["text_codepoint_edits"] = "1"
rows[0]["text_grapheme_edits"] = "1"
rows[0]["text_cer_codepoint"] = "0.3333333333333333"
rows[0]["text_cer_grapheme"] = "0.3333333333333333"
with page_metrics_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["text_codepoint_edits"] = 1
metrics["text_grapheme_edits"] = 1
metrics["text_cer_codepoint_micro"] = 1 / metrics["text_codepoint_ref_len"]
metrics["text_cer_grapheme_micro"] = 1 / metrics["text_grapheme_ref_len"]
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page-metrics.csv page-1 text_codepoint_edits mismatch" in issue for issue in result["issues"])
assert any("metrics text_codepoint_edits does not match page-metrics.csv sum" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_slice_metrics(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
metrics["slices"][0]["text_cer_grapheme_micro"] = 0.5
metrics["slices"][0]["text_exact_accuracy"] = 0.5
metrics["slices"][0]["metric_means"]["line_detection_f1"] = 0.5
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any("page slice" in issue and "text_cer_grapheme_micro mismatch" in issue for issue in result["issues"])
assert any("page slice" in issue and "text_exact_accuracy mismatch" in issue for issue in result["issues"])
assert any("page slice" in issue and "metric_means.line_detection_f1 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_tampered_page_claim_safe_blockers(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
metrics["metadata_contradictions"] = ["page-1: machine/synthetic provenance is declared as human_verified GT"]
metrics["leakage"]["overlaps"]["sample_id"] = ["page-1"]
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page claim_safe is true despite metadata contradictions" in result["issues"]
assert "page claim_safe is true despite train/dev/eval leakage" in result["issues"]
assert any("missing claim-safety warning for train/dev/eval leakage" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_claim_safe_with_undersized_page_slice(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
manifest_rows = [json.loads(line) for line in args.eval_manifest.read_text(encoding="utf-8").splitlines()]
manifest_rows[0]["metadata"]["language"] = "other-lang"
args.eval_manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in manifest_rows) + "\n", encoding="utf-8")
_refresh_page_pack_digest(args)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page claim_safe is true despite mixed languages" in result["issues"]
assert "page claim_safe is true despite slice sample size too small" in result["issues"]
assert any("missing claim-safety warning for slice sample size too small" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_tampered_page_unrun_leakage_check(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
args.train_manifest = []
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page claim_safe is true despite unrun leakage check" in result["issues"]
assert any("missing claim-safety warning for unrun leakage check" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_claim_safe_with_missing_page_model_identity(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["system_name"] = ""
eval_run["model_name"] = ""
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page claim_safe is true despite missing system identity" in result["issues"]
assert "page claim_safe is true despite missing model identity" in result["issues"]
assert any("missing claim-safety warning for missing system identity" in issue for issue in result["issues"])
assert any("missing claim-safety warning for missing model identity" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_tampered_page_missing_summary_blocker(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
args.summary_file = None
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page claim_safe is true despite missing eval pack summary/datasheet" in result["issues"]
assert any("missing claim-safety warning for missing eval pack summary/datasheet" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_claim_safe_with_normalizer_defaults(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
summary = json.loads(args.summary_file.read_text(encoding="utf-8"))
summary["counts"] = {"defaulted_domain": 1}
args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8")
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["claim_safe"] = True
eval_run["warnings"] = []
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer normalizer default count missing warning: defaulted_domain" in result["issues"]
assert "recognizer claim_safe is true despite normalizer-defaulted metadata" in result["issues"]
def test_audit_eval_run_recomputes_normalizer_counts_from_recognizer_summary(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
summary = json.loads(args.summary_file.read_text(encoding="utf-8"))
summary["counts"] = {"labels": 30, "defaulted_domain": 1}
args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8")
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
eval_run["normalizer_counts"] = {}
metrics["normalizer_counts"] = {}
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "metrics.json normalizer_counts do not match recognizer summary counts" in result["issues"]
assert "eval-run.json normalizer_counts do not match recognizer summary counts" in result["issues"]
def test_audit_eval_run_rejects_recognizer_summary_metadata_contradictions(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
summary = json.loads(args.summary_file.read_text(encoding="utf-8"))
summary.update(
{
"language": "wrong-lang",
"script": "WrongScript",
"domain": "synthetic",
"gt_status": "machine_converted",
"split": "train",
"eval_pack_version": "wrong-v1",
"frozen_eval_pack": False,
"unseen_for_selection": False,
"claim_eligible_default": False,
"eval_pack": {"task": "page_level_ocr"},
}
)
args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8")
eval_run_path = args.out / "eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["summary_file_sha256"] = sha256_file(args.summary_file)
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer summary language mismatch: summary='wrong-lang' run=['fixture-lang']" in result["issues"]
assert "recognizer summary script mismatch: summary='WrongScript' run=['FixtureScript']" in result["issues"]
assert "recognizer summary domain mismatch: summary='synthetic' run=['real_print']" in result["issues"]
assert "recognizer summary gt_status mismatch: summary='machine_converted' run=['human_verified']" in result["issues"]
assert "recognizer summary split mismatch: summary='train' run=['eval']" in result["issues"]
assert "recognizer summary eval_pack_version mismatch: summary='wrong-v1' run='fixture-v1'" in result["issues"]
assert "recognizer summary frozen_eval_pack mismatch: summary=False run=True" in result["issues"]
assert "recognizer summary unseen_for_selection mismatch: summary=False run=True" in result["issues"]
assert "recognizer summary claim_eligible_default mismatch: summary=False run=True" in result["issues"]
assert "recognizer summary task mismatch: summary='page_level_ocr' run='recognizer_oracle_crop'" in result["issues"]
def test_audit_eval_run_rejects_page_summary_metadata_contradictions(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
scorer.run_eval(args)
summary = json.loads(args.summary_file.read_text(encoding="utf-8"))
summary.update(
{
"language": "wrong-lang",
"script": "WrongScript",
"domain": "synthetic",
"gt_status": "machine_converted",
"split": "train",
"eval_pack_version": "wrong-v1",
"frozen_eval_pack": False,
"unseen_for_selection": False,
"claim_eligible_default": False,
"eval_pack": {"task": "recognizer_oracle_crop"},
}
)
args.summary_file.write_text(json.dumps(summary) + "\n", encoding="utf-8")
eval_run_path = args.out / "page-eval-run.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
eval_run["summary_file_sha256"] = sha256_file(args.summary_file)
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page summary language mismatch: summary='wrong-lang' run=['fixture-lang']" in result["issues"]
assert "page summary script mismatch: summary='WrongScript' run=['FixtureScript']" in result["issues"]
assert "page summary domain mismatch: summary='synthetic' run=['real_print']" in result["issues"]
assert "page summary gt_status mismatch: summary='machine_converted' run=['human_verified']" in result["issues"]
assert "page summary split mismatch: summary='train' run=['eval']" in result["issues"]
assert "page summary eval_pack_version mismatch: summary='wrong-v1' run='page-v1'" in result["issues"]
assert "page summary frozen_eval_pack mismatch: summary=False run=True" in result["issues"]
assert "page summary unseen_for_selection mismatch: summary=False run=True" in result["issues"]
assert "page summary claim_eligible_default mismatch: summary=False run=True" in result["issues"]
assert "page summary task mismatch: summary='recognizer_oracle_crop' run='page_level_ocr'" in result["issues"]
def test_audit_eval_run_recomputes_recognizer_dictionary_coverage(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
Path(args.dictionary).write_text("a\nb\n", encoding="utf-8")
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["dictionary_coverage"]["missing_gt_codepoints"] == ["U+0063"]
metrics["dictionary_coverage"]["missing_gt_codepoints"] = []
metrics["dictionary_coverage"]["missing_gt_codepoint_count"] = 0
eval_run["claim_safe"] = True
eval_run["warnings"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any(
"recognizer metrics dictionary_coverage missing_gt_codepoints mismatch" in issue
and "U+0063" in issue
for issue in result["issues"]
)
assert "recognizer claim_safe is true despite GT codepoints missing from dictionary" in result["issues"]
assert any("recognizer missing claim-safety warning for GT codepoints missing from dictionary" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_page_dictionary_coverage(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
Path(args.dictionary).write_text("a\nb\n", encoding="utf-8")
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["dictionary_coverage"]["missing_gt_codepoints"] == ["U+0063"]
metrics["dictionary_coverage"]["missing_gt_codepoints"] = []
metrics["dictionary_coverage"]["missing_gt_codepoint_count"] = 0
eval_run["claim_safe"] = True
eval_run["warnings"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert any(
"page metrics dictionary_coverage missing_gt_codepoints mismatch" in issue
and "U+0063" in issue
for issue in result["issues"]
)
assert "page claim_safe is true despite GT codepoints missing from dictionary" in result["issues"]
assert any("page missing claim-safety warning for GT codepoints missing from dictionary" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_zero_length_recognizer_gt_rows(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
label_lines = args.label_file.read_text(encoding="utf-8").splitlines()
label_lines[0] = "images/line-1.png\t"
args.label_file.write_text("\n".join(label_lines) + "\n", encoding="utf-8")
prediction_lines = args.prediction_file.read_text(encoding="utf-8").splitlines()
prediction_lines[0] = "line-1.png\t\t1.0"
args.prediction_file.write_text("\n".join(prediction_lines) + "\n", encoding="utf-8")
_refresh_recognizer_pack_digest(args)
scorer.run_eval(args)
eval_run_path = args.out / "eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["zero_length_gt_lines"] == 1
metrics["zero_length_gt_lines"] = 0
eval_run["claim_safe"] = True
eval_run["warnings"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "metrics zero_length_gt_lines does not match recomputed predictions.tsv value: recorded=0 recomputed=1" in result["issues"]
assert "recognizer claim_safe is true despite zero-length GT rows" in result["issues"]
assert any("recognizer missing claim-safety warning for zero-length GT rows" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_zero_length_page_text_references(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
reference = tmp_path / "page.ref.json"
payload = json.loads(reference.read_text(encoding="utf-8"))
payload["text"] = ""
reference.write_text(json.dumps(payload) + "\n", encoding="utf-8")
scorer.run_eval(args)
eval_run_path = args.out / "page-eval-run.json"
metrics_path = args.out / "metrics.json"
eval_run = json.loads(eval_run_path.read_text(encoding="utf-8"))
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["zero_length_text_references"] == 1
metrics["zero_length_text_references"] = 0
eval_run["claim_safe"] = True
eval_run["warnings"] = []
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
eval_run_path.write_text(json.dumps(eval_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "metrics zero_length_text_references does not match page-metrics.csv sum: recorded=0 recomputed=1" in result["issues"]
assert "page claim_safe is true despite zero-length text references" in result["issues"]
assert any("page missing claim-safety warning for zero-length text references" in issue for issue in result["issues"])
def test_audit_eval_run_recomputes_recognizer_normalization_summary(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.require_claim_safe = False
Path(args.dictionary).write_text("a\nb\nc\ná\n", encoding="utf-8")
label_lines = args.label_file.read_text(encoding="utf-8").splitlines()
label_lines[0] = "images/line-1.png\ta\u0301bc"
args.label_file.write_text("\n".join(label_lines) + "\n", encoding="utf-8")
prediction_lines = args.prediction_file.read_text(encoding="utf-8").splitlines()
prediction_lines[0] = "line-1.png\ta\u0301bc\t1.0"
args.prediction_file.write_text("\n".join(prediction_lines) + "\n", encoding="utf-8")
_refresh_recognizer_pack_digest(args)
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["normalization_summary"]["gt_changed"] == 1
assert metrics["normalization_summary"]["prediction_changed"] == 1
metrics["normalization_summary"]["gt_changed"] = 0
metrics["normalization_summary"]["prediction_changed"] = 0
metrics["normalization_summary"]["any_changed"] = 0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "recognizer metrics normalization_summary gt_changed mismatch: recorded=0 recomputed=1" in result["issues"]
assert "recognizer metrics normalization_summary prediction_changed mismatch: recorded=0 recomputed=1" in result["issues"]
assert "recognizer metrics normalization_summary any_changed mismatch: recorded=0 recomputed=1" in result["issues"]
def test_audit_eval_run_recomputes_page_normalization_summary(tmp_path: Path) -> None:
scorer = _load_tool("run_page_ocr_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_page_fixture(tmp_path)
args.require_claim_safe = False
Path(args.dictionary).write_text("a\nb\nc\ná\n", encoding="utf-8")
document_path = tmp_path / "documents" / "page-1" / "document.json"
payload = json.loads(document_path.read_text(encoding="utf-8"))
payload["pages"][0]["text_lines"][0]["text"] = "a\u0301bc"
payload["pages"][0]["blocks"][0]["text"] = "a\u0301bc"
document_path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
scorer.run_eval(args)
metrics_path = args.out / "metrics.json"
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
assert metrics["normalization_summary"]["prediction_changed"] == 1
metrics["normalization_summary"]["prediction_changed"] = 0
metrics["normalization_summary"]["any_changed"] = 0
metrics_path.write_text(json.dumps(metrics) + "\n", encoding="utf-8")
result = audit_tool.audit_run(args.out)
assert result["audit_passed"] is False
assert "page metrics normalization_summary prediction_changed mismatch: recorded=0 recomputed=1" in result["issues"]
assert "page metrics normalization_summary any_changed mismatch: recorded=0 recomputed=1" in result["issues"]
def test_audit_eval_run_accepts_complete_recipe_run_root(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["score_dir"].endswith("recipe-run/score")
assert result["audit_passed"] is True
def test_audit_eval_run_rejects_stale_recipe_materialization_file(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
materialization_path = tmp_path / "recipe-run" / "materialize" / "output-command-run.json"
materialization = json.loads(materialization_path.read_text(encoding="utf-8"))
materialization["records"][0]["output_sha256"] = "0" * 64
materialization_path.write_text(json.dumps(materialization) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "materialize/output-command-run.json does not match recipe-run.json materialization" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_validation_artifact(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
validation_path = tmp_path / "recipe-run" / "recipe-validation.json"
validation = json.loads(validation_path.read_text(encoding="utf-8"))
validation["claim_ready"] = False
validation_path.write_text(json.dumps(validation) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-validation.json does not match recipe-run.json recipe_validation" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_validation_against_validator(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
validation_path = tmp_path / "recipe-run" / "recipe-validation.json"
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
validation = json.loads(validation_path.read_text(encoding="utf-8"))
validation["claim_ready"] = False
validation["claim_blockers"] = ["stale blocker"]
validation_path.write_text(json.dumps(validation) + "\n", encoding="utf-8")
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["recipe_validation"] = validation
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json recipe_validation does not match current recipe validator result" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_artifact_against_registry(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_path = tmp_path / "recipe-run" / "recipe.json"
recipe = json.loads(recipe_path.read_text(encoding="utf-8"))
recipe["task"] = "stale-task"
recipe_path.write_text(json.dumps(recipe) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe.json does not match selected recipe in recipe registry file" in result["issues"]
def test_audit_eval_run_rejects_missing_recipe_run_provenance(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["machine"] = {"platform": ""}
recipe_run["command"] = ""
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe run machine.platform is missing" in result["issues"]
assert "recipe run machine.python is missing" in result["issues"]
assert "recipe run command is missing" in result["issues"]
def test_audit_eval_run_rejects_missing_recipe_run_root(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["root"] = str(tmp_path / "missing-root")
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert any("recipe-run.json root does not exist or is not a directory" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_recipe_registry_outside_declared_root(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
declared_root = tmp_path / "declared-root"
declared_root.mkdir()
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["root"] = str(declared_root)
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json recipe_file is not under declared root" in result["issues"]
def test_audit_eval_run_rejects_recipe_outputs_pointing_outside_root_artifacts(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
stale_dir = tmp_path / "stale"
stale_dir.mkdir()
stale_audit = stale_dir / "audit.json"
stale_claim = stale_dir / "claim-safety.md"
stale_audit.write_text((tmp_path / "recipe-run" / "audit.json").read_text(encoding="utf-8"), encoding="utf-8")
stale_claim.write_text((tmp_path / "recipe-run" / "claim-safety.md").read_text(encoding="utf-8"), encoding="utf-8")
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["outputs"]["materialization_dir"] = str(stale_dir)
recipe_run["outputs"]["audit_file"] = str(stale_audit)
recipe_run["outputs"]["claim_safety_report"] = str(stale_claim)
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json outputs.materialization_dir does not point at root materialize" in result["issues"]
assert "recipe-run.json outputs.audit_file does not point at root audit.json" in result["issues"]
assert "recipe-run.json outputs.claim_safety_report does not point at root claim-safety.md" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_run_audit_issues(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["audit"]["issues"] = ["stale issue"]
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json audit issues do not match current score audit" in result["issues"]
def test_audit_eval_run_rejects_stale_root_audit_summary(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
audit_path = tmp_path / "recipe-run" / "audit.json"
audit = json.loads(audit_path.read_text(encoding="utf-8"))
audit["issues"] = ["stale issue"]
audit["warnings"] = ["stale warning"]
audit["score_dir"] = str(tmp_path / "stale-score")
audit_path.write_text(json.dumps(audit) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "audit.json warnings do not match eval run" in result["issues"]
assert "audit.json issues do not match current score audit" in result["issues"]
assert "audit.json score_dir does not match audited score directory" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_score_artifact_hashes(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["score"]["metrics_file_sha256"] = "0" * 64
recipe_run["score"]["eval_run_file_sha256"] = "1" * 64
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert any("recipe-run.json score metrics_file_sha256 does not match score/metrics.json" in issue for issue in result["issues"])
assert any("recipe-run.json score eval_run_file_sha256 does not match score/eval-run.json" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_recipe_score_artifacts_outside_score_dir(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
stale_dir = tmp_path / "stale-score"
stale_dir.mkdir()
stale_metrics = stale_dir / "metrics.json"
stale_eval_run = stale_dir / "eval-run.json"
stale_metrics.write_text((args.out / "metrics.json").read_text(encoding="utf-8"), encoding="utf-8")
stale_eval_run.write_text((args.out / "eval-run.json").read_text(encoding="utf-8"), encoding="utf-8")
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["score"]["metrics_file"] = str(stale_metrics)
recipe_run["score"]["eval_run_file"] = str(stale_eval_run)
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json score metrics_file does not point at score/metrics.json" in result["issues"]
assert "recipe-run.json score eval_run_file does not point at score/eval-run.json" in result["issues"]
def test_audit_eval_run_rejects_recipe_train_manifest_identity_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
other_manifest = tmp_path / "other-train.jsonl"
other_manifest.write_text("", encoding="utf-8")
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["train_manifests"][0]["path"] = str(other_manifest)
recipe_run["train_manifests"][0]["sha256"] = "0" * 64
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json train_manifests[0] path does not match eval-run.json" in result["issues"]
assert "recipe-run.json train_manifests[0] sha256 does not match eval-run.json" in result["issues"]
def test_audit_eval_run_rejects_recipe_train_manifest_count_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
scorer.run_eval(args)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True, exist_ok=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["train_manifests"] = []
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json train_manifests count does not match eval-run.json: recorded=0 score=1" in result["issues"]
def test_audit_eval_run_rejects_missing_recipe_materialization_provenance(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["materialization"].pop("created_at")
recipe_run["materialization"]["machine"] = {"platform": ""}
recipe_run["materialization"].pop("invocation")
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "materialization run created_at timestamp is missing" in result["issues"]
assert "materialization machine.platform is missing" in result["issues"]
assert "materialization machine.python is missing" in result["issues"]
assert "materialization missing required field: invocation" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_materialization_record_count(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["materialization"]["record_count"] = 2
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "materialization record_count does not match records length: recorded=2 actual=1" in result["issues"]
def test_audit_eval_run_rejects_recipe_materialization_score_identity_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["materialization"]["model_name"] = "stale-model"
recipe_run["materialization"]["checkpoint"] = "stale-checkpoint"
recipe_run["materialization"]["dictionary"] = "stale-dictionary"
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert any("recipe-run.json materialization model_name does not match score eval-run model_name" in issue for issue in result["issues"])
assert any("recipe-run.json materialization checkpoint does not match score eval-run checkpoint" in issue for issue in result["issues"])
assert any("recipe-run.json materialization dictionary does not match score eval-run dictionary" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_recipe_command_template_file(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
(tmp_path / "recipe-run" / "materialize" / "command-template.txt").write_text("stale command\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "materialize/command-template.txt does not match recorded materialization command_template" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_materialization_record_command(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
materialization_path = tmp_path / "recipe-run" / "materialize" / "output-command-run.json"
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
materialization = json.loads(materialization_path.read_text(encoding="utf-8"))
materialization["records"][0]["command"] = ["stale-command"]
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["materialization"] = materialization
materialization_path.write_text(json.dumps(materialization) + "\n", encoding="utf-8")
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "materialization records[0] command does not match materialization command_template" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_materialization_log_hash(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
(tmp_path / "recipe-run" / "materialize" / "stdout.txt").write_text("stale stdout\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert any("materialization records[0] stdout_sha256 mismatch" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_stale_recipe_materialization_report(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
(tmp_path / "recipe-run" / "materialize" / "output-command-run.md").write_text("# stale\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "materialize/output-command-run.md missing or stale line: - records: 1" in result["issues"]
assert any("materialize/output-command-run.md missing or stale output line" in issue for issue in result["issues"])
def test_audit_eval_run_rejects_recipe_prediction_path_mismatch(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
other_prediction = tmp_path / "recipe-run" / "other-predictions.txt"
other_prediction.write_text(root_prediction.read_text(encoding="utf-8"), encoding="utf-8")
recipe_run_path = tmp_path / "recipe-run" / "recipe-run.json"
recipe_run = json.loads(recipe_run_path.read_text(encoding="utf-8"))
recipe_run["outputs"]["prediction_file"] = str(other_prediction)
recipe_run_path.write_text(json.dumps(recipe_run) + "\n", encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "recipe-run.json outputs.prediction_file does not match score eval-run prediction_file" in result["issues"]
assert "recipe-run.json materialization records do not include outputs.prediction_file" in result["issues"]
def test_audit_eval_run_rejects_stale_recipe_root_claim_report(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
root_prediction = tmp_path / "recipe-run" / "predictions.txt"
root_prediction.parent.mkdir(parents=True)
root_prediction.write_text(args.prediction_file.read_text(encoding="utf-8"), encoding="utf-8")
args.prediction_file = root_prediction
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
_write_recipe_root_artifacts(tmp_path, audit_tool, args.out, args.prediction_file)
report_path = tmp_path / "recipe-run" / "claim-safety.md"
report = report_path.read_text(encoding="utf-8")
assert "- claim safe: True" in report
report_path.write_text(report.replace("- claim safe: True", "- claim safe: False"), encoding="utf-8")
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "root claim-safety.md missing or stale line: - claim safe: True" in result["issues"]
def test_audit_eval_run_rejects_incomplete_recipe_run_root(tmp_path: Path) -> None:
scorer = _load_tool("run_recognizer_eval")
audit_tool = _load_tool("audit_eval_run")
args = _write_recognizer_fixture(tmp_path)
args.out = tmp_path / "recipe-run" / "score"
scorer.run_eval(args)
result = audit_tool.audit_run(tmp_path / "recipe-run")
assert result["audit_passed"] is False
assert "missing required artifact: recipe-run.json" in result["issues"]
assert "missing required artifact: claim-safety.md" in result["issues"]