Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_claim_review.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 83.6 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_claim_review.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_claim_review.py
-
curl -L -o test_claim_review.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_claim_review.py
83.6 kB
from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech.errors import ClaimReviewError
from ocrtech.claim_review import review_claim
from ocrtech.evalpack import create_eval_pack
from ocrtech.experiments import run_experiment
from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest
from ocrtech.models import write_model_card
def test_review_claim_passes_for_internal_and_public_bundle(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model)
public = _build_public_experiment(tmp_path / "public", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"required_slices": ["nepali", "english", "table"],
},
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
},
]
}
),
encoding="utf-8",
)
report = review_claim([internal, public], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
assert len(report.experiments) == 2
assert (tmp_path / "review" / "claim-review.json").exists()
def test_review_claim_fails_when_required_metric_is_missing(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer", "figure_detection_f1", "not_a_real_metric"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "missing metrics: candidate:not_a_real_metric" in required_check.summary
def test_review_claim_derives_error_counts_from_benchmark_rows(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
rows[0]["status"] = "error"
rows[0]["error"] = "simulated stale status summary"
benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
payload["benchmark_status_counts"] = {"ok": len(rows)}
stale = tmp_path / "stale-status-counts-experiment.json"
stale.write_text(json.dumps(payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([stale], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "benchmark has 1 error rows" in required_check.summary
def test_review_claim_fails_when_eval_manifest_hash_is_stale(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
payload["environment"]["eval_manifest_sha256"] = "0" * 64
stale = tmp_path / "stale-eval-hash-experiment.json"
stale.write_text(json.dumps(payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([stale], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "experiment eval_manifest sha256 does not match environment.eval_manifest_sha256" in required_check.summary
def test_review_claim_fails_when_benchmark_sample_is_not_in_eval_manifest(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
rows[0]["sample_id"] = "not-in-eval-manifest"
benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "benchmark row sample_id not in eval_manifest: not-in-eval-manifest" in required_check.summary
def test_review_claim_fails_when_benchmark_slices_do_not_match_eval_manifest(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
rows[0]["slices"] = ["table"]
benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "benchmark row slices do not match eval_manifest for sample_id: ocrbench-s1" in required_check.summary
def test_review_claim_fails_when_required_baseline_skips_eval_sample(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
benchmark_report.write_text(json.dumps([], indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert rows
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "missing baselines: candidate" in required_check.summary
assert "missing baseline sample coverage: candidate:ocrbench-s1" in required_check.summary
def test_review_claim_accepts_required_metric_slice_coverage(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["table_cell_f1"],
"required_metric_slices": {"table_cell_f1": ["table"]},
"required_slices": ["table"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "pass"
def test_review_claim_fails_when_required_metric_is_missing_on_slice(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
for row in rows:
if row.get("baseline") != "candidate" or row.get("status") != "ok":
continue
metrics = row.get("metrics")
if not isinstance(metrics, dict):
continue
if "table" in set(row.get("slices") or []):
metrics.pop("table_cell_f1", None)
else:
metrics["table_cell_f1"] = 1.0
benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["table_cell_f1"],
"required_metric_slices": {"table_cell_f1": ["table"]},
"required_slices": ["table"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "missing metric slice coverage: candidate:table_cell_f1@table" in required_check.summary
def test_review_claim_accepts_required_metric_paired_coverage(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
_clone_candidate_rows_to_baseline(public, "tesseract")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate", "tesseract"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
def test_review_claim_fails_when_required_metric_has_no_candidate_baseline_pairs(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
baseline_row = dict(next(row for row in rows if row.get("baseline") == "candidate"))
baseline_row["baseline"] = "tesseract"
baseline_row["sample_id"] = "different-sample"
baseline_row["input_path"] = str(tmp_path / "different-input.png")
rows.append(baseline_row)
benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate", "tesseract"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "missing paired metric coverage: candidate:tesseract:cer@all" in required_check.summary
def test_review_claim_accepts_candidate_fallback_policy(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
_set_candidate_metric(public, "fallback_triggered", 0.0)
_set_candidate_metric(public, "fallback_failed", 0.0)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"require_candidate_fallback_metrics": True,
"max_candidate_fallback_trigger_rate": 0.5,
"max_candidate_fallback_failed_rate": 0.0,
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "pass"
def test_review_claim_fails_when_required_output_artifact_is_missing(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
candidate_row = next(row for row in rows if row.get("baseline") == "candidate")
document_md = benchmark_report.parent / "candidate" / Path(candidate_row["input_path"]).stem / "document.md"
document_md.unlink()
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"required_output_artifacts": ["document_json", "document_md"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "missing output artifacts: candidate:ocrbench-s1:document_md" in required_check.summary
def test_review_claim_fails_when_required_output_artifact_is_invalid_json(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
candidate_row = next(row for row in rows if row.get("baseline") == "candidate")
document_json = benchmark_report.parent / "candidate" / Path(candidate_row["input_path"]).stem / "document.json"
document_json.write_text("{not-json", encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"required_output_artifacts": ["document_json"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "candidate:ocrbench-s1:document_json (invalid json:" in required_check.summary
def test_review_claim_fails_when_candidate_fallback_metrics_are_missing(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"require_candidate_fallback_metrics": True,
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "missing candidate fallback_triggered metrics" in required_check.summary
assert "missing candidate fallback_failed metrics" in required_check.summary
def test_review_claim_fails_when_candidate_fallback_trigger_rate_is_too_high(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
_set_candidate_metric(public, "fallback_triggered", 1.0)
_set_candidate_metric(public, "fallback_failed", 0.0)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"require_candidate_fallback_metrics": True,
"max_candidate_fallback_trigger_rate": 0.5,
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2")
assert required_check.status == "fail"
assert "candidate fallback trigger rate 1.000000 exceeds max 0.500000" in required_check.summary
def test_review_claim_audits_recorded_fallback_model_config(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
fallback_check = next(check for check in report.checks if check.name == "fallback-model")
assert fallback_check.status == "pass"
assert any("audited model_id=ocrtech-fallback" in detail for detail in fallback_check.details)
def test_review_claim_can_require_fallback_model_config_presence(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_fallback_model_present": True,
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
fallback_check = next(check for check in report.checks if check.name == "fallback-model")
assert fallback_check.status == "fail"
assert "missing required fallback_model_config provenance" in fallback_check.summary
assert fallback_check.details == ["experiment"]
def test_review_claim_requires_candidate_fallback_model_card_provenance(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_fallback_model_present": True,
"require_candidate_fallback_provenance": True,
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance")
assert provenance_check.status == "fail"
assert "missing provenance.fallback_model_card" in provenance_check.summary
def test_review_claim_accepts_candidate_fallback_model_card_provenance(tmp_path: Path) -> None:
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
model = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-sidecar",
backend="sidecar",
base_model="fixture",
fallback_model_card=fallback,
)
public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_fallback_model_present": True,
"require_candidate_fallback_provenance": True,
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance")
assert provenance_check.status == "pass"
assert "matches reviewed experiment fallback configs" in provenance_check.summary
def test_review_claim_rejects_candidate_fallback_model_card_mismatch(tmp_path: Path) -> None:
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
other_fallback = write_model_card(tmp_path / "other-fallback.json", model_id="ocrtech-other-fallback", backend="sidecar", base_model="fixture")
model = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-sidecar",
backend="sidecar",
base_model="fixture",
fallback_model_card=other_fallback,
)
public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_fallback_model_present": True,
"require_candidate_fallback_provenance": True,
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance")
assert provenance_check.status == "fail"
assert "does not match experiment fallback_model_config" in provenance_check.summary
def test_review_claim_fails_when_fallback_model_hash_is_missing(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback)
payload = json.loads(public.read_text(encoding="utf-8"))
payload["environment"]["fallback_model_config_sha256"] = None
public.write_text(json.dumps(payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
fallback_check = next(check for check in report.checks if check.name == "fallback-model")
assert fallback_check.status == "fail"
assert "fallback model config path is recorded but sha256 is missing" in fallback_check.summary
def test_review_claim_fails_when_fallback_model_card_audit_fails(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-bad-fallback", backend="paddleocr", base_model="fixture")
public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([public], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
fallback_check = next(check for check in report.checks if check.name == "fallback-model")
assert fallback_check.status == "fail"
assert "fallback model card failed audit" in fallback_check.summary
assert any("paddleocr candidate requires at least one recorded artifact" in detail for detail in fallback_check.details)
def test_review_claim_verifies_validation_report_status_matches_experiment(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
validation_report = Path(payload["validation_report"])
validation_payload = json.loads(validation_report.read_text(encoding="utf-8"))
validation_payload["claim_status"] = "not_validated"
validation_payload["passed"] = False
validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "validation_report claim_status not_validated != required validated" in required_check.summary
def test_review_claim_verifies_validation_report_benchmark_provenance(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
validation_report = Path(payload["validation_report"])
validation_payload = json.loads(validation_report.read_text(encoding="utf-8"))
validation_payload["provenance"]["benchmark_report"]["sha256"] = "0" * 64
validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "validation_report benchmark_report sha256 does not match experiment benchmark_report" in required_check.summary
def test_review_claim_verifies_validation_report_config_provenance(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
validation_report = Path(payload["validation_report"])
validation_payload = json.loads(validation_report.read_text(encoding="utf-8"))
validation_payload["provenance"]["validation_config"]["sha256"] = "0" * 64
validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "validation_report validation_config sha256 does not match experiment validation_config" in required_check.summary
def test_review_claim_fails_when_experiment_used_wrong_validation_config(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
required_config = tmp_path / "required-validation.json"
required_config.write_text(
json.dumps(
{
"candidate": "candidate",
"min_samples": 99,
"required_slices": {"nepali": 99},
"gates": [],
}
),
encoding="utf-8",
)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"required_validation_config": str(required_config),
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "experiment validation_config sha256 does not match required config" in required_check.summary
def test_review_claim_verifies_validation_report_train_manifest_provenance(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
validation_report = Path(payload["validation_report"])
validation_payload = json.loads(validation_report.read_text(encoding="utf-8"))
validation_payload["provenance"]["train_manifests"] = [{"path": "data/train.jsonl", "sha256": "0" * 64}]
validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "validation_report train_manifests provenance does not match experiment train_manifests" in required_check.summary
def test_review_claim_verifies_validation_report_candidate_model_provenance(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
validation_report = Path(payload["validation_report"])
validation_payload = json.loads(validation_report.read_text(encoding="utf-8"))
validation_payload["provenance"]["candidate_model_config"]["sha256"] = "0" * 64
validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "validation_report candidate_model_config sha256 does not match experiment candidate_model_config" in required_check.summary
def test_review_claim_requires_validation_report_fallback_model_provenance_when_experiment_has_fallback(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
internal = _rewrite_fallback_model_path(_build_internal_experiment(tmp_path / "internal", model), fallback)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "validation_report missing provenance.fallback_model_config" in required_check.summary
def test_review_claim_accepts_required_passing_validation_gate(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer", "quality_score"],
"required_validation_gates": [
{"baseline": "candidate", "metric": "quality_score"}
],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "pass"
def test_review_claim_fails_when_required_passing_validation_gate_is_missing(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate", "tesseract"],
"required_metrics": ["cer"],
"required_validation_gates": [
{"baseline": "tesseract", "metric": "cer", "slices": ["nepali"]}
],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "missing passing validation gate: tesseract:cer@nepali" in required_check.summary
def test_review_claim_rejects_passing_validation_gate_without_quantitative_evidence(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
validation_report = Path(payload["validation_report"])
validation_payload = json.loads(validation_report.read_text(encoding="utf-8"))
validation_payload["gates"] = [
{
"passed": True,
"gate": {"baseline": "candidate", "metric": "quality_score"},
}
]
validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["quality_score"],
"required_validation_gates": [
{"baseline": "candidate", "metric": "quality_score"}
],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "missing passing validation gate: candidate:quality_score" in required_check.summary
def test_review_claim_rejects_passing_validation_gate_below_own_min_pairs(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
validation_report = Path(payload["validation_report"])
validation_payload = json.loads(validation_report.read_text(encoding="utf-8"))
validation_payload["gates"] = [
{
"passed": True,
"pairs": 1,
"candidate_mean": 1.0,
"baseline_mean": 0.0,
"absolute_improvement": 1.0,
"relative_improvement": 1.0,
"improvement_ci_low": 0.1,
"win_count": 1,
"loss_count": 0,
"tie_count": 0,
"pair_win_rate": 1.0,
"sign_test_p_value": 0.0,
"gate": {
"baseline": "candidate",
"metric": "quality_score",
"min_pairs": 2,
"require_ci_above_zero": True,
},
}
]
validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["quality_score"],
"required_validation_gates": [
{"baseline": "candidate", "metric": "quality_score"}
],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "missing passing validation gate: candidate:quality_score" in required_check.summary
def test_review_claim_config_rejects_validation_gate_without_validation_status(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_baselines": ["candidate"],
"required_metrics": ["quality_score"],
"required_validation_gates": [{"baseline": "candidate", "metric": "quality_score"}],
}
]
}
),
encoding="utf-8",
)
with pytest.raises(ClaimReviewError, match="required_validation_gates without require_validation_status"):
review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
def test_review_claim_config_rejects_validation_gate_outside_required_baselines(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"required_validation_gates": [{"baseline": "tesseract", "metric": "cer"}],
}
]
}
),
encoding="utf-8",
)
with pytest.raises(ClaimReviewError, match="validation gate baselines not listed"):
review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
def test_review_claim_config_rejects_validation_gate_outside_required_metrics(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"required_validation_gates": [{"baseline": "candidate", "metric": "quality_score"}],
}
]
}
),
encoding="utf-8",
)
with pytest.raises(ClaimReviewError, match="validation gate metrics not listed"):
review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
def test_review_claim_requires_trained_recognizer_when_configured(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
seed_model = write_model_card(tmp_path / "seed-trained-pass.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_trained_recognizer": True,
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
trained_check = next(check for check in report.checks if check.name == "trained-recognizer")
assert trained_check.status == "fail"
assert "lacks trained local recognizer provenance" in trained_check.summary
def test_review_claim_accepts_trained_recognizer_provenance(tmp_path: Path) -> None:
train_manifest = _write_fixture_manifest(tmp_path / "train.jsonl", sample_id="train-s1")
eval_manifest = _write_fixture_manifest(tmp_path / "eval.jsonl", sample_id="eval-s1")
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"trained weights")
training_summary = tmp_path / "recognizer-training-summary.json"
training_summary.write_text('{"backend": "paddleocr"}\n', encoding="utf-8")
model = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-trained",
backend="paddleocr",
base_model="fixture-pretrained",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
)
seed_model = write_model_card(tmp_path / "seed-trained-pass.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_trained_recognizer": True,
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
trained_check = next(check for check in report.checks if check.name == "trained-recognizer")
assert trained_check.status == "pass"
def test_review_claim_accepts_nested_trained_recognizer_provenance(tmp_path: Path) -> None:
train_manifest = _write_fixture_manifest(tmp_path / "train.jsonl", sample_id="train-s1")
eval_manifest = _write_fixture_manifest(tmp_path / "eval.jsonl", sample_id="eval-s1")
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"trained weights")
training_summary = tmp_path / "recognizer-training-summary.json"
training_summary.write_text('{"backend": "hf_vision_encoder_decoder"}\n', encoding="utf-8")
trained_model = write_model_card(
tmp_path / "trained-model.json",
model_id="ocrtech-trained",
backend="hf_vision_encoder_decoder",
base_model="fixture-pretrained",
artifact_paths=[artifact],
backend_kwargs={"model_dir": str(tmp_path)},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
)
composite_model = write_model_card(
tmp_path / "composite.json",
model_id="ocrtech-composite-with-trained-component",
backend="table_cell_refine_composite",
base_model="trained+tesseract",
backend_kwargs={"structure_engine": "surya", "crop_engine": "tesseract"},
provenance={
"trained_recognizer_model_card": {
"path": str(trained_model),
"sha256": sha256_file(trained_model),
}
},
)
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), composite_model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_trained_recognizer": True,
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=composite_model)
assert report.passed
trained_check = next(check for check in report.checks if check.name == "trained-recognizer")
assert trained_check.status == "pass"
def test_review_claim_requires_model_admission_when_configured(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_model_admission": True,
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
admission_check = next(check for check in report.checks if check.name == "model-admission")
assert admission_check.status == "fail"
assert "not passed admission validation" in admission_check.summary
def test_review_claim_accepts_model_admission_when_configured(tmp_path: Path) -> None:
validation = tmp_path / "validation.json"
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
model_path = tmp_path / "model.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(model_path)}},
"gates": [_passing_admission_validation_gate()],
}
)
+ "\n",
encoding="utf-8",
)
model = write_model_card(
model_path,
model_id="ocrtech-admitted",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
admission_validation_report=validation,
)
seed_model = write_model_card(tmp_path / "seed-admission-pass.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"require_model_admission": True,
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
],
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
admission_check = next(check for check in report.checks if check.name == "model-admission")
assert admission_check.status == "pass"
def test_review_claim_eval_manifest_patterns_use_wildcards(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
manifest = Path(payload["eval_manifest"])
real5_manifest = manifest.with_name("PaddlePaddle-Real5-OmniDocBench-eval.jsonl")
real5_manifest.write_text(manifest.read_text(encoding="utf-8"), encoding="utf-8")
payload["eval_manifest"] = str(real5_manifest)
rewritten = tmp_path / "public-real5-experiment.json"
rewritten.write_text(json.dumps(payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "real5-slot",
"eval_manifest_patterns": ["*Real5-OmniDocBench*"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([rewritten], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
required_check = next(check for check in report.checks if check.name == "required:real5-slot")
assert required_check.status == "pass"
def test_review_claim_rejects_stale_benchmark_report_path(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
payload["benchmark_report"] = str(tmp_path / "missing" / "report.json")
stale = tmp_path / "stale-experiment.json"
stale.write_text(json.dumps(payload, indent=2), encoding="utf-8")
with pytest.raises(ClaimReviewError, match="benchmark_report .* does not exist"):
review_claim([stale], tmp_path / "review", model_config=model)
def test_review_claim_fails_on_inconsistent_candidate_model(tmp_path: Path) -> None:
first_model = write_model_card(tmp_path / "model-a.json", model_id="ocrtech-sidecar-a", backend="sidecar", base_model="fixture")
second_model = write_model_card(tmp_path / "model-b.json", model_id="ocrtech-sidecar-b", backend="sidecar", base_model="fixture", notes="different")
first = _build_internal_experiment(tmp_path / "internal-a", first_model)
second = _build_public_experiment(tmp_path / "public-b", second_model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"]},
{"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]},
]
}
),
encoding="utf-8",
)
report = review_claim([first, second], tmp_path / "review", config_path=config)
assert not report.passed
model_check = next(check for check in report.checks if check.name == "candidate-model")
assert model_check.status == "fail"
assert "do not share one model config" in model_check.summary
def test_review_claim_fails_when_candidate_model_hash_is_stale_without_explicit_model(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", model)
payload = json.loads(internal.read_text(encoding="utf-8"))
payload["environment"]["candidate_model_config_sha256"] = "0" * 64
stale = tmp_path / "stale-candidate-hash-experiment.json"
stale.write_text(json.dumps(payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([stale], tmp_path / "review", config_path=config)
assert not report.passed
model_check = next(check for check in report.checks if check.name == "candidate-model")
assert model_check.status == "fail"
assert "candidate model config sha256 does not match experiment evidence" in model_check.summary
def test_review_claim_fails_when_quality_router_calibration_matches_reviewed_eval(tmp_path: Path) -> None:
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", seed_model)
internal_payload = json.loads(internal.read_text(encoding="utf-8"))
internal_eval_manifest = Path(internal_payload["eval_manifest"])
routed_model = write_model_card(
tmp_path / "quality-router.json",
model_id="ocrtech-quality-router",
backend="quality_select_composite",
base_model="tesseract-ocr",
backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "tesseract", "secondary_quality_margin": 0.02},
provenance={
"calibration": {
"eval_manifest": {
"path": str(internal_eval_manifest),
"sha256": sha256_file(internal_eval_manifest),
},
"metric": "cer",
"selected_threshold": 0.02,
}
},
)
public = _build_public_experiment(tmp_path / "public", seed_model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"]},
{"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]},
]
}
),
encoding="utf-8",
)
internal_for_review = _rewrite_candidate_model_path(internal, routed_model)
public_for_review = _rewrite_candidate_model_path(public, routed_model)
report = review_claim([internal_for_review, public_for_review], tmp_path / "review", config_path=config, model_config=routed_model)
assert not report.passed
calibration_check = next(check for check in report.checks if check.name == "candidate-calibration")
assert calibration_check.status == "fail"
assert "matches a reviewed experiment eval manifest" in calibration_check.summary
def test_review_claim_fails_when_script_router_collapses_to_secondary_engine(tmp_path: Path) -> None:
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", seed_model)
public = _build_public_experiment(tmp_path / "public", seed_model)
calibration = tmp_path / "calibration.json"
calibration.write_text(
json.dumps(
{
"eval_manifest": "different.jsonl",
"metric": "cer",
"selected_threshold": 0.0,
"selected_trial": {
"threshold": 0.0,
"pairs": 5,
"secondary_picks": 5,
"chosen_mean": 0.1,
"primary_mean": 0.5,
"absolute_improvement": 0.4,
"relative_improvement": 0.8,
},
}
),
encoding="utf-8",
)
routed_model = write_model_card(
tmp_path / "script-router.json",
model_id="ocrtech-script-router",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.0},
metrics_report=calibration,
provenance={
"calibration": {
"eval_manifest": {
"path": "different-eval.jsonl",
"sha256": "1234567890abcdef",
},
"metric": "cer",
"selected_threshold": 0.0,
}
},
)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"]},
{"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]},
]
}
),
encoding="utf-8",
)
internal_for_review = _rewrite_candidate_model_path(internal, routed_model)
public_for_review = _rewrite_candidate_model_path(public, routed_model)
report = review_claim([internal_for_review, public_for_review], tmp_path / "review", config_path=config, model_config=routed_model)
assert not report.passed
calibration_check = next(check for check in report.checks if check.name == "candidate-calibration")
assert calibration_check.status == "fail"
assert "collapses to the secondary engine" in calibration_check.summary
def test_review_claim_fails_when_routed_candidate_is_identical_to_component_baseline_on_holdout(tmp_path: Path) -> None:
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _build_internal_experiment(tmp_path / "internal", seed_model)
_clone_candidate_rows_to_baseline(internal, "surya")
public = _build_public_experiment(tmp_path / "public", seed_model)
calibration = tmp_path / "calibration.json"
calibration.write_text(
json.dumps(
{
"eval_manifest": "separate-dev.jsonl",
"metric": "cer",
"selected_threshold": 0.4,
"selected_trial": {
"threshold": 0.4,
"pairs": 5,
"secondary_picks": 3,
"chosen_mean": 0.1,
"primary_mean": 0.2,
"absolute_improvement": 0.1,
"relative_improvement": 0.5,
},
}
),
encoding="utf-8",
)
routed_model = write_model_card(
tmp_path / "script-router.json",
model_id="ocrtech-script-router",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.4},
metrics_report=calibration,
provenance={
"calibration": {
"eval_manifest": {
"path": "separate-dev.jsonl",
"sha256": "abcdef1234567890",
},
"metric": "cer",
"selected_threshold": 0.4,
}
},
)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate", "surya"]},
{"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]},
]
}
),
encoding="utf-8",
)
internal_for_review = _rewrite_candidate_model_path(internal, routed_model)
public_for_review = _rewrite_candidate_model_path(public, routed_model)
report = review_claim([internal_for_review, public_for_review], tmp_path / "review", config_path=config, model_config=routed_model)
assert not report.passed
distinctness_check = next(check for check in report.checks if check.name == "candidate-distinctness")
assert distinctness_check.status == "fail"
assert "indistinguishable from component baseline surya" in distinctness_check.summary
def _write_fixture_manifest(path: Path, *, sample_id: str) -> Path:
source = path.with_suffix(".txt")
source.write_text("नेपाल सरकार\n", encoding="utf-8")
write_manifest(
[
ManifestEntry(
sample_id=sample_id,
dataset="fixture",
split="train" if "train" in path.stem else "eval",
image_path=str(source),
text="नेपाल सरकार",
sha256=sha256_file(source),
)
],
path,
)
return path
def _build_internal_experiment(root: Path, model: Path) -> Path:
pack = create_eval_pack(root / "pack", input_format="text", count_per_template=1, degradations=["clean"])
config = root / "validation.json"
config.parent.mkdir(parents=True, exist_ok=True)
config.write_text(
json.dumps(
{
"candidate": "candidate",
"min_samples": 5,
"required_slices": {"nepali": 1, "english": 1, "multi_page": 1},
"gates": [
{
"baseline": "candidate",
"metric": "quality_score",
"direction": "higher",
"min_absolute_margin": 0.0,
"require_ci_above_zero": False,
}
],
}
),
encoding="utf-8",
)
report = run_experiment(
pack.manifest_path,
root / "experiment",
baselines=["candidate"],
references_dir=root / "pack" / "references",
validation_config=config,
candidate_model_config=model,
)
return Path(report.output_dir) / "experiment.json"
def _build_public_experiment(root: Path, model: Path) -> Path:
inputs = root / "pack" / "inputs"
refs = root / "pack" / "references"
manifests = root / "pack" / "manifests"
inputs.mkdir(parents=True)
refs.mkdir()
manifests.mkdir()
source = inputs / "ocrbench-sample.txt"
source.write_text("Invoice Number 42\nनेपाल सरकार\n", encoding="utf-8")
refs.joinpath("ocrbench-sample.ref.txt").write_text("Invoice Number 42\nनेपाल सरकार\n", encoding="utf-8")
manifest = manifests / "ocrbench-v2-eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="ocrbench-s1",
dataset="ocrbench-v2",
split="eval",
image_path=str(source),
text="Invoice Number 42\nनेपाल सरकार",
metadata={"slices": ["english", "nepali", "mixed_script"], "reference_path": str(refs / "ocrbench-sample.ref.txt")},
)
],
manifest,
)
report = run_experiment(
manifest,
root / "experiment",
baselines=["candidate"],
references_dir=refs,
candidate_model_config=model,
run_validation=False,
)
return Path(report.output_dir) / "experiment.json"
def _rewrite_candidate_model_path(experiment_path: Path, model: Path) -> Path:
payload = json.loads(experiment_path.read_text(encoding="utf-8"))
payload.setdefault("environment", {})
payload["environment"]["candidate_model_config"] = str(model)
payload["environment"]["candidate_model_config_sha256"] = sha256_file(model)
validation_report = payload.get("validation_report")
if validation_report:
validation_path = Path(validation_report)
if validation_path.exists():
validation_payload = json.loads(validation_path.read_text(encoding="utf-8"))
validation_payload.setdefault("provenance", {})["candidate_model_config"] = {
"path": str(model),
"sha256": sha256_file(model),
}
validation_path.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8")
rewritten = experiment_path.parent / "experiment-rewritten.json"
rewritten.write_text(json.dumps(payload, indent=2), encoding="utf-8")
return rewritten
def _rewrite_fallback_model_path(experiment_path: Path, model: Path) -> Path:
payload = json.loads(experiment_path.read_text(encoding="utf-8"))
payload.setdefault("environment", {})
payload["environment"]["fallback_model_config"] = str(model)
payload["environment"]["fallback_model_config_sha256"] = sha256_file(model)
rewritten = experiment_path.parent / "experiment-fallback-rewritten.json"
rewritten.write_text(json.dumps(payload, indent=2), encoding="utf-8")
return rewritten
def _clone_candidate_rows_to_baseline(experiment_path: Path, baseline_name: str) -> None:
payload = json.loads(experiment_path.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
cloned_rows = []
for row in rows:
if row.get("baseline") != "candidate":
continue
clone = dict(row)
clone["baseline"] = baseline_name
cloned_rows.append(clone)
benchmark_report.write_text(json.dumps([*rows, *cloned_rows], indent=2), encoding="utf-8")
def _set_candidate_metric(experiment_path: Path, metric: str, value: float) -> None:
payload = json.loads(experiment_path.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
for row in rows:
if row.get("baseline") != "candidate" or row.get("status") != "ok":
continue
row.setdefault("metrics", {})[metric] = value
benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
def _passing_admission_validation_gate() -> dict[str, object]:
return {
"passed": True,
"pairs": 2,
"candidate_mean": 0.1,
"baseline_mean": 0.2,
"absolute_improvement": 0.1,
"relative_improvement": 0.5,
"improvement_ci_low": 0.01,
"win_count": 2,
"loss_count": 0,
"tie_count": 0,
"pair_win_rate": 1.0,
"sign_test_p_value": 0.0,
"gate": {
"baseline": "tesseract",
"metric": "cer",
"min_pairs": 2,
"min_pair_win_rate": 0.5,
"require_ci_above_zero": True,
},
}
from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech.errors import ClaimReviewError
from ocrtech.claim_review import review_claim
from ocrtech.evalpack import create_eval_pack
from ocrtech.experiments import run_experiment
from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest
from ocrtech.models import write_model_card
def test_review_claim_passes_for_internal_and_public_bundle(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model)
public = _build_public_experiment(tmp_path / "public", model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
"required_slices": ["nepali", "english", "table"],
},
{
"name": "ocrbench-v2",
"datasets": ["ocrbench-v2"],
"require_baselines": ["candidate"],
"required_metrics": ["cer"],
},
]
}
),
encoding="utf-8",
)
report = review_claim([internal, public], tmp_path / "review", config_path=config, model_config=model)
assert report.passed
assert len(report.experiments) == 2
assert (tmp_path / "review" / "claim-review.json").exists()
def test_review_claim_fails_when_required_metric_is_missing(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model)
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_metrics": ["cer", "figure_detection_f1", "not_a_real_metric"],
}
]
}
),
encoding="utf-8",
)
report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model)
assert not report.passed
required_check = next(check for check in report.checks if check.name == "required:internal-pack")
assert required_check.status == "fail"
assert "missing metrics: candidate:not_a_real_metric" in required_check.summary
def test_review_claim_derives_error_counts_from_benchmark_rows(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
public = _build_public_experiment(tmp_path / "public", model)
payload = json.loads(public.read_text(encoding="utf-8"))
benchmark_report = Path(payload["benchmark_report"])
rows = json.loads(benchmark_report.read_text(encoding="utf-8"))
rows[0]["status"] = "error"
rows[0]["error"] = "simulated stale status summary"
benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8")
payload["benchmark_status_counts"] = {"ok": len(rows)}
stale = tmp_path / "stale-status-counts-experiment.json"
stale.write_text(json.dumps(payload, indent=2), encoding="utf-8")
config = tmp_path / "claim-review.json"
config.write_text(
json.dumps(
{
"required_experiments": [
{