from __future__ import annotations import json from pathlib import Path import pytest from ocrtech.errors import ClaimReviewError from ocrtech.claim_review import review_claim from ocrtech.evalpack import create_eval_pack from ocrtech.experiments import run_experiment from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest from ocrtech.models import write_model_card def test_review_claim_passes_for_internal_and_public_bundle(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model) public = _build_public_experiment(tmp_path / "public", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], "required_slices": ["nepali", "english", "table"], }, { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], }, ] } ), encoding="utf-8", ) report = review_claim([internal, public], tmp_path / "review", config_path=config, model_config=model) assert report.passed assert len(report.experiments) == 2 assert (tmp_path / "review" / "claim-review.json").exists() def test_review_claim_fails_when_required_metric_is_missing(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer", "figure_detection_f1", "not_a_real_metric"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "missing metrics: candidate:not_a_real_metric" in required_check.summary def test_review_claim_derives_error_counts_from_benchmark_rows(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) rows[0]["status"] = "error" rows[0]["error"] = "simulated stale status summary" benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8") payload["benchmark_status_counts"] = {"ok": len(rows)} stale = tmp_path / "stale-status-counts-experiment.json" stale.write_text(json.dumps(payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], } ] } ), encoding="utf-8", ) report = review_claim([stale], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "benchmark has 1 error rows" in required_check.summary def test_review_claim_fails_when_eval_manifest_hash_is_stale(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) payload["environment"]["eval_manifest_sha256"] = "0" * 64 stale = tmp_path / "stale-eval-hash-experiment.json" stale.write_text(json.dumps(payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], } ] } ), encoding="utf-8", ) report = review_claim([stale], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "experiment eval_manifest sha256 does not match environment.eval_manifest_sha256" in required_check.summary def test_review_claim_fails_when_benchmark_sample_is_not_in_eval_manifest(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) rows[0]["sample_id"] = "not-in-eval-manifest" benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "benchmark row sample_id not in eval_manifest: not-in-eval-manifest" in required_check.summary def test_review_claim_fails_when_benchmark_slices_do_not_match_eval_manifest(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) rows[0]["slices"] = ["table"] benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "benchmark row slices do not match eval_manifest for sample_id: ocrbench-s1" in required_check.summary def test_review_claim_fails_when_required_baseline_skips_eval_sample(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) benchmark_report.write_text(json.dumps([], indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert rows assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "missing baselines: candidate" in required_check.summary assert "missing baseline sample coverage: candidate:ocrbench-s1" in required_check.summary def test_review_claim_accepts_required_metric_slice_coverage(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["table_cell_f1"], "required_metric_slices": {"table_cell_f1": ["table"]}, "required_slices": ["table"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "pass" def test_review_claim_fails_when_required_metric_is_missing_on_slice(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) for row in rows: if row.get("baseline") != "candidate" or row.get("status") != "ok": continue metrics = row.get("metrics") if not isinstance(metrics, dict): continue if "table" in set(row.get("slices") or []): metrics.pop("table_cell_f1", None) else: metrics["table_cell_f1"] = 1.0 benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["table_cell_f1"], "required_metric_slices": {"table_cell_f1": ["table"]}, "required_slices": ["table"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "missing metric slice coverage: candidate:table_cell_f1@table" in required_check.summary def test_review_claim_accepts_required_metric_paired_coverage(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) _clone_candidate_rows_to_baseline(public, "tesseract") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate", "tesseract"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert report.passed def test_review_claim_fails_when_required_metric_has_no_candidate_baseline_pairs(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) baseline_row = dict(next(row for row in rows if row.get("baseline") == "candidate")) baseline_row["baseline"] = "tesseract" baseline_row["sample_id"] = "different-sample" baseline_row["input_path"] = str(tmp_path / "different-input.png") rows.append(baseline_row) benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate", "tesseract"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "missing paired metric coverage: candidate:tesseract:cer@all" in required_check.summary def test_review_claim_accepts_candidate_fallback_policy(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) _set_candidate_metric(public, "fallback_triggered", 0.0) _set_candidate_metric(public, "fallback_failed", 0.0) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], "require_candidate_fallback_metrics": True, "max_candidate_fallback_trigger_rate": 0.5, "max_candidate_fallback_failed_rate": 0.0, } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "pass" def test_review_claim_fails_when_required_output_artifact_is_missing(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) candidate_row = next(row for row in rows if row.get("baseline") == "candidate") document_md = benchmark_report.parent / "candidate" / Path(candidate_row["input_path"]).stem / "document.md" document_md.unlink() config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], "required_output_artifacts": ["document_json", "document_md"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "missing output artifacts: candidate:ocrbench-s1:document_md" in required_check.summary def test_review_claim_fails_when_required_output_artifact_is_invalid_json(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) candidate_row = next(row for row in rows if row.get("baseline") == "candidate") document_json = benchmark_report.parent / "candidate" / Path(candidate_row["input_path"]).stem / "document.json" document_json.write_text("{not-json", encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], "required_output_artifacts": ["document_json"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "candidate:ocrbench-s1:document_json (invalid json:" in required_check.summary def test_review_claim_fails_when_candidate_fallback_metrics_are_missing(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], "require_candidate_fallback_metrics": True, } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "missing candidate fallback_triggered metrics" in required_check.summary assert "missing candidate fallback_failed metrics" in required_check.summary def test_review_claim_fails_when_candidate_fallback_trigger_rate_is_too_high(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) _set_candidate_metric(public, "fallback_triggered", 1.0) _set_candidate_metric(public, "fallback_failed", 0.0) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], "require_candidate_fallback_metrics": True, "max_candidate_fallback_trigger_rate": 0.5, } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:ocrbench-v2") assert required_check.status == "fail" assert "candidate fallback trigger rate 1.000000 exceeds max 0.500000" in required_check.summary def test_review_claim_audits_recorded_fallback_model_config(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert report.passed fallback_check = next(check for check in report.checks if check.name == "fallback-model") assert fallback_check.status == "pass" assert any("audited model_id=ocrtech-fallback" in detail for detail in fallback_check.details) def test_review_claim_can_require_fallback_model_config_presence(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_fallback_model_present": True, "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed fallback_check = next(check for check in report.checks if check.name == "fallback-model") assert fallback_check.status == "fail" assert "missing required fallback_model_config provenance" in fallback_check.summary assert fallback_check.details == ["experiment"] def test_review_claim_requires_candidate_fallback_model_card_provenance(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_fallback_model_present": True, "require_candidate_fallback_provenance": True, "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") assert provenance_check.status == "fail" assert "missing provenance.fallback_model_card" in provenance_check.summary def test_review_claim_accepts_candidate_fallback_model_card_provenance(tmp_path: Path) -> None: fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") model = write_model_card( tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture", fallback_model_card=fallback, ) public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_fallback_model_present": True, "require_candidate_fallback_provenance": True, "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert report.passed provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") assert provenance_check.status == "pass" assert "matches reviewed experiment fallback configs" in provenance_check.summary def test_review_claim_rejects_candidate_fallback_model_card_mismatch(tmp_path: Path) -> None: fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") other_fallback = write_model_card(tmp_path / "other-fallback.json", model_id="ocrtech-other-fallback", backend="sidecar", base_model="fixture") model = write_model_card( tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture", fallback_model_card=other_fallback, ) public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_fallback_model_present": True, "require_candidate_fallback_provenance": True, "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") assert provenance_check.status == "fail" assert "does not match experiment fallback_model_config" in provenance_check.summary def test_review_claim_fails_when_fallback_model_hash_is_missing(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback) payload = json.loads(public.read_text(encoding="utf-8")) payload["environment"]["fallback_model_config_sha256"] = None public.write_text(json.dumps(payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed fallback_check = next(check for check in report.checks if check.name == "fallback-model") assert fallback_check.status == "fail" assert "fallback model config path is recorded but sha256 is missing" in fallback_check.summary def test_review_claim_fails_when_fallback_model_card_audit_fails(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-bad-fallback", backend="paddleocr", base_model="fixture") public = _rewrite_fallback_model_path(_build_public_experiment(tmp_path / "public", model), fallback) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([public], tmp_path / "review", config_path=config, model_config=model) assert not report.passed fallback_check = next(check for check in report.checks if check.name == "fallback-model") assert fallback_check.status == "fail" assert "fallback model card failed audit" in fallback_check.summary assert any("paddleocr candidate requires at least one recorded artifact" in detail for detail in fallback_check.details) def test_review_claim_verifies_validation_report_status_matches_experiment(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) validation_report = Path(payload["validation_report"]) validation_payload = json.loads(validation_report.read_text(encoding="utf-8")) validation_payload["claim_status"] = "not_validated" validation_payload["passed"] = False validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "validation_report claim_status not_validated != required validated" in required_check.summary def test_review_claim_verifies_validation_report_benchmark_provenance(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) validation_report = Path(payload["validation_report"]) validation_payload = json.loads(validation_report.read_text(encoding="utf-8")) validation_payload["provenance"]["benchmark_report"]["sha256"] = "0" * 64 validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "validation_report benchmark_report sha256 does not match experiment benchmark_report" in required_check.summary def test_review_claim_verifies_validation_report_config_provenance(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) validation_report = Path(payload["validation_report"]) validation_payload = json.loads(validation_report.read_text(encoding="utf-8")) validation_payload["provenance"]["validation_config"]["sha256"] = "0" * 64 validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "validation_report validation_config sha256 does not match experiment validation_config" in required_check.summary def test_review_claim_fails_when_experiment_used_wrong_validation_config(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) required_config = tmp_path / "required-validation.json" required_config.write_text( json.dumps( { "candidate": "candidate", "min_samples": 99, "required_slices": {"nepali": 99}, "gates": [], } ), encoding="utf-8", ) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "required_validation_config": str(required_config), "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "experiment validation_config sha256 does not match required config" in required_check.summary def test_review_claim_verifies_validation_report_train_manifest_provenance(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) validation_report = Path(payload["validation_report"]) validation_payload = json.loads(validation_report.read_text(encoding="utf-8")) validation_payload["provenance"]["train_manifests"] = [{"path": "data/train.jsonl", "sha256": "0" * 64}] validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "validation_report train_manifests provenance does not match experiment train_manifests" in required_check.summary def test_review_claim_verifies_validation_report_candidate_model_provenance(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) validation_report = Path(payload["validation_report"]) validation_payload = json.loads(validation_report.read_text(encoding="utf-8")) validation_payload["provenance"]["candidate_model_config"]["sha256"] = "0" * 64 validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "validation_report candidate_model_config sha256 does not match experiment candidate_model_config" in required_check.summary def test_review_claim_requires_validation_report_fallback_model_provenance_when_experiment_has_fallback(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") internal = _rewrite_fallback_model_path(_build_internal_experiment(tmp_path / "internal", model), fallback) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "validation_report missing provenance.fallback_model_config" in required_check.summary def test_review_claim_accepts_required_passing_validation_gate(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer", "quality_score"], "required_validation_gates": [ {"baseline": "candidate", "metric": "quality_score"} ], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "pass" def test_review_claim_fails_when_required_passing_validation_gate_is_missing(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate", "tesseract"], "required_metrics": ["cer"], "required_validation_gates": [ {"baseline": "tesseract", "metric": "cer", "slices": ["nepali"]} ], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "missing passing validation gate: tesseract:cer@nepali" in required_check.summary def test_review_claim_rejects_passing_validation_gate_without_quantitative_evidence(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) validation_report = Path(payload["validation_report"]) validation_payload = json.loads(validation_report.read_text(encoding="utf-8")) validation_payload["gates"] = [ { "passed": True, "gate": {"baseline": "candidate", "metric": "quality_score"}, } ] validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["quality_score"], "required_validation_gates": [ {"baseline": "candidate", "metric": "quality_score"} ], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "missing passing validation gate: candidate:quality_score" in required_check.summary def test_review_claim_rejects_passing_validation_gate_below_own_min_pairs(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) validation_report = Path(payload["validation_report"]) validation_payload = json.loads(validation_report.read_text(encoding="utf-8")) validation_payload["gates"] = [ { "passed": True, "pairs": 1, "candidate_mean": 1.0, "baseline_mean": 0.0, "absolute_improvement": 1.0, "relative_improvement": 1.0, "improvement_ci_low": 0.1, "win_count": 1, "loss_count": 0, "tie_count": 0, "pair_win_rate": 1.0, "sign_test_p_value": 0.0, "gate": { "baseline": "candidate", "metric": "quality_score", "min_pairs": 2, "require_ci_above_zero": True, }, } ] validation_report.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["quality_score"], "required_validation_gates": [ {"baseline": "candidate", "metric": "quality_score"} ], } ] } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed required_check = next(check for check in report.checks if check.name == "required:internal-pack") assert required_check.status == "fail" assert "missing passing validation gate: candidate:quality_score" in required_check.summary def test_review_claim_config_rejects_validation_gate_without_validation_status(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_baselines": ["candidate"], "required_metrics": ["quality_score"], "required_validation_gates": [{"baseline": "candidate", "metric": "quality_score"}], } ] } ), encoding="utf-8", ) with pytest.raises(ClaimReviewError, match="required_validation_gates without require_validation_status"): review_claim([internal], tmp_path / "review", config_path=config, model_config=model) def test_review_claim_config_rejects_validation_gate_outside_required_baselines(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], "required_validation_gates": [{"baseline": "tesseract", "metric": "cer"}], } ] } ), encoding="utf-8", ) with pytest.raises(ClaimReviewError, match="validation gate baselines not listed"): review_claim([internal], tmp_path / "review", config_path=config, model_config=model) def test_review_claim_config_rejects_validation_gate_outside_required_metrics(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], "required_validation_gates": [{"baseline": "candidate", "metric": "quality_score"}], } ] } ), encoding="utf-8", ) with pytest.raises(ClaimReviewError, match="validation gate metrics not listed"): review_claim([internal], tmp_path / "review", config_path=config, model_config=model) def test_review_claim_requires_trained_recognizer_when_configured(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") seed_model = write_model_card(tmp_path / "seed-trained-pass.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_trained_recognizer": True, "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed trained_check = next(check for check in report.checks if check.name == "trained-recognizer") assert trained_check.status == "fail" assert "lacks trained local recognizer provenance" in trained_check.summary def test_review_claim_accepts_trained_recognizer_provenance(tmp_path: Path) -> None: train_manifest = _write_fixture_manifest(tmp_path / "train.jsonl", sample_id="train-s1") eval_manifest = _write_fixture_manifest(tmp_path / "eval.jsonl", sample_id="eval-s1") artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"trained weights") training_summary = tmp_path / "recognizer-training-summary.json" training_summary.write_text('{"backend": "paddleocr"}\n', encoding="utf-8") model = write_model_card( tmp_path / "model.json", model_id="ocrtech-trained", backend="paddleocr", base_model="fixture-pretrained", artifact_paths=[artifact], backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)}, train_manifest=train_manifest, eval_manifest=eval_manifest, training_summary=training_summary, ) seed_model = write_model_card(tmp_path / "seed-trained-pass.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_trained_recognizer": True, "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert report.passed trained_check = next(check for check in report.checks if check.name == "trained-recognizer") assert trained_check.status == "pass" def test_review_claim_accepts_nested_trained_recognizer_provenance(tmp_path: Path) -> None: train_manifest = _write_fixture_manifest(tmp_path / "train.jsonl", sample_id="train-s1") eval_manifest = _write_fixture_manifest(tmp_path / "eval.jsonl", sample_id="eval-s1") artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"trained weights") training_summary = tmp_path / "recognizer-training-summary.json" training_summary.write_text('{"backend": "hf_vision_encoder_decoder"}\n', encoding="utf-8") trained_model = write_model_card( tmp_path / "trained-model.json", model_id="ocrtech-trained", backend="hf_vision_encoder_decoder", base_model="fixture-pretrained", artifact_paths=[artifact], backend_kwargs={"model_dir": str(tmp_path)}, train_manifest=train_manifest, eval_manifest=eval_manifest, training_summary=training_summary, ) composite_model = write_model_card( tmp_path / "composite.json", model_id="ocrtech-composite-with-trained-component", backend="table_cell_refine_composite", base_model="trained+tesseract", backend_kwargs={"structure_engine": "surya", "crop_engine": "tesseract"}, provenance={ "trained_recognizer_model_card": { "path": str(trained_model), "sha256": sha256_file(trained_model), } }, ) seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), composite_model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_trained_recognizer": True, "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=composite_model) assert report.passed trained_check = next(check for check in report.checks if check.name == "trained-recognizer") assert trained_check.status == "pass" def test_review_claim_requires_model_admission_when_configured(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_model_admission": True, "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert not report.passed admission_check = next(check for check in report.checks if check.name == "model-admission") assert admission_check.status == "fail" assert "not passed admission validation" in admission_check.summary def test_review_claim_accepts_model_admission_when_configured(tmp_path: Path) -> None: validation = tmp_path / "validation.json" artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") model_path = tmp_path / "model.json" validation.write_text( json.dumps( { "claim_status": "validated", "passed": True, "provenance": {"candidate_model_config": {"path": str(model_path)}}, "gates": [_passing_admission_validation_gate()], } ) + "\n", encoding="utf-8", ) model = write_model_card( model_path, model_id="ocrtech-admitted", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne"}, admission_validation_report=validation, ) seed_model = write_model_card(tmp_path / "seed-admission-pass.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _rewrite_candidate_model_path(_build_internal_experiment(tmp_path / "internal", seed_model), model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "require_model_admission": True, "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_metrics": ["cer"], } ], } ), encoding="utf-8", ) report = review_claim([internal], tmp_path / "review", config_path=config, model_config=model) assert report.passed admission_check = next(check for check in report.checks if check.name == "model-admission") assert admission_check.status == "pass" def test_review_claim_eval_manifest_patterns_use_wildcards(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") public = _build_public_experiment(tmp_path / "public", model) payload = json.loads(public.read_text(encoding="utf-8")) manifest = Path(payload["eval_manifest"]) real5_manifest = manifest.with_name("PaddlePaddle-Real5-OmniDocBench-eval.jsonl") real5_manifest.write_text(manifest.read_text(encoding="utf-8"), encoding="utf-8") payload["eval_manifest"] = str(real5_manifest) rewritten = tmp_path / "public-real5-experiment.json" rewritten.write_text(json.dumps(payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "real5-slot", "eval_manifest_patterns": ["*Real5-OmniDocBench*"], "require_baselines": ["candidate"], "required_metrics": ["cer"], } ] } ), encoding="utf-8", ) report = review_claim([rewritten], tmp_path / "review", config_path=config, model_config=model) assert report.passed required_check = next(check for check in report.checks if check.name == "required:real5-slot") assert required_check.status == "pass" def test_review_claim_rejects_stale_benchmark_report_path(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) payload["benchmark_report"] = str(tmp_path / "missing" / "report.json") stale = tmp_path / "stale-experiment.json" stale.write_text(json.dumps(payload, indent=2), encoding="utf-8") with pytest.raises(ClaimReviewError, match="benchmark_report .* does not exist"): review_claim([stale], tmp_path / "review", model_config=model) def test_review_claim_fails_on_inconsistent_candidate_model(tmp_path: Path) -> None: first_model = write_model_card(tmp_path / "model-a.json", model_id="ocrtech-sidecar-a", backend="sidecar", base_model="fixture") second_model = write_model_card(tmp_path / "model-b.json", model_id="ocrtech-sidecar-b", backend="sidecar", base_model="fixture", notes="different") first = _build_internal_experiment(tmp_path / "internal-a", first_model) second = _build_public_experiment(tmp_path / "public-b", second_model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ {"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"]}, {"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]}, ] } ), encoding="utf-8", ) report = review_claim([first, second], tmp_path / "review", config_path=config) assert not report.passed model_check = next(check for check in report.checks if check.name == "candidate-model") assert model_check.status == "fail" assert "do not share one model config" in model_check.summary def test_review_claim_fails_when_candidate_model_hash_is_stale_without_explicit_model(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", model) payload = json.loads(internal.read_text(encoding="utf-8")) payload["environment"]["candidate_model_config_sha256"] = "0" * 64 stale = tmp_path / "stale-candidate-hash-experiment.json" stale.write_text(json.dumps(payload, indent=2), encoding="utf-8") config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], } ] } ), encoding="utf-8", ) report = review_claim([stale], tmp_path / "review", config_path=config) assert not report.passed model_check = next(check for check in report.checks if check.name == "candidate-model") assert model_check.status == "fail" assert "candidate model config sha256 does not match experiment evidence" in model_check.summary def test_review_claim_fails_when_quality_router_calibration_matches_reviewed_eval(tmp_path: Path) -> None: seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", seed_model) internal_payload = json.loads(internal.read_text(encoding="utf-8")) internal_eval_manifest = Path(internal_payload["eval_manifest"]) routed_model = write_model_card( tmp_path / "quality-router.json", model_id="ocrtech-quality-router", backend="quality_select_composite", base_model="tesseract-ocr", backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "tesseract", "secondary_quality_margin": 0.02}, provenance={ "calibration": { "eval_manifest": { "path": str(internal_eval_manifest), "sha256": sha256_file(internal_eval_manifest), }, "metric": "cer", "selected_threshold": 0.02, } }, ) public = _build_public_experiment(tmp_path / "public", seed_model) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ {"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"]}, {"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]}, ] } ), encoding="utf-8", ) internal_for_review = _rewrite_candidate_model_path(internal, routed_model) public_for_review = _rewrite_candidate_model_path(public, routed_model) report = review_claim([internal_for_review, public_for_review], tmp_path / "review", config_path=config, model_config=routed_model) assert not report.passed calibration_check = next(check for check in report.checks if check.name == "candidate-calibration") assert calibration_check.status == "fail" assert "matches a reviewed experiment eval manifest" in calibration_check.summary def test_review_claim_fails_when_script_router_collapses_to_secondary_engine(tmp_path: Path) -> None: seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", seed_model) public = _build_public_experiment(tmp_path / "public", seed_model) calibration = tmp_path / "calibration.json" calibration.write_text( json.dumps( { "eval_manifest": "different.jsonl", "metric": "cer", "selected_threshold": 0.0, "selected_trial": { "threshold": 0.0, "pairs": 5, "secondary_picks": 5, "chosen_mean": 0.1, "primary_mean": 0.5, "absolute_improvement": 0.4, "relative_improvement": 0.8, }, } ), encoding="utf-8", ) routed_model = write_model_card( tmp_path / "script-router.json", model_id="ocrtech-script-router", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.0}, metrics_report=calibration, provenance={ "calibration": { "eval_manifest": { "path": "different-eval.jsonl", "sha256": "1234567890abcdef", }, "metric": "cer", "selected_threshold": 0.0, } }, ) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ {"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"]}, {"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]}, ] } ), encoding="utf-8", ) internal_for_review = _rewrite_candidate_model_path(internal, routed_model) public_for_review = _rewrite_candidate_model_path(public, routed_model) report = review_claim([internal_for_review, public_for_review], tmp_path / "review", config_path=config, model_config=routed_model) assert not report.passed calibration_check = next(check for check in report.checks if check.name == "candidate-calibration") assert calibration_check.status == "fail" assert "collapses to the secondary engine" in calibration_check.summary def test_review_claim_fails_when_routed_candidate_is_identical_to_component_baseline_on_holdout(tmp_path: Path) -> None: seed_model = write_model_card(tmp_path / "seed.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") internal = _build_internal_experiment(tmp_path / "internal", seed_model) _clone_candidate_rows_to_baseline(internal, "surya") public = _build_public_experiment(tmp_path / "public", seed_model) calibration = tmp_path / "calibration.json" calibration.write_text( json.dumps( { "eval_manifest": "separate-dev.jsonl", "metric": "cer", "selected_threshold": 0.4, "selected_trial": { "threshold": 0.4, "pairs": 5, "secondary_picks": 3, "chosen_mean": 0.1, "primary_mean": 0.2, "absolute_improvement": 0.1, "relative_improvement": 0.5, }, } ), encoding="utf-8", ) routed_model = write_model_card( tmp_path / "script-router.json", model_id="ocrtech-script-router", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.4}, metrics_report=calibration, provenance={ "calibration": { "eval_manifest": { "path": "separate-dev.jsonl", "sha256": "abcdef1234567890", }, "metric": "cer", "selected_threshold": 0.4, } }, ) config = tmp_path / "claim-review.json" config.write_text( json.dumps( { "required_experiments": [ {"name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate", "surya"]}, {"name": "ocrbench-v2", "datasets": ["ocrbench-v2"], "require_baselines": ["candidate"]}, ] } ), encoding="utf-8", ) internal_for_review = _rewrite_candidate_model_path(internal, routed_model) public_for_review = _rewrite_candidate_model_path(public, routed_model) report = review_claim([internal_for_review, public_for_review], tmp_path / "review", config_path=config, model_config=routed_model) assert not report.passed distinctness_check = next(check for check in report.checks if check.name == "candidate-distinctness") assert distinctness_check.status == "fail" assert "indistinguishable from component baseline surya" in distinctness_check.summary def _write_fixture_manifest(path: Path, *, sample_id: str) -> Path: source = path.with_suffix(".txt") source.write_text("नेपाल सरकार\n", encoding="utf-8") write_manifest( [ ManifestEntry( sample_id=sample_id, dataset="fixture", split="train" if "train" in path.stem else "eval", image_path=str(source), text="नेपाल सरकार", sha256=sha256_file(source), ) ], path, ) return path def _build_internal_experiment(root: Path, model: Path) -> Path: pack = create_eval_pack(root / "pack", input_format="text", count_per_template=1, degradations=["clean"]) config = root / "validation.json" config.parent.mkdir(parents=True, exist_ok=True) config.write_text( json.dumps( { "candidate": "candidate", "min_samples": 5, "required_slices": {"nepali": 1, "english": 1, "multi_page": 1}, "gates": [ { "baseline": "candidate", "metric": "quality_score", "direction": "higher", "min_absolute_margin": 0.0, "require_ci_above_zero": False, } ], } ), encoding="utf-8", ) report = run_experiment( pack.manifest_path, root / "experiment", baselines=["candidate"], references_dir=root / "pack" / "references", validation_config=config, candidate_model_config=model, ) return Path(report.output_dir) / "experiment.json" def _build_public_experiment(root: Path, model: Path) -> Path: inputs = root / "pack" / "inputs" refs = root / "pack" / "references" manifests = root / "pack" / "manifests" inputs.mkdir(parents=True) refs.mkdir() manifests.mkdir() source = inputs / "ocrbench-sample.txt" source.write_text("Invoice Number 42\nनेपाल सरकार\n", encoding="utf-8") refs.joinpath("ocrbench-sample.ref.txt").write_text("Invoice Number 42\nनेपाल सरकार\n", encoding="utf-8") manifest = manifests / "ocrbench-v2-eval.jsonl" write_manifest( [ ManifestEntry( sample_id="ocrbench-s1", dataset="ocrbench-v2", split="eval", image_path=str(source), text="Invoice Number 42\nनेपाल सरकार", metadata={"slices": ["english", "nepali", "mixed_script"], "reference_path": str(refs / "ocrbench-sample.ref.txt")}, ) ], manifest, ) report = run_experiment( manifest, root / "experiment", baselines=["candidate"], references_dir=refs, candidate_model_config=model, run_validation=False, ) return Path(report.output_dir) / "experiment.json" def _rewrite_candidate_model_path(experiment_path: Path, model: Path) -> Path: payload = json.loads(experiment_path.read_text(encoding="utf-8")) payload.setdefault("environment", {}) payload["environment"]["candidate_model_config"] = str(model) payload["environment"]["candidate_model_config_sha256"] = sha256_file(model) validation_report = payload.get("validation_report") if validation_report: validation_path = Path(validation_report) if validation_path.exists(): validation_payload = json.loads(validation_path.read_text(encoding="utf-8")) validation_payload.setdefault("provenance", {})["candidate_model_config"] = { "path": str(model), "sha256": sha256_file(model), } validation_path.write_text(json.dumps(validation_payload, indent=2), encoding="utf-8") rewritten = experiment_path.parent / "experiment-rewritten.json" rewritten.write_text(json.dumps(payload, indent=2), encoding="utf-8") return rewritten def _rewrite_fallback_model_path(experiment_path: Path, model: Path) -> Path: payload = json.loads(experiment_path.read_text(encoding="utf-8")) payload.setdefault("environment", {}) payload["environment"]["fallback_model_config"] = str(model) payload["environment"]["fallback_model_config_sha256"] = sha256_file(model) rewritten = experiment_path.parent / "experiment-fallback-rewritten.json" rewritten.write_text(json.dumps(payload, indent=2), encoding="utf-8") return rewritten def _clone_candidate_rows_to_baseline(experiment_path: Path, baseline_name: str) -> None: payload = json.loads(experiment_path.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) cloned_rows = [] for row in rows: if row.get("baseline") != "candidate": continue clone = dict(row) clone["baseline"] = baseline_name cloned_rows.append(clone) benchmark_report.write_text(json.dumps([*rows, *cloned_rows], indent=2), encoding="utf-8") def _set_candidate_metric(experiment_path: Path, metric: str, value: float) -> None: payload = json.loads(experiment_path.read_text(encoding="utf-8")) benchmark_report = Path(payload["benchmark_report"]) rows = json.loads(benchmark_report.read_text(encoding="utf-8")) for row in rows: if row.get("baseline") != "candidate" or row.get("status") != "ok": continue row.setdefault("metrics", {})[metric] = value benchmark_report.write_text(json.dumps(rows, indent=2), encoding="utf-8") def _passing_admission_validation_gate() -> dict[str, object]: return { "passed": True, "pairs": 2, "candidate_mean": 0.1, "baseline_mean": 0.2, "absolute_improvement": 0.1, "relative_improvement": 0.5, "improvement_ci_low": 0.01, "win_count": 2, "loss_count": 0, "tie_count": 0, "pair_win_rate": 1.0, "sign_test_p_value": 0.0, "gate": { "baseline": "tesseract", "metric": "cer", "min_pairs": 2, "min_pair_win_rate": 0.5, "require_ci_above_zero": True, }, }