ocr-tech-k2 / tests /test_preflight.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
31.1 kB
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from ocrtech.evalpack import create_eval_pack
from ocrtech.manifest import ManifestEntry, sha256_file, sha256_text, write_manifest
from ocrtech.models import write_model_card
from ocrtech.preflight import run_claim_preflight
def test_preflight_passes_for_smoke_pack_with_candidate_model(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["ours"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
)
assert report.passed
assert any(check.name == "eval-pack" and check.status == "warn" for check in report.checks)
assert (tmp_path / "preflight" / "preflight.json").exists()
def test_preflight_fails_without_candidate_model_config(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
require_claim_ready_eval_pack=False,
)
assert not report.passed
candidate_check = next(check for check in report.checks if check.name == "baseline:candidate")
assert candidate_check.status == "fail"
assert "--candidate-model-config" in candidate_check.summary
def test_preflight_audits_fallback_model_config(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-sidecar",
backend="sidecar",
base_model="fixture",
fallback_model_card=fallback,
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
fallback_model_config=fallback,
require_claim_ready_eval_pack=False,
)
fallback_check = next(check for check in report.checks if check.name == "fallback-model")
assert fallback_check.status == "pass"
assert "ocrtech-fallback" in fallback_check.summary
provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance")
assert provenance_check.status == "pass"
def test_preflight_fails_when_fallback_model_card_audit_fails(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-bad-fallback", backend="paddleocr", base_model="fixture")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-sidecar",
backend="sidecar",
base_model="fixture",
fallback_model_card=fallback,
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
fallback_model_config=fallback,
require_claim_ready_eval_pack=False,
)
assert not report.passed
fallback_check = next(check for check in report.checks if check.name == "fallback-model")
assert fallback_check.status == "fail"
assert "fallback model card failed audit" in fallback_check.summary
assert any("paddleocr candidate requires at least one recorded artifact" in detail for detail in fallback_check.details)
def test_preflight_requires_candidate_fallback_provenance_for_configured_fallback(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
fallback_model_config=fallback,
require_claim_ready_eval_pack=False,
)
assert not report.passed
provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance")
assert provenance_check.status == "fail"
assert "missing provenance.fallback_model_card" in provenance_check.summary
def test_preflight_rejects_candidate_fallback_provenance_mismatch(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
other_fallback = write_model_card(tmp_path / "other-fallback.json", model_id="ocrtech-other-fallback", backend="sidecar", base_model="fixture")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-sidecar",
backend="sidecar",
base_model="fixture",
fallback_model_card=other_fallback,
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
fallback_model_config=fallback,
require_claim_ready_eval_pack=False,
)
assert not report.passed
provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance")
assert provenance_check.status == "fail"
assert "does not match configured fallback model" in provenance_check.summary
def test_preflight_requires_trained_recognizer_when_requested(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
require_trained_recognizer=True,
)
assert not report.passed
trained_check = next(check for check in report.checks if check.name == "trained-recognizer")
assert trained_check.status == "fail"
assert "lacks trained local recognizer provenance" in trained_check.summary
def test_preflight_accepts_trained_recognizer_when_requested(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
source = tmp_path / "source.txt"
source.write_text("नेपाल सरकार\n", encoding="utf-8")
entry = ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="train",
image_path=str(source),
text="नेपाल सरकार",
sha256=sha256_file(source),
)
write_manifest([entry], train_manifest)
write_manifest([ManifestEntry(**{**entry.to_dict(), "split": "eval"})], eval_manifest)
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
training_summary = tmp_path / "recognizer-training-summary.json"
training_summary.write_text('{"backend": "paddleocr"}\n', encoding="utf-8")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-trained",
backend="paddleocr",
base_model="fixture-pretrained",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
require_trained_recognizer=True,
)
trained_check = next(check for check in report.checks if check.name == "trained-recognizer")
assert trained_check.status == "pass"
def test_preflight_requires_model_admission_when_requested(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
require_model_admission=True,
)
assert not report.passed
admission_check = next(check for check in report.checks if check.name == "model-admission")
assert admission_check.status == "fail"
assert "not passed admission validation" in admission_check.summary
def test_preflight_accepts_model_admission_when_requested(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
validation = tmp_path / "validation.json"
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
model_path = tmp_path / "model.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(model_path)}},
"gates": [_passing_validation_gate()],
}
)
+ "\n",
encoding="utf-8",
)
card = write_model_card(
model_path,
model_id="ocrtech-admitted",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)},
admission_validation_report=validation,
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
require_model_admission=True,
)
admission_check = next(check for check in report.checks if check.name == "model-admission")
assert admission_check.status == "pass"
def test_preflight_fails_when_glm_hook_is_missing(tmp_path: Path, monkeypatch) -> None:
monkeypatch.delenv("OCRTECH_GLM_OCR_CMD", raising=False)
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["glm-ocr"],
references_dir=tmp_path / "pack" / "references",
require_claim_ready_eval_pack=False,
)
assert not report.passed
glm_check = next(check for check in report.checks if check.name == "baseline:glm-ocr")
assert glm_check.status == "fail"
assert "OCRTECH_GLM_OCR_CMD" in glm_check.summary
def test_preflight_fails_when_glm_hook_is_not_runnable(tmp_path: Path, monkeypatch) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
monkeypatch.setenv("OCRTECH_GLM_OCR_CMD", "missing-python scripts/run_glm_ocr.py --input {input}")
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["glm-ocr"],
references_dir=tmp_path / "pack" / "references",
require_claim_ready_eval_pack=False,
)
assert not report.passed
glm_check = next(check for check in report.checks if check.name == "baseline:glm-ocr")
assert glm_check.status == "fail"
assert "not runnable" in glm_check.summary
assert any("executable not found" in detail for detail in glm_check.details)
assert any("{out}" in detail for detail in glm_check.details)
def test_preflight_fails_when_paddle_runtime_is_missing(tmp_path: Path, monkeypatch) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
real_find_spec = __import__("importlib").util.find_spec
def fake_find_spec(name: str):
if name == "paddleocr":
return SimpleNamespace()
if name == "paddle":
return None
return real_find_spec(name)
monkeypatch.setattr("importlib.util.find_spec", fake_find_spec)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["stock-paddle"],
references_dir=tmp_path / "pack" / "references",
require_claim_ready_eval_pack=False,
)
assert not report.passed
paddle_check = next(check for check in report.checks if check.name == "baseline:stock-paddle")
assert paddle_check.status == "fail"
assert "dependencies are incomplete" in paddle_check.summary
assert "python module missing: paddle" in paddle_check.details
def test_preflight_fails_when_surya_runtime_is_missing(tmp_path: Path, monkeypatch) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
real_find_spec = __import__("importlib").util.find_spec
def fake_find_spec(name: str):
if name == "surya":
return None
return real_find_spec(name)
monkeypatch.setattr("importlib.util.find_spec", fake_find_spec)
monkeypatch.setattr("shutil.which", lambda name: None if name == "llama-server" else None)
monkeypatch.delenv("LLAMA_CPP_BINARY", raising=False)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["surya"],
references_dir=tmp_path / "pack" / "references",
require_claim_ready_eval_pack=False,
)
assert not report.passed
surya_check = next(check for check in report.checks if check.name == "baseline:surya")
assert surya_check.status == "fail"
assert "dependencies are incomplete" in surya_check.summary
assert "python module missing: surya" in surya_check.details
def test_preflight_fails_on_train_eval_leakage(tmp_path: Path) -> None:
source = tmp_path / "sample.txt"
source.write_text("hello world\n", encoding="utf-8")
reference = tmp_path / "sample.ref.txt"
reference.write_text("hello world\n", encoding="utf-8")
image_hash = sha256_file(source)
text_hash = sha256_text("hello world")
sample_hash = sha256_text(f"{image_hash}\n{text_hash}")
entry = ManifestEntry(
sample_id="shared-sample",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=image_hash,
metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sample_hash, "input_format": "text"},
)
eval_manifest = tmp_path / "eval.jsonl"
train_manifest = tmp_path / "train.jsonl"
write_manifest([entry], eval_manifest)
write_manifest([ManifestEntry(**{**entry.to_dict(), "split": "train"})], train_manifest)
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
eval_manifest,
tmp_path / "preflight",
baselines=["ours"],
candidate_model_config=card,
train_manifests=[train_manifest],
require_claim_ready_eval_pack=False,
)
assert not report.passed
leakage_check = next(check for check in report.checks if check.name == "leakage")
assert leakage_check.status == "fail"
assert "leakage" in leakage_check.summary
def test_preflight_fails_on_unsupported_ocr_image_suffix(tmp_path: Path) -> None:
source = tmp_path / "sample.img"
source.write_bytes(b"fake image bytes")
reference = tmp_path / "sample.ref.txt"
reference.write_text("hello world\n", encoding="utf-8")
image_hash = sha256_file(source)
text_hash = sha256_text("hello world")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-img",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=image_hash,
metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
],
manifest,
)
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
manifest,
tmp_path / "preflight",
baselines=["candidate"],
candidate_model_config=card,
require_claim_ready_eval_pack=False,
)
assert not report.passed
suffix_check = next(check for check in report.checks if check.name == "eval-input-suffixes")
assert suffix_check.status == "fail"
assert "normalize-manifest-images" in suffix_check.summary
assert any(".img" in detail for detail in suffix_check.details)
def test_preflight_accepts_context_compatible_validation_config(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/tesseract" if name == "tesseract" else None)
source = tmp_path / "sample.txt"
source.write_text("hello world\n", encoding="utf-8")
reference = tmp_path / "sample.ref.txt"
reference.write_text("hello world\n", encoding="utf-8")
image_hash = sha256_file(source)
text_hash = sha256_text("hello world")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=image_hash,
metadata={
"reference_path": str(reference),
"text_sha256": text_hash,
"sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"),
"input_format": "text",
"slices": ["english"],
},
)
],
manifest,
)
config = tmp_path / "validation.json"
config.write_text(
json.dumps(
{
"candidate": "candidate",
"min_samples": 1,
"required_slices": {"english": 1},
"gates": [
{
"baseline": "tesseract",
"metric": "cer",
"direction": "lower",
"slices": ["english"],
"min_pairs": 1,
}
],
}
),
encoding="utf-8",
)
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
manifest,
tmp_path / "preflight",
baselines=["candidate", "tesseract"],
candidate_model_config=card,
validation_config=config,
require_claim_ready_eval_pack=False,
)
validation_check = next(check for check in report.checks if check.name == "validation-config")
assert validation_check.status == "pass"
def test_preflight_rejects_validation_config_that_cannot_match_run_context(tmp_path: Path) -> None:
source = tmp_path / "sample.txt"
source.write_text("hello world\n", encoding="utf-8")
reference = tmp_path / "sample.ref.txt"
reference.write_text("hello world\n", encoding="utf-8")
image_hash = sha256_file(source)
text_hash = sha256_text("hello world")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=image_hash,
metadata={
"reference_path": str(reference),
"text_sha256": text_hash,
"sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"),
"input_format": "text",
"slices": ["english"],
},
)
],
manifest,
)
config = tmp_path / "validation.json"
config.write_text(
json.dumps(
{
"candidate": "candidate",
"min_samples": 2,
"required_slices": {"nepali": 2},
"gates": [
{
"baseline": "tesseract",
"metric": "cer",
"direction": "lower",
"slices": ["nepali"],
"min_pairs": 2,
}
],
}
),
encoding="utf-8",
)
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
manifest,
tmp_path / "preflight",
baselines=["candidate"],
candidate_model_config=card,
validation_config=config,
require_claim_ready_eval_pack=False,
)
assert not report.passed
validation_check = next(check for check in report.checks if check.name == "validation-config")
assert validation_check.status == "fail"
assert "cannot be satisfied" in validation_check.summary
assert "gate baselines not included in preflight baselines: tesseract" in validation_check.details
assert "validation min_samples 2 exceeds eval manifest sample count 1" in validation_check.details
assert "required slice 'nepali' count 2 exceeds eval manifest count 0" in validation_check.details
assert any("min_pairs 2 exceeds eval manifest capacity 0" in detail for detail in validation_check.details)
def test_preflight_allows_text_input_format_with_txt_suffix(tmp_path: Path) -> None:
source = tmp_path / "sample.txt"
source.write_text("hello world\n", encoding="utf-8")
reference = tmp_path / "sample.ref.txt"
reference.write_text("hello world\n", encoding="utf-8")
image_hash = sha256_file(source)
text_hash = sha256_text("hello world")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-text",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=image_hash,
metadata={
"reference_path": str(reference),
"text_sha256": text_hash,
"sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"),
"input_format": "text",
},
)
],
manifest,
)
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
report = run_claim_preflight(
manifest,
tmp_path / "preflight",
baselines=["candidate"],
candidate_model_config=card,
require_claim_ready_eval_pack=False,
)
suffix_check = next(check for check in report.checks if check.name == "eval-input-suffixes")
assert suffix_check.status == "pass"
def test_preflight_fails_when_quality_router_is_calibrated_on_same_eval_manifest(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-quality-router",
backend="quality_select_composite",
base_model="tesseract-ocr",
backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "tesseract", "secondary_quality_margin": 0.02},
provenance={
"calibration": {
"eval_manifest": {
"path": str(pack.manifest_path),
"sha256": sha256_file(Path(pack.manifest_path)),
},
"metric": "cer",
"selected_threshold": 0.02,
}
},
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate", "tesseract"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
)
assert not report.passed
calibration_check = next(check for check in report.checks if check.name == "candidate-calibration")
assert calibration_check.status == "fail"
assert "matches the claim eval manifest" in calibration_check.summary
def test_preflight_fails_when_script_router_collapses_to_secondary_engine(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
calibration = tmp_path / "calibration.json"
calibration.write_text(
json.dumps(
{
"eval_manifest": "other.jsonl",
"metric": "cer",
"selected_threshold": 0.0,
"selected_trial": {
"threshold": 0.0,
"pairs": 5,
"secondary_picks": 5,
"chosen_mean": 0.1,
"primary_mean": 0.5,
"absolute_improvement": 0.4,
"relative_improvement": 0.8,
},
}
),
encoding="utf-8",
)
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.0},
metrics_report=calibration,
provenance={
"calibration": {
"eval_manifest": {
"path": "different-eval.jsonl",
"sha256": "1234567890abcdef",
},
"metric": "cer",
"selected_threshold": 0.0,
}
},
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate", "tesseract", "surya"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
)
assert not report.passed
calibration_check = next(check for check in report.checks if check.name == "candidate-calibration")
assert calibration_check.status == "fail"
assert "collapses to the secondary engine" in calibration_check.summary
def test_preflight_fails_when_composite_candidate_component_baseline_is_missing(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
calibration = tmp_path / "calibration.json"
calibration.write_text(
json.dumps(
{
"eval_manifest": "separate-dev.jsonl",
"metric": "cer",
"selected_threshold": 0.4,
"selected_trial": {
"threshold": 0.4,
"pairs": 5,
"secondary_picks": 3,
"chosen_mean": 0.1,
"primary_mean": 0.2,
"absolute_improvement": 0.1,
"relative_improvement": 0.5,
},
}
),
encoding="utf-8",
)
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.4},
metrics_report=calibration,
provenance={
"calibration": {
"eval_manifest": {
"path": "separate-dev.jsonl",
"sha256": "abcdef1234567890",
},
"metric": "cer",
"selected_threshold": 0.4,
}
},
)
report = run_claim_preflight(
pack.manifest_path,
tmp_path / "preflight",
baselines=["candidate", "tesseract"],
references_dir=tmp_path / "pack" / "references",
candidate_model_config=card,
require_claim_ready_eval_pack=False,
)
assert not report.passed
component_check = next(check for check in report.checks if check.name == "candidate-components")
assert component_check.status == "fail"
assert "component baselines" in component_check.summary
assert any("surya" in detail for detail in component_check.details)
def _passing_validation_gate() -> dict[str, object]:
return {
"passed": True,
"pairs": 2,
"candidate_mean": 0.1,
"baseline_mean": 0.2,
"absolute_improvement": 0.1,
"relative_improvement": 0.5,
"improvement_ci_low": 0.01,
"win_count": 2,
"loss_count": 0,
"tie_count": 0,
"pair_win_rate": 1.0,
"sign_test_p_value": 0.0,
"gate": {
"baseline": "tesseract",
"metric": "cer",
"min_pairs": 2,
"min_pair_win_rate": 0.5,
"require_ci_above_zero": True,
},
}