Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_preflight.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 31.1 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_preflight.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_preflight.py
-
curl -L -o test_preflight.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_preflight.py
31.1 kB
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| from ocrtech.evalpack import create_eval_pack | |
| from ocrtech.manifest import ManifestEntry, sha256_file, sha256_text, write_manifest | |
| from ocrtech.models import write_model_card | |
| from ocrtech.preflight import run_claim_preflight | |
| def test_preflight_passes_for_smoke_pack_with_candidate_model(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["ours"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert report.passed | |
| assert any(check.name == "eval-pack" and check.status == "warn" for check in report.checks) | |
| assert (tmp_path / "preflight" / "preflight.json").exists() | |
| def test_preflight_fails_without_candidate_model_config(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| candidate_check = next(check for check in report.checks if check.name == "baseline:candidate") | |
| assert candidate_check.status == "fail" | |
| assert "--candidate-model-config" in candidate_check.summary | |
| def test_preflight_audits_fallback_model_config(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") | |
| card = write_model_card( | |
| tmp_path / "model.json", | |
| model_id="ocrtech-sidecar", | |
| backend="sidecar", | |
| base_model="fixture", | |
| fallback_model_card=fallback, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| fallback_model_config=fallback, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| fallback_check = next(check for check in report.checks if check.name == "fallback-model") | |
| assert fallback_check.status == "pass" | |
| assert "ocrtech-fallback" in fallback_check.summary | |
| provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") | |
| assert provenance_check.status == "pass" | |
| def test_preflight_fails_when_fallback_model_card_audit_fails(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-bad-fallback", backend="paddleocr", base_model="fixture") | |
| card = write_model_card( | |
| tmp_path / "model.json", | |
| model_id="ocrtech-sidecar", | |
| backend="sidecar", | |
| base_model="fixture", | |
| fallback_model_card=fallback, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| fallback_model_config=fallback, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| fallback_check = next(check for check in report.checks if check.name == "fallback-model") | |
| assert fallback_check.status == "fail" | |
| assert "fallback model card failed audit" in fallback_check.summary | |
| assert any("paddleocr candidate requires at least one recorded artifact" in detail for detail in fallback_check.details) | |
| def test_preflight_requires_candidate_fallback_provenance_for_configured_fallback(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| fallback_model_config=fallback, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") | |
| assert provenance_check.status == "fail" | |
| assert "missing provenance.fallback_model_card" in provenance_check.summary | |
| def test_preflight_rejects_candidate_fallback_provenance_mismatch(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") | |
| other_fallback = write_model_card(tmp_path / "other-fallback.json", model_id="ocrtech-other-fallback", backend="sidecar", base_model="fixture") | |
| card = write_model_card( | |
| tmp_path / "model.json", | |
| model_id="ocrtech-sidecar", | |
| backend="sidecar", | |
| base_model="fixture", | |
| fallback_model_card=other_fallback, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| fallback_model_config=fallback, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") | |
| assert provenance_check.status == "fail" | |
| assert "does not match configured fallback model" in provenance_check.summary | |
| def test_preflight_requires_trained_recognizer_when_requested(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| require_trained_recognizer=True, | |
| ) | |
| assert not report.passed | |
| trained_check = next(check for check in report.checks if check.name == "trained-recognizer") | |
| assert trained_check.status == "fail" | |
| assert "lacks trained local recognizer provenance" in trained_check.summary | |
| def test_preflight_accepts_trained_recognizer_when_requested(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| train_manifest = tmp_path / "train.jsonl" | |
| eval_manifest = tmp_path / "eval.jsonl" | |
| source = tmp_path / "source.txt" | |
| source.write_text("नेपाल सरकार\n", encoding="utf-8") | |
| entry = ManifestEntry( | |
| sample_id="sample-1", | |
| dataset="fixture", | |
| split="train", | |
| image_path=str(source), | |
| text="नेपाल सरकार", | |
| sha256=sha256_file(source), | |
| ) | |
| write_manifest([entry], train_manifest) | |
| write_manifest([ManifestEntry(**{**entry.to_dict(), "split": "eval"})], eval_manifest) | |
| artifact = tmp_path / "inference.pdiparams" | |
| artifact.write_bytes(b"weights") | |
| training_summary = tmp_path / "recognizer-training-summary.json" | |
| training_summary.write_text('{"backend": "paddleocr"}\n', encoding="utf-8") | |
| card = write_model_card( | |
| tmp_path / "model.json", | |
| model_id="ocrtech-trained", | |
| backend="paddleocr", | |
| base_model="fixture-pretrained", | |
| artifact_paths=[artifact], | |
| backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)}, | |
| train_manifest=train_manifest, | |
| eval_manifest=eval_manifest, | |
| training_summary=training_summary, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| require_trained_recognizer=True, | |
| ) | |
| trained_check = next(check for check in report.checks if check.name == "trained-recognizer") | |
| assert trained_check.status == "pass" | |
| def test_preflight_requires_model_admission_when_requested(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| require_model_admission=True, | |
| ) | |
| assert not report.passed | |
| admission_check = next(check for check in report.checks if check.name == "model-admission") | |
| assert admission_check.status == "fail" | |
| assert "not passed admission validation" in admission_check.summary | |
| def test_preflight_accepts_model_admission_when_requested(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| validation = tmp_path / "validation.json" | |
| artifact = tmp_path / "inference.pdiparams" | |
| artifact.write_bytes(b"weights") | |
| model_path = tmp_path / "model.json" | |
| validation.write_text( | |
| json.dumps( | |
| { | |
| "claim_status": "validated", | |
| "passed": True, | |
| "provenance": {"candidate_model_config": {"path": str(model_path)}}, | |
| "gates": [_passing_validation_gate()], | |
| } | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| card = write_model_card( | |
| model_path, | |
| model_id="ocrtech-admitted", | |
| backend="paddleocr", | |
| base_model="fixture", | |
| artifact_paths=[artifact], | |
| backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)}, | |
| admission_validation_report=validation, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| require_model_admission=True, | |
| ) | |
| admission_check = next(check for check in report.checks if check.name == "model-admission") | |
| assert admission_check.status == "pass" | |
| def test_preflight_fails_when_glm_hook_is_missing(tmp_path: Path, monkeypatch) -> None: | |
| monkeypatch.delenv("OCRTECH_GLM_OCR_CMD", raising=False) | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["glm-ocr"], | |
| references_dir=tmp_path / "pack" / "references", | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| glm_check = next(check for check in report.checks if check.name == "baseline:glm-ocr") | |
| assert glm_check.status == "fail" | |
| assert "OCRTECH_GLM_OCR_CMD" in glm_check.summary | |
| def test_preflight_fails_when_glm_hook_is_not_runnable(tmp_path: Path, monkeypatch) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| monkeypatch.setenv("OCRTECH_GLM_OCR_CMD", "missing-python scripts/run_glm_ocr.py --input {input}") | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["glm-ocr"], | |
| references_dir=tmp_path / "pack" / "references", | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| glm_check = next(check for check in report.checks if check.name == "baseline:glm-ocr") | |
| assert glm_check.status == "fail" | |
| assert "not runnable" in glm_check.summary | |
| assert any("executable not found" in detail for detail in glm_check.details) | |
| assert any("{out}" in detail for detail in glm_check.details) | |
| def test_preflight_fails_when_paddle_runtime_is_missing(tmp_path: Path, monkeypatch) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| real_find_spec = __import__("importlib").util.find_spec | |
| def fake_find_spec(name: str): | |
| if name == "paddleocr": | |
| return SimpleNamespace() | |
| if name == "paddle": | |
| return None | |
| return real_find_spec(name) | |
| monkeypatch.setattr("importlib.util.find_spec", fake_find_spec) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["stock-paddle"], | |
| references_dir=tmp_path / "pack" / "references", | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| paddle_check = next(check for check in report.checks if check.name == "baseline:stock-paddle") | |
| assert paddle_check.status == "fail" | |
| assert "dependencies are incomplete" in paddle_check.summary | |
| assert "python module missing: paddle" in paddle_check.details | |
| def test_preflight_fails_when_surya_runtime_is_missing(tmp_path: Path, monkeypatch) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| real_find_spec = __import__("importlib").util.find_spec | |
| def fake_find_spec(name: str): | |
| if name == "surya": | |
| return None | |
| return real_find_spec(name) | |
| monkeypatch.setattr("importlib.util.find_spec", fake_find_spec) | |
| monkeypatch.setattr("shutil.which", lambda name: None if name == "llama-server" else None) | |
| monkeypatch.delenv("LLAMA_CPP_BINARY", raising=False) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["surya"], | |
| references_dir=tmp_path / "pack" / "references", | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| surya_check = next(check for check in report.checks if check.name == "baseline:surya") | |
| assert surya_check.status == "fail" | |
| assert "dependencies are incomplete" in surya_check.summary | |
| assert "python module missing: surya" in surya_check.details | |
| def test_preflight_fails_on_train_eval_leakage(tmp_path: Path) -> None: | |
| source = tmp_path / "sample.txt" | |
| source.write_text("hello world\n", encoding="utf-8") | |
| reference = tmp_path / "sample.ref.txt" | |
| reference.write_text("hello world\n", encoding="utf-8") | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("hello world") | |
| sample_hash = sha256_text(f"{image_hash}\n{text_hash}") | |
| entry = ManifestEntry( | |
| sample_id="shared-sample", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="hello world", | |
| sha256=image_hash, | |
| metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sample_hash, "input_format": "text"}, | |
| ) | |
| eval_manifest = tmp_path / "eval.jsonl" | |
| train_manifest = tmp_path / "train.jsonl" | |
| write_manifest([entry], eval_manifest) | |
| write_manifest([ManifestEntry(**{**entry.to_dict(), "split": "train"})], train_manifest) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| eval_manifest, | |
| tmp_path / "preflight", | |
| baselines=["ours"], | |
| candidate_model_config=card, | |
| train_manifests=[train_manifest], | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| leakage_check = next(check for check in report.checks if check.name == "leakage") | |
| assert leakage_check.status == "fail" | |
| assert "leakage" in leakage_check.summary | |
| def test_preflight_fails_on_unsupported_ocr_image_suffix(tmp_path: Path) -> None: | |
| source = tmp_path / "sample.img" | |
| source.write_bytes(b"fake image bytes") | |
| reference = tmp_path / "sample.ref.txt" | |
| reference.write_text("hello world\n", encoding="utf-8") | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("hello world") | |
| manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-img", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="hello world", | |
| sha256=image_hash, | |
| metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| manifest, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| suffix_check = next(check for check in report.checks if check.name == "eval-input-suffixes") | |
| assert suffix_check.status == "fail" | |
| assert "normalize-manifest-images" in suffix_check.summary | |
| assert any(".img" in detail for detail in suffix_check.details) | |
| def test_preflight_accepts_context_compatible_validation_config(tmp_path: Path, monkeypatch) -> None: | |
| monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/tesseract" if name == "tesseract" else None) | |
| source = tmp_path / "sample.txt" | |
| source.write_text("hello world\n", encoding="utf-8") | |
| reference = tmp_path / "sample.ref.txt" | |
| reference.write_text("hello world\n", encoding="utf-8") | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("hello world") | |
| manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="hello world", | |
| sha256=image_hash, | |
| metadata={ | |
| "reference_path": str(reference), | |
| "text_sha256": text_hash, | |
| "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), | |
| "input_format": "text", | |
| "slices": ["english"], | |
| }, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| config = tmp_path / "validation.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "candidate", | |
| "min_samples": 1, | |
| "required_slices": {"english": 1}, | |
| "gates": [ | |
| { | |
| "baseline": "tesseract", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slices": ["english"], | |
| "min_pairs": 1, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| manifest, | |
| tmp_path / "preflight", | |
| baselines=["candidate", "tesseract"], | |
| candidate_model_config=card, | |
| validation_config=config, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| validation_check = next(check for check in report.checks if check.name == "validation-config") | |
| assert validation_check.status == "pass" | |
| def test_preflight_rejects_validation_config_that_cannot_match_run_context(tmp_path: Path) -> None: | |
| source = tmp_path / "sample.txt" | |
| source.write_text("hello world\n", encoding="utf-8") | |
| reference = tmp_path / "sample.ref.txt" | |
| reference.write_text("hello world\n", encoding="utf-8") | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("hello world") | |
| manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="hello world", | |
| sha256=image_hash, | |
| metadata={ | |
| "reference_path": str(reference), | |
| "text_sha256": text_hash, | |
| "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), | |
| "input_format": "text", | |
| "slices": ["english"], | |
| }, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| config = tmp_path / "validation.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "candidate", | |
| "min_samples": 2, | |
| "required_slices": {"nepali": 2}, | |
| "gates": [ | |
| { | |
| "baseline": "tesseract", | |
| "metric": "cer", | |
| "direction": "lower", | |
| "slices": ["nepali"], | |
| "min_pairs": 2, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| manifest, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| candidate_model_config=card, | |
| validation_config=config, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| validation_check = next(check for check in report.checks if check.name == "validation-config") | |
| assert validation_check.status == "fail" | |
| assert "cannot be satisfied" in validation_check.summary | |
| assert "gate baselines not included in preflight baselines: tesseract" in validation_check.details | |
| assert "validation min_samples 2 exceeds eval manifest sample count 1" in validation_check.details | |
| assert "required slice 'nepali' count 2 exceeds eval manifest count 0" in validation_check.details | |
| assert any("min_pairs 2 exceeds eval manifest capacity 0" in detail for detail in validation_check.details) | |
| def test_preflight_allows_text_input_format_with_txt_suffix(tmp_path: Path) -> None: | |
| source = tmp_path / "sample.txt" | |
| source.write_text("hello world\n", encoding="utf-8") | |
| reference = tmp_path / "sample.ref.txt" | |
| reference.write_text("hello world\n", encoding="utf-8") | |
| image_hash = sha256_file(source) | |
| text_hash = sha256_text("hello world") | |
| manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-text", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="hello world", | |
| sha256=image_hash, | |
| metadata={ | |
| "reference_path": str(reference), | |
| "text_sha256": text_hash, | |
| "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), | |
| "input_format": "text", | |
| }, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| report = run_claim_preflight( | |
| manifest, | |
| tmp_path / "preflight", | |
| baselines=["candidate"], | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| suffix_check = next(check for check in report.checks if check.name == "eval-input-suffixes") | |
| assert suffix_check.status == "pass" | |
| def test_preflight_fails_when_quality_router_is_calibrated_on_same_eval_manifest(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| card = write_model_card( | |
| tmp_path / "model.json", | |
| model_id="ocrtech-quality-router", | |
| backend="quality_select_composite", | |
| base_model="tesseract-ocr", | |
| backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "tesseract", "secondary_quality_margin": 0.02}, | |
| provenance={ | |
| "calibration": { | |
| "eval_manifest": { | |
| "path": str(pack.manifest_path), | |
| "sha256": sha256_file(Path(pack.manifest_path)), | |
| }, | |
| "metric": "cer", | |
| "selected_threshold": 0.02, | |
| } | |
| }, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate", "tesseract"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| calibration_check = next(check for check in report.checks if check.name == "candidate-calibration") | |
| assert calibration_check.status == "fail" | |
| assert "matches the claim eval manifest" in calibration_check.summary | |
| def test_preflight_fails_when_script_router_collapses_to_secondary_engine(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| calibration = tmp_path / "calibration.json" | |
| calibration.write_text( | |
| json.dumps( | |
| { | |
| "eval_manifest": "other.jsonl", | |
| "metric": "cer", | |
| "selected_threshold": 0.0, | |
| "selected_trial": { | |
| "threshold": 0.0, | |
| "pairs": 5, | |
| "secondary_picks": 5, | |
| "chosen_mean": 0.1, | |
| "primary_mean": 0.5, | |
| "absolute_improvement": 0.4, | |
| "relative_improvement": 0.8, | |
| }, | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| card = write_model_card( | |
| tmp_path / "model.json", | |
| model_id="ocrtech-script-router", | |
| backend="script_select_composite", | |
| base_model="tesseract+surya", | |
| backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.0}, | |
| metrics_report=calibration, | |
| provenance={ | |
| "calibration": { | |
| "eval_manifest": { | |
| "path": "different-eval.jsonl", | |
| "sha256": "1234567890abcdef", | |
| }, | |
| "metric": "cer", | |
| "selected_threshold": 0.0, | |
| } | |
| }, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate", "tesseract", "surya"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| calibration_check = next(check for check in report.checks if check.name == "candidate-calibration") | |
| assert calibration_check.status == "fail" | |
| assert "collapses to the secondary engine" in calibration_check.summary | |
| def test_preflight_fails_when_composite_candidate_component_baseline_is_missing(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| calibration = tmp_path / "calibration.json" | |
| calibration.write_text( | |
| json.dumps( | |
| { | |
| "eval_manifest": "separate-dev.jsonl", | |
| "metric": "cer", | |
| "selected_threshold": 0.4, | |
| "selected_trial": { | |
| "threshold": 0.4, | |
| "pairs": 5, | |
| "secondary_picks": 3, | |
| "chosen_mean": 0.1, | |
| "primary_mean": 0.2, | |
| "absolute_improvement": 0.1, | |
| "relative_improvement": 0.5, | |
| }, | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| card = write_model_card( | |
| tmp_path / "model.json", | |
| model_id="ocrtech-script-router", | |
| backend="script_select_composite", | |
| base_model="tesseract+surya", | |
| backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.4}, | |
| metrics_report=calibration, | |
| provenance={ | |
| "calibration": { | |
| "eval_manifest": { | |
| "path": "separate-dev.jsonl", | |
| "sha256": "abcdef1234567890", | |
| }, | |
| "metric": "cer", | |
| "selected_threshold": 0.4, | |
| } | |
| }, | |
| ) | |
| report = run_claim_preflight( | |
| pack.manifest_path, | |
| tmp_path / "preflight", | |
| baselines=["candidate", "tesseract"], | |
| references_dir=tmp_path / "pack" / "references", | |
| candidate_model_config=card, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert not report.passed | |
| component_check = next(check for check in report.checks if check.name == "candidate-components") | |
| assert component_check.status == "fail" | |
| assert "component baselines" in component_check.summary | |
| assert any("surya" in detail for detail in component_check.details) | |
| def _passing_validation_gate() -> dict[str, object]: | |
| return { | |
| "passed": True, | |
| "pairs": 2, | |
| "candidate_mean": 0.1, | |
| "baseline_mean": 0.2, | |
| "absolute_improvement": 0.1, | |
| "relative_improvement": 0.5, | |
| "improvement_ci_low": 0.01, | |
| "win_count": 2, | |
| "loss_count": 0, | |
| "tie_count": 0, | |
| "pair_win_rate": 1.0, | |
| "sign_test_p_value": 0.0, | |
| "gate": { | |
| "baseline": "tesseract", | |
| "metric": "cer", | |
| "min_pairs": 2, | |
| "min_pair_win_rate": 0.5, | |
| "require_ci_above_zero": True, | |
| }, | |
| } | |