from __future__ import annotations import json from pathlib import Path from types import SimpleNamespace from ocrtech.evalpack import create_eval_pack from ocrtech.manifest import ManifestEntry, sha256_file, sha256_text, write_manifest from ocrtech.models import write_model_card from ocrtech.preflight import run_claim_preflight def test_preflight_passes_for_smoke_pack_with_candidate_model(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["ours"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, ) assert report.passed assert any(check.name == "eval-pack" and check.status == "warn" for check in report.checks) assert (tmp_path / "preflight" / "preflight.json").exists() def test_preflight_fails_without_candidate_model_config(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", require_claim_ready_eval_pack=False, ) assert not report.passed candidate_check = next(check for check in report.checks if check.name == "baseline:candidate") assert candidate_check.status == "fail" assert "--candidate-model-config" in candidate_check.summary def test_preflight_audits_fallback_model_config(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture", fallback_model_card=fallback, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, fallback_model_config=fallback, require_claim_ready_eval_pack=False, ) fallback_check = next(check for check in report.checks if check.name == "fallback-model") assert fallback_check.status == "pass" assert "ocrtech-fallback" in fallback_check.summary provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") assert provenance_check.status == "pass" def test_preflight_fails_when_fallback_model_card_audit_fails(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-bad-fallback", backend="paddleocr", base_model="fixture") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture", fallback_model_card=fallback, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, fallback_model_config=fallback, require_claim_ready_eval_pack=False, ) assert not report.passed fallback_check = next(check for check in report.checks if check.name == "fallback-model") assert fallback_check.status == "fail" assert "fallback model card failed audit" in fallback_check.summary assert any("paddleocr candidate requires at least one recorded artifact" in detail for detail in fallback_check.details) def test_preflight_requires_candidate_fallback_provenance_for_configured_fallback(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, fallback_model_config=fallback, require_claim_ready_eval_pack=False, ) assert not report.passed provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") assert provenance_check.status == "fail" assert "missing provenance.fallback_model_card" in provenance_check.summary def test_preflight_rejects_candidate_fallback_provenance_mismatch(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") other_fallback = write_model_card(tmp_path / "other-fallback.json", model_id="ocrtech-other-fallback", backend="sidecar", base_model="fixture") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture", fallback_model_card=other_fallback, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, fallback_model_config=fallback, require_claim_ready_eval_pack=False, ) assert not report.passed provenance_check = next(check for check in report.checks if check.name == "candidate-fallback-provenance") assert provenance_check.status == "fail" assert "does not match configured fallback model" in provenance_check.summary def test_preflight_requires_trained_recognizer_when_requested(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, require_trained_recognizer=True, ) assert not report.passed trained_check = next(check for check in report.checks if check.name == "trained-recognizer") assert trained_check.status == "fail" assert "lacks trained local recognizer provenance" in trained_check.summary def test_preflight_accepts_trained_recognizer_when_requested(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" source = tmp_path / "source.txt" source.write_text("नेपाल सरकार\n", encoding="utf-8") entry = ManifestEntry( sample_id="sample-1", dataset="fixture", split="train", image_path=str(source), text="नेपाल सरकार", sha256=sha256_file(source), ) write_manifest([entry], train_manifest) write_manifest([ManifestEntry(**{**entry.to_dict(), "split": "eval"})], eval_manifest) artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") training_summary = tmp_path / "recognizer-training-summary.json" training_summary.write_text('{"backend": "paddleocr"}\n', encoding="utf-8") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-trained", backend="paddleocr", base_model="fixture-pretrained", artifact_paths=[artifact], backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)}, train_manifest=train_manifest, eval_manifest=eval_manifest, training_summary=training_summary, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, require_trained_recognizer=True, ) trained_check = next(check for check in report.checks if check.name == "trained-recognizer") assert trained_check.status == "pass" def test_preflight_requires_model_admission_when_requested(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, require_model_admission=True, ) assert not report.passed admission_check = next(check for check in report.checks if check.name == "model-admission") assert admission_check.status == "fail" assert "not passed admission validation" in admission_check.summary def test_preflight_accepts_model_admission_when_requested(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) validation = tmp_path / "validation.json" artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") model_path = tmp_path / "model.json" validation.write_text( json.dumps( { "claim_status": "validated", "passed": True, "provenance": {"candidate_model_config": {"path": str(model_path)}}, "gates": [_passing_validation_gate()], } ) + "\n", encoding="utf-8", ) card = write_model_card( model_path, model_id="ocrtech-admitted", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne", "text_recognition_model_dir": str(tmp_path)}, admission_validation_report=validation, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, require_model_admission=True, ) admission_check = next(check for check in report.checks if check.name == "model-admission") assert admission_check.status == "pass" def test_preflight_fails_when_glm_hook_is_missing(tmp_path: Path, monkeypatch) -> None: monkeypatch.delenv("OCRTECH_GLM_OCR_CMD", raising=False) pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["glm-ocr"], references_dir=tmp_path / "pack" / "references", require_claim_ready_eval_pack=False, ) assert not report.passed glm_check = next(check for check in report.checks if check.name == "baseline:glm-ocr") assert glm_check.status == "fail" assert "OCRTECH_GLM_OCR_CMD" in glm_check.summary def test_preflight_fails_when_glm_hook_is_not_runnable(tmp_path: Path, monkeypatch) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) monkeypatch.setenv("OCRTECH_GLM_OCR_CMD", "missing-python scripts/run_glm_ocr.py --input {input}") report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["glm-ocr"], references_dir=tmp_path / "pack" / "references", require_claim_ready_eval_pack=False, ) assert not report.passed glm_check = next(check for check in report.checks if check.name == "baseline:glm-ocr") assert glm_check.status == "fail" assert "not runnable" in glm_check.summary assert any("executable not found" in detail for detail in glm_check.details) assert any("{out}" in detail for detail in glm_check.details) def test_preflight_fails_when_paddle_runtime_is_missing(tmp_path: Path, monkeypatch) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) real_find_spec = __import__("importlib").util.find_spec def fake_find_spec(name: str): if name == "paddleocr": return SimpleNamespace() if name == "paddle": return None return real_find_spec(name) monkeypatch.setattr("importlib.util.find_spec", fake_find_spec) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["stock-paddle"], references_dir=tmp_path / "pack" / "references", require_claim_ready_eval_pack=False, ) assert not report.passed paddle_check = next(check for check in report.checks if check.name == "baseline:stock-paddle") assert paddle_check.status == "fail" assert "dependencies are incomplete" in paddle_check.summary assert "python module missing: paddle" in paddle_check.details def test_preflight_fails_when_surya_runtime_is_missing(tmp_path: Path, monkeypatch) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) real_find_spec = __import__("importlib").util.find_spec def fake_find_spec(name: str): if name == "surya": return None return real_find_spec(name) monkeypatch.setattr("importlib.util.find_spec", fake_find_spec) monkeypatch.setattr("shutil.which", lambda name: None if name == "llama-server" else None) monkeypatch.delenv("LLAMA_CPP_BINARY", raising=False) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["surya"], references_dir=tmp_path / "pack" / "references", require_claim_ready_eval_pack=False, ) assert not report.passed surya_check = next(check for check in report.checks if check.name == "baseline:surya") assert surya_check.status == "fail" assert "dependencies are incomplete" in surya_check.summary assert "python module missing: surya" in surya_check.details def test_preflight_fails_on_train_eval_leakage(tmp_path: Path) -> None: source = tmp_path / "sample.txt" source.write_text("hello world\n", encoding="utf-8") reference = tmp_path / "sample.ref.txt" reference.write_text("hello world\n", encoding="utf-8") image_hash = sha256_file(source) text_hash = sha256_text("hello world") sample_hash = sha256_text(f"{image_hash}\n{text_hash}") entry = ManifestEntry( sample_id="shared-sample", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=image_hash, metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sample_hash, "input_format": "text"}, ) eval_manifest = tmp_path / "eval.jsonl" train_manifest = tmp_path / "train.jsonl" write_manifest([entry], eval_manifest) write_manifest([ManifestEntry(**{**entry.to_dict(), "split": "train"})], train_manifest) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( eval_manifest, tmp_path / "preflight", baselines=["ours"], candidate_model_config=card, train_manifests=[train_manifest], require_claim_ready_eval_pack=False, ) assert not report.passed leakage_check = next(check for check in report.checks if check.name == "leakage") assert leakage_check.status == "fail" assert "leakage" in leakage_check.summary def test_preflight_fails_on_unsupported_ocr_image_suffix(tmp_path: Path) -> None: source = tmp_path / "sample.img" source.write_bytes(b"fake image bytes") reference = tmp_path / "sample.ref.txt" reference.write_text("hello world\n", encoding="utf-8") image_hash = sha256_file(source) text_hash = sha256_text("hello world") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-img", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=image_hash, metadata={"reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ], manifest, ) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( manifest, tmp_path / "preflight", baselines=["candidate"], candidate_model_config=card, require_claim_ready_eval_pack=False, ) assert not report.passed suffix_check = next(check for check in report.checks if check.name == "eval-input-suffixes") assert suffix_check.status == "fail" assert "normalize-manifest-images" in suffix_check.summary assert any(".img" in detail for detail in suffix_check.details) def test_preflight_accepts_context_compatible_validation_config(tmp_path: Path, monkeypatch) -> None: monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/tesseract" if name == "tesseract" else None) source = tmp_path / "sample.txt" source.write_text("hello world\n", encoding="utf-8") reference = tmp_path / "sample.ref.txt" reference.write_text("hello world\n", encoding="utf-8") image_hash = sha256_file(source) text_hash = sha256_text("hello world") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=image_hash, metadata={ "reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), "input_format": "text", "slices": ["english"], }, ) ], manifest, ) config = tmp_path / "validation.json" config.write_text( json.dumps( { "candidate": "candidate", "min_samples": 1, "required_slices": {"english": 1}, "gates": [ { "baseline": "tesseract", "metric": "cer", "direction": "lower", "slices": ["english"], "min_pairs": 1, } ], } ), encoding="utf-8", ) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( manifest, tmp_path / "preflight", baselines=["candidate", "tesseract"], candidate_model_config=card, validation_config=config, require_claim_ready_eval_pack=False, ) validation_check = next(check for check in report.checks if check.name == "validation-config") assert validation_check.status == "pass" def test_preflight_rejects_validation_config_that_cannot_match_run_context(tmp_path: Path) -> None: source = tmp_path / "sample.txt" source.write_text("hello world\n", encoding="utf-8") reference = tmp_path / "sample.ref.txt" reference.write_text("hello world\n", encoding="utf-8") image_hash = sha256_file(source) text_hash = sha256_text("hello world") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=image_hash, metadata={ "reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), "input_format": "text", "slices": ["english"], }, ) ], manifest, ) config = tmp_path / "validation.json" config.write_text( json.dumps( { "candidate": "candidate", "min_samples": 2, "required_slices": {"nepali": 2}, "gates": [ { "baseline": "tesseract", "metric": "cer", "direction": "lower", "slices": ["nepali"], "min_pairs": 2, } ], } ), encoding="utf-8", ) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( manifest, tmp_path / "preflight", baselines=["candidate"], candidate_model_config=card, validation_config=config, require_claim_ready_eval_pack=False, ) assert not report.passed validation_check = next(check for check in report.checks if check.name == "validation-config") assert validation_check.status == "fail" assert "cannot be satisfied" in validation_check.summary assert "gate baselines not included in preflight baselines: tesseract" in validation_check.details assert "validation min_samples 2 exceeds eval manifest sample count 1" in validation_check.details assert "required slice 'nepali' count 2 exceeds eval manifest count 0" in validation_check.details assert any("min_pairs 2 exceeds eval manifest capacity 0" in detail for detail in validation_check.details) def test_preflight_allows_text_input_format_with_txt_suffix(tmp_path: Path) -> None: source = tmp_path / "sample.txt" source.write_text("hello world\n", encoding="utf-8") reference = tmp_path / "sample.ref.txt" reference.write_text("hello world\n", encoding="utf-8") image_hash = sha256_file(source) text_hash = sha256_text("hello world") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-text", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=image_hash, metadata={ "reference_path": str(reference), "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), "input_format": "text", }, ) ], manifest, ) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") report = run_claim_preflight( manifest, tmp_path / "preflight", baselines=["candidate"], candidate_model_config=card, require_claim_ready_eval_pack=False, ) suffix_check = next(check for check in report.checks if check.name == "eval-input-suffixes") assert suffix_check.status == "pass" def test_preflight_fails_when_quality_router_is_calibrated_on_same_eval_manifest(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) card = write_model_card( tmp_path / "model.json", model_id="ocrtech-quality-router", backend="quality_select_composite", base_model="tesseract-ocr", backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "tesseract", "secondary_quality_margin": 0.02}, provenance={ "calibration": { "eval_manifest": { "path": str(pack.manifest_path), "sha256": sha256_file(Path(pack.manifest_path)), }, "metric": "cer", "selected_threshold": 0.02, } }, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate", "tesseract"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, ) assert not report.passed calibration_check = next(check for check in report.checks if check.name == "candidate-calibration") assert calibration_check.status == "fail" assert "matches the claim eval manifest" in calibration_check.summary def test_preflight_fails_when_script_router_collapses_to_secondary_engine(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) calibration = tmp_path / "calibration.json" calibration.write_text( json.dumps( { "eval_manifest": "other.jsonl", "metric": "cer", "selected_threshold": 0.0, "selected_trial": { "threshold": 0.0, "pairs": 5, "secondary_picks": 5, "chosen_mean": 0.1, "primary_mean": 0.5, "absolute_improvement": 0.4, "relative_improvement": 0.8, }, } ), encoding="utf-8", ) card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.0}, metrics_report=calibration, provenance={ "calibration": { "eval_manifest": { "path": "different-eval.jsonl", "sha256": "1234567890abcdef", }, "metric": "cer", "selected_threshold": 0.0, } }, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate", "tesseract", "surya"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, ) assert not report.passed calibration_check = next(check for check in report.checks if check.name == "candidate-calibration") assert calibration_check.status == "fail" assert "collapses to the secondary engine" in calibration_check.summary def test_preflight_fails_when_composite_candidate_component_baseline_is_missing(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) calibration = tmp_path / "calibration.json" calibration.write_text( json.dumps( { "eval_manifest": "separate-dev.jsonl", "metric": "cer", "selected_threshold": 0.4, "selected_trial": { "threshold": 0.4, "pairs": 5, "secondary_picks": 3, "chosen_mean": 0.1, "primary_mean": 0.2, "absolute_improvement": 0.1, "relative_improvement": 0.5, }, } ), encoding="utf-8", ) card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.4}, metrics_report=calibration, provenance={ "calibration": { "eval_manifest": { "path": "separate-dev.jsonl", "sha256": "abcdef1234567890", }, "metric": "cer", "selected_threshold": 0.4, } }, ) report = run_claim_preflight( pack.manifest_path, tmp_path / "preflight", baselines=["candidate", "tesseract"], references_dir=tmp_path / "pack" / "references", candidate_model_config=card, require_claim_ready_eval_pack=False, ) assert not report.passed component_check = next(check for check in report.checks if check.name == "candidate-components") assert component_check.status == "fail" assert "component baselines" in component_check.summary assert any("surya" in detail for detail in component_check.details) def _passing_validation_gate() -> dict[str, object]: return { "passed": True, "pairs": 2, "candidate_mean": 0.1, "baseline_mean": 0.2, "absolute_improvement": 0.1, "relative_improvement": 0.5, "improvement_ci_low": 0.01, "win_count": 2, "loss_count": 0, "tie_count": 0, "pair_win_rate": 1.0, "sign_test_p_value": 0.0, "gate": { "baseline": "tesseract", "metric": "cer", "min_pairs": 2, "min_pair_win_rate": 0.5, "require_ci_above_zero": True, }, }