from __future__ import annotations import json import sys from pathlib import Path from types import SimpleNamespace import pytest from ocrtech.cli import main from ocrtech.experiments import run_experiment from ocrtech.models import write_model_card from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest def test_cli_prepare_data_with_skip_invalid(tmp_path: Path, capsys) -> None: image = tmp_path / "line.png" image.write_bytes(b"line") labels = tmp_path / "labels.csv" labels.write_text("image_path,text\nline.png,नेपाल\nline.png,\n", encoding="utf-8") exit_code = main( [ "prepare-data", str(labels), "--out", str(tmp_path / "manifests"), "--dataset-name", "local", "--skip-invalid", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "local-train.jsonl" in captured.out assert "local-train-rejects.jsonl" in captured.out assert "prepared=1 rejected=1" in captured.out def test_cli_prepare_data_can_stamp_slices(tmp_path: Path, capsys) -> None: image = tmp_path / "line.png" image.write_bytes(b"line") labels = tmp_path / "labels.csv" labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8") exit_code = main( [ "prepare-data", str(labels), "--out", str(tmp_path / "manifests"), "--dataset-name", "local", "--slice", "nepali", "--slice", "real", ] ) entries = [json.loads(line) for line in (tmp_path / "manifests" / "local-train.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()] captured = capsys.readouterr() assert exit_code == 0 assert entries[0]["metadata"]["slices"] == ["nepali", "real"] assert "local-train.jsonl" in captured.out def test_cli_parse_doc_sidecar_text(tmp_path: Path, capsys) -> None: source = tmp_path / "sample.txt" source.write_text("नेपाल सरकार\n\nhello world\n", encoding="utf-8") exit_code = main(["parse-doc", str(source), "--out", str(tmp_path / "out"), "--engine", "sidecar"]) captured = capsys.readouterr() assert exit_code == 0 assert "document.json" in captured.out assert (tmp_path / "out" / "document.md").exists() def test_cli_parse_doc_passes_fallback_model_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: captured: dict[str, object] = {} source = tmp_path / "sample.png" source.write_bytes(b"image") class DocumentStub: pages: list[object] = [] tables: list[object] = [] figures: list[object] = [] def fake_parse_document(*args: object, **kwargs: object) -> DocumentStub: captured["args"] = args captured["kwargs"] = kwargs return DocumentStub() monkeypatch.setattr("ocrtech.cli.parse_document", fake_parse_document) exit_code = main( [ "parse-doc", str(source), "--out", str(tmp_path / "out"), "--engine", "sidecar", "--fallback-model-config", str(tmp_path / "fallback.json"), ] ) assert exit_code == 0 assert captured["kwargs"]["fallback_model_config"] == str(tmp_path / "fallback.json") def test_cli_validate_claim_returns_success_only_when_validated(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ { "baseline": "ours", "input_path": "a.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.01}, "error": None, }, { "baseline": "tesseract", "input_path": "a.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.10}, "error": None, }, ] ), encoding="utf-8", ) config = tmp_path / "config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 1, "required_slices": {"nepali": 1}, "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower", "slice": "nepali"}], } ), encoding="utf-8", ) exit_code = main(["validate-claim", "--benchmark-report", str(report), "--config", str(config), "--out", str(tmp_path / "validation")]) assert exit_code == 0 assert (tmp_path / "validation" / "validation.json").exists() def test_cli_recognizer_admission_decision_exit_codes(tmp_path: Path) -> None: admitted = tmp_path / "admitted.json" admitted.write_text('{"claim_status":"validated","passed":true,"sample_count":3}\n', encoding="utf-8") rejected = tmp_path / "rejected.json" rejected.write_text('{"claim_status":"not_validated","passed":false,"sample_count":3}\n', encoding="utf-8") admitted_exit = main( [ "recognizer-admission-decision", "--validation-report", str(admitted), "--out", str(tmp_path / "admitted-decision"), ] ) rejected_exit = main( [ "recognizer-admission-decision", "--validation-report", str(rejected), "--out", str(tmp_path / "rejected-decision"), ] ) assert admitted_exit == 0 assert rejected_exit == 1 assert (tmp_path / "admitted-decision" / "recognizer-admission-decision.json").exists() assert (tmp_path / "rejected-decision" / "recognizer-admission-decision.md").exists() def test_cli_recognizer_failure_review_stops_family(tmp_path: Path) -> None: args = ["recognizer-failure-review", "--out", str(tmp_path / "review")] for index in range(3): card = tmp_path / f"model-{index}.json" card.write_text( json.dumps( { "model_id": f"candidate-{index}", "backend": "paddleocr", "base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", "artifacts": [], "backend_kwargs": {}, "metrics": {}, } ), encoding="utf-8", ) decision = tmp_path / f"decision-{index}.json" decision.write_text( json.dumps( { "decision": "reject", "claim_status": "not_validated", "passed": False, "sample_count": 50, "failed_gates": [ { "baseline": "tesseract", "metric": "cer", "slices": ["nepali"], "pairs": 50, "win_count": 0, "loss_count": 50, "tie_count": 0, } ], } ), encoding="utf-8", ) args.extend(["--admission-decision", str(decision), "--model-card", str(card)]) exit_code = main(args) review = json.loads((tmp_path / "review" / "recognizer-failure-review.json").read_text(encoding="utf-8")) assert exit_code == 1 assert review["families"][0]["recommendation"] == "stop_family" assert (tmp_path / "review" / "recognizer-failure-review.md").exists() def test_cli_create_and_audit_eval_pack(tmp_path: Path) -> None: pack_dir = tmp_path / "pack" create_exit = main(["create-eval-pack", "--out", str(pack_dir), "--input-format", "text"]) manifest = pack_dir / "manifests" / "ocrtech-eval-pack.jsonl" assert create_exit == 0 assert manifest.exists() audit_exit = main(["audit-eval-pack", str(manifest), "--out", str(tmp_path / "audit")]) assert audit_exit == 0 assert (tmp_path / "audit" / "eval-pack-audit.json").exists() def test_cli_audit_and_split_manifest(tmp_path: Path) -> None: manifest = tmp_path / "combined.jsonl" entries = [] for index in range(6): image = tmp_path / f"line-{index}.png" image.write_bytes(f"image-{index}".encode("utf-8")) image_hash = sha256_file(image) text = "नेपाल" if index % 2 == 0 else "English" text_hash = sha256_text(text) entries.append( ManifestEntry( sample_id=f"sample-{index}", dataset="local", split="all", image_path=str(image), text=text, sha256=image_hash, metadata={"slices": ["nepali" if index % 2 == 0 else "english"], "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ) write_manifest(entries, manifest) audit_exit = main(["audit-manifest", str(manifest), "--out", str(tmp_path / "audit"), "--require-slice", "nepali"]) split_exit = main(["split-manifest", str(manifest), "--out", str(tmp_path / "split"), "--eval-ratio", "0.33", "--seed", "1"]) assert audit_exit == 0 assert split_exit == 0 assert (tmp_path / "audit" / "manifest-audit.json").exists() assert (tmp_path / "split" / "train.jsonl").exists() assert (tmp_path / "split" / "eval.jsonl").exists() def test_cli_merge_manifests(tmp_path: Path) -> None: first = tmp_path / "first.jsonl" second = tmp_path / "second.jsonl" write_manifest(_entries(tmp_path, start=0, count=2), first) write_manifest(_entries(tmp_path, start=2, count=2), second) merge_exit = main(["merge-manifests", str(first), str(second), "--out", str(tmp_path / "combined.jsonl")]) assert merge_exit == 0 assert (tmp_path / "combined.jsonl").exists() assert (tmp_path / "combined-merge.json").exists() assert (tmp_path / "combined-merge.md").exists() def test_cli_filter_manifest(tmp_path: Path, capsys) -> None: manifest = tmp_path / "combined.jsonl" write_manifest(_entries(tmp_path, start=0, count=4), manifest) exit_code = main( [ "filter-manifest", str(manifest), "--out", str(tmp_path / "filtered.jsonl"), "--sample-id", "merge-sample-0", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "selected=1 source=4" in captured.out assert (tmp_path / "filtered.jsonl").exists() assert (tmp_path / "filtered-filter.json").exists() assert (tmp_path / "filtered-filter.md").exists() def test_cli_rebalance_manifest(tmp_path: Path, capsys) -> None: manifest = tmp_path / "combined.jsonl" write_manifest(_entries(tmp_path, start=0, count=5), manifest) exit_code = main( [ "rebalance-manifest", str(manifest), "--out", str(tmp_path / "rebalanced.jsonl"), "--slice", "english", "--slice", "nepali", "--target-count", "4", "--seed", "7", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "rebalanced.jsonl" in captured.out assert "rebalanced-rebalance.json" in captured.out assert "original=5 rebalanced=8 target=4" in captured.out assert (tmp_path / "rebalanced.jsonl").exists() def test_cli_audit_recognizer_corpus(tmp_path: Path, capsys) -> None: train = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" image_train_en = tmp_path / "train-en.png" image_train_ne = tmp_path / "train-ne.png" image_train_mix = tmp_path / "train-mix.png" image_eval_mix = tmp_path / "eval-mix.png" for path in [image_train_en, image_train_ne, image_train_mix, image_eval_mix]: path.write_bytes(path.name.encode("utf-8")) write_manifest( [ ManifestEntry(sample_id="tr-en", dataset="fixture", split="train", image_path=str(image_train_en), text="Invoice", sha256=sha256_file(image_train_en), metadata={"slices": ["english", "real"]}), ManifestEntry(sample_id="tr-ne", dataset="fixture", split="train", image_path=str(image_train_ne), text="नेपाल", sha256=sha256_file(image_train_ne), metadata={"slices": ["nepali", "synthetic"]}), ManifestEntry(sample_id="tr-mix", dataset="fixture", split="train", image_path=str(image_train_mix), text="नाम Ram", sha256=sha256_file(image_train_mix), metadata={"slices": ["mixed_script", "real", "synthetic"]}), ], train, ) write_manifest( [ ManifestEntry(sample_id="ev-mix", dataset="fixture", split="eval", image_path=str(image_eval_mix), text="बिल No", sha256=sha256_file(image_eval_mix), metadata={"slices": ["mixed_script", "hard_eval"]}), ], eval_manifest, ) exit_code = main( [ "audit-recognizer-corpus", "--train-manifest", str(train), "--eval-manifest", str(eval_manifest), "--out", str(tmp_path / "audit"), "--min-train-samples", "3", "--min-eval-samples", "1", "--min-train-english", "2", "--min-train-devanagari", "2", "--min-train-mixed", "1", "--min-eval-english", "1", "--min-eval-devanagari", "1", "--min-eval-mixed", "1", "--min-train-latin-only", "1", "--min-train-devanagari-only", "1", "--min-eval-latin-only", "0", "--min-eval-devanagari-only", "0", "--min-train-real", "2", "--min-train-synthetic", "2", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "recognizer-corpus-audit.json" in captured.out assert "passed=True train=3 eval=1" in captured.out assert (tmp_path / "audit" / "recognizer-corpus-audit.md").exists() def test_cli_build_recognizer_corpus(tmp_path: Path, capsys) -> None: source = tmp_path / "source.jsonl" images = { "en": tmp_path / "en.png", "ne": tmp_path / "ne.png", "mix": tmp_path / "mix.png", } for path in images.values(): path.write_bytes(path.name.encode("utf-8")) write_manifest( [ ManifestEntry(sample_id="en", dataset="fixture", split="train", image_path=str(images["en"]), text="Invoice", sha256=sha256_file(images["en"]), metadata={"slices": ["english", "real"]}), ManifestEntry(sample_id="ne", dataset="fixture", split="train", image_path=str(images["ne"]), text="नेपाल", sha256=sha256_file(images["ne"]), metadata={"slices": ["nepali", "real"]}), ManifestEntry(sample_id="mix", dataset="fixture", split="train", image_path=str(images["mix"]), text="नाम Ram", sha256=sha256_file(images["mix"]), metadata={"slices": ["mixed_script", "synthetic"]}), ], source, ) exit_code = main( [ "build-recognizer-corpus", str(source), "--out", str(tmp_path / "built.jsonl"), "--target-latin-only", "2", "--target-devanagari-only", "2", "--target-mixed", "2", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "source=3" in captured.out assert (tmp_path / "built.jsonl").exists() assert (tmp_path / "built-recognizer-corpus-build.json").exists() def test_cli_materialize_references(tmp_path: Path, capsys) -> None: source = tmp_path / "sample.png" source.write_bytes(b"png") manifest = tmp_path / "eval.jsonl" image_hash = sha256_file(source) text_hash = sha256_text("hello world") write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=image_hash, metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ], manifest, ) exit_code = main( [ "materialize-references", str(manifest), "--out", str(tmp_path / "references"), "--rewritten-manifest", str(tmp_path / "eval-with-refs.jsonl"), ] ) captured = capsys.readouterr() assert exit_code == 0 assert "reference-pack.json" in captured.out assert (tmp_path / "references" / "sample-1" / "document.md").exists() def test_cli_normalize_manifest_images(tmp_path: Path, capsys, monkeypatch: pytest.MonkeyPatch) -> None: source = tmp_path / "sample.img" source.write_bytes(b"raw image bytes") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="hello world", sha256=sha256_file(source), ) ], manifest, ) class FakeImage: def __enter__(self): return self def __exit__(self, exc_type, exc, tb) -> None: return None def save(self, target: Path, format: str) -> None: assert format == "PNG" target.write_bytes(b"converted png bytes") monkeypatch.setitem(sys.modules, "PIL", SimpleNamespace(Image=SimpleNamespace(open=lambda path: FakeImage()))) exit_code = main(["normalize-manifest-images", str(manifest), "--out", str(tmp_path / "normalized")]) captured = capsys.readouterr() assert exit_code == 0 assert "manifest-images-normalized.jsonl" in captured.out assert "converted=1" in captured.out entries = load_manifest(tmp_path / "normalized" / "manifest-images-normalized.jsonl") assert Path(entries[0].image_path).suffix == ".png" assert (tmp_path / "normalized" / "image-normalization.json").exists() def test_cli_train_recognizer_hf_backend(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setattr("ocrtech.cli.write_hf_recognizer_recipe", lambda *args, **kwargs: tmp_path / "run" / "hf_recognizer_config.json") exit_code = main( [ "train-recognizer", "--backend", "hf-vision-encoder-decoder", "--train-manifest", "train.jsonl", "--eval-manifest", "eval.jsonl", "--out", str(tmp_path / "run"), ] ) assert exit_code == 0 def test_cli_bundle_recognizer_training_passes_corpus_audit_options(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: captured: dict[str, object] = {} class FakeBundle: archive_path = str(tmp_path / "bundle.tar.gz") manifest_path = str(tmp_path / "bundle.json") readme_path = str(tmp_path / "bundle.md") file_count = 3 total_bytes = 42 def fake_bundle(*args, **kwargs): captured["args"] = args captured["kwargs"] = kwargs return FakeBundle() monkeypatch.setattr("ocrtech.cli.bundle_recognizer_training", fake_bundle) exit_code = main( [ "bundle-recognizer-training", "--recipe-dir", "runs/recipe", "--train-manifest", "data/train.jsonl", "--eval-manifest", "data/eval.jsonl", "--out", str(tmp_path / "bundle"), "--require-corpus-audit", "--min-train-samples", "11", "--min-eval-samples", "7", "--min-train-english", "5", "--min-train-latin-only", "3", "--failure-review", str(tmp_path / "review.json"), ] ) assert exit_code == 0 kwargs = captured["kwargs"] assert isinstance(kwargs, dict) assert kwargs["require_corpus_audit"] is True assert kwargs["min_train_samples"] == 11 assert kwargs["min_eval_samples"] == 7 assert kwargs["min_train_english"] == 5 assert kwargs["min_train_latin_only"] == 3 assert kwargs["failure_review"] == str(tmp_path / "review.json") def test_cli_generate_correction_pairs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: output = tmp_path / "pairs.jsonl" monkeypatch.setattr("ocrtech.cli.generate_correction_pairs", lambda *args, **kwargs: output) exit_code = main( [ "generate-correction-pairs", "--manifest", "train.jsonl", "--out", str(output), "--engine", "tesseract", ] ) assert exit_code == 0 def test_cli_train_corrector_hf_backend(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: def fake_recipe(*args, **kwargs): pairs = tmp_path / "corrector" / "correction_pairs.jsonl" pairs.parent.mkdir(parents=True, exist_ok=True) pairs.write_text('{"noisy_text":"गलत","clean_text":"सही"}\n{"noisy_text":"A","clean_text":"A"}\n', encoding="utf-8") return tmp_path / "corrector" / "corrector_config.json" monkeypatch.setattr("ocrtech.cli.write_corrector_recipe", fake_recipe) monkeypatch.setattr("ocrtech.cli.write_hf_text_corrector_recipe", lambda *args, **kwargs: tmp_path / "corrector" / "hf_text_corrector_config.json") exit_code = main( [ "train-corrector", "--pairs-source", "pairs.jsonl", "--out", str(tmp_path / "corrector"), "--backend", "hf-seq2seq", ] ) assert exit_code == 0 def test_cli_run_experiment_without_validation(tmp_path: Path) -> None: pack_dir = tmp_path / "pack" create_exit = main(["create-eval-pack", "--out", str(pack_dir), "--input-format", "text"]) manifest = pack_dir / "manifests" / "ocrtech-eval-pack.jsonl" experiment_exit = main( [ "run-experiment", "--eval-manifest", str(manifest), "--out", str(tmp_path / "experiment"), "--baselines", "ours", "--skip-validation", ] ) assert create_exit == 0 assert experiment_exit == 0 assert (tmp_path / "experiment" / "experiment.json").exists() assert (tmp_path / "experiment" / "benchmark" / "report.json").exists() def test_cli_preflight_claim_with_smoke_pack(tmp_path: Path) -> None: pack_dir = tmp_path / "pack" create_exit = main(["create-eval-pack", "--out", str(pack_dir), "--input-format", "text"]) manifest = pack_dir / "manifests" / "ocrtech-eval-pack.jsonl" fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-cli-fallback", backend="sidecar", base_model="fixture") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-cli-sidecar", backend="sidecar", base_model="fixture", fallback_model_card=fallback, ) preflight_exit = main( [ "preflight-claim", "--eval-manifest", str(manifest), "--references-dir", str(pack_dir / "references"), "--baselines", "ours", "--candidate-model-config", str(card), "--fallback-model-config", str(fallback), "--allow-smoke-eval-pack", "--out", str(tmp_path / "preflight"), ] ) assert create_exit == 0 assert preflight_exit == 0 assert (tmp_path / "preflight" / "preflight.json").exists() payload = json.loads((tmp_path / "preflight" / "preflight.json").read_text(encoding="utf-8")) fallback_check = next(check for check in payload["checks"] if check["name"] == "fallback-model") assert fallback_check["status"] == "pass" provenance_check = next(check for check in payload["checks"] if check["name"] == "candidate-fallback-provenance") assert provenance_check["status"] == "pass" def test_cli_review_claim(tmp_path: Path) -> None: model = write_model_card(tmp_path / "model.json", model_id="ocrtech-cli-sidecar", backend="sidecar", base_model="fixture") source = tmp_path / "sample.txt" source.write_text("hello world\n", encoding="utf-8") source.with_suffix(".ref.txt").write_text("hello world\n", encoding="utf-8") manifest = tmp_path / "eval.jsonl" manifest.write_text( '{"sample_id":"s1","dataset":"ocrtech-eval-pack","split":"eval","image_path":"' + str(source) + '","text":"hello world","sha256":null,"metadata":{"slices":["english"],"reference_path":"' + str(source.with_suffix(".ref.txt")) + '"}}\n', encoding="utf-8", ) validation_config = tmp_path / "validation.json" validation_config.write_text( json.dumps( { "candidate": "candidate", "min_samples": 1, "required_slices": {"english": 1}, "gates": [{"baseline": "candidate", "metric": "quality_score", "direction": "higher", "require_ci_above_zero": False}], } ), encoding="utf-8", ) experiment = run_experiment( manifest, tmp_path / "experiment", baselines=["candidate"], candidate_model_config=model, run_validation=True, validation_config=validation_config, ) review_config = tmp_path / "claim-review.json" review_config.write_text( json.dumps( { "required_experiments": [ { "name": "internal-pack", "datasets": ["ocrtech-eval-pack"], "require_validation_status": "validated", "require_baselines": ["candidate"], "required_slices": ["english"], } ] } ), encoding="utf-8", ) review_exit = main( [ "review-claim", "--experiment", str(Path(experiment.output_dir) / "experiment.json"), "--config", str(review_config), "--model-config", str(model), "--out", str(tmp_path / "review"), ] ) assert review_exit == 0 assert (tmp_path / "review" / "claim-review.json").exists() def test_cli_audit_remote_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: captured: dict[str, object] = {} def fake_audit_remote_host(host, out, **kwargs): captured.update(kwargs) return type( "RemoteAuditResult", (), {"passed": True, "host": host}, )() monkeypatch.setattr( "ocrtech.cli.audit_remote_host", fake_audit_remote_host, ) exit_code = main( [ "audit-remote-host", "--host", "100.97.238.103", "--user", "k2", "--require-paddle-training", "--out", str(tmp_path / "remote"), ] ) assert exit_code == 0 assert captured["require_paddle_training"] is True def test_cli_bootstrap_remote_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setattr( "ocrtech.cli.bootstrap_remote_host", lambda host, out, **kwargs: type( "RemoteBootstrapResult", (), {"passed": True, "workdir": "/Users/k2/ocr-tech", "venv_dir": "/Users/k2/ocr-tech/.venv"}, )(), ) exit_code = main( [ "bootstrap-remote-host", "--host", "100.97.238.103", "--user", "k2", "--workdir", "~/ocr-tech", "--recreate-venv", "--out", str(tmp_path / "bootstrap"), ] ) assert exit_code == 0 def test_cli_sync_remote_workspace(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setattr( "ocrtech.cli.sync_remote_workspace", lambda local_root, out, **kwargs: type( "RemoteSyncResult", (), {"passed": True, "remote_workdir": "/Users/k2/ocr-tech"}, )(), ) exit_code = main( [ "sync-remote-workspace", "--host", "100.97.238.103", "--user", "k2", "--remote-workdir", "~/ocr-tech", "--out", str(tmp_path / "sync"), ] ) assert exit_code == 0 def test_cli_package_hf_recognizer(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: captured: dict[str, object] = {} def fake_package_hf_recognizer_model(*args: object, **kwargs: object) -> Path: captured["args"] = args captured["kwargs"] = kwargs return tmp_path / "package" / "model-card.json" monkeypatch.setattr("ocrtech.cli.package_hf_recognizer_model", fake_package_hf_recognizer_model) exit_code = main( [ "package-hf-recognizer", "--model-dir", "runs/hf/final-model", "--out", str(tmp_path / "package"), "--model-id", "ocrtech-hf", "--base-model", "microsoft/trocr-small-printed", "--admission-validation-report", "outputs/validation-next-recognizer-hard50/validation.json", "--training-summary", "runs/hf/recognizer-training-summary.json", ] ) assert exit_code == 0 assert captured["kwargs"]["admission_validation_report"] == "outputs/validation-next-recognizer-hard50/validation.json" assert captured["kwargs"]["training_summary"] == "runs/hf/recognizer-training-summary.json" def _entries(tmp_path: Path, *, start: int, count: int) -> list[ManifestEntry]: entries: list[ManifestEntry] = [] for index in range(start, start + count): image = tmp_path / f"merge-line-{index}.png" image.write_bytes(f"merge-image-{index}".encode("utf-8")) image_hash = sha256_file(image) text = "नेपाल" if index % 2 == 0 else "English" text_hash = sha256_text(text) entries.append( ManifestEntry( sample_id=f"merge-sample-{index}", dataset="local", split="all", image_path=str(image), text=text, sha256=image_hash, metadata={"slices": ["nepali" if index % 2 == 0 else "english"], "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ) return entries def test_cli_parse_python_version_rejects_invalid_format() -> None: exit_code = main(["audit-remote-host", "--host", "100.97.238.103", "--min-python", "3"]) assert exit_code == 2 def test_cli_create_and_audit_model_card(tmp_path: Path) -> None: artifact = tmp_path / "model.pdmodel" artifact.write_bytes(b"model") card = tmp_path / "model.json" create_exit = main( [ "create-model-card", "--out", str(card), "--model-id", "ocrtech-cli", "--backend", "sidecar", "--base-model", "fixture", "--artifact", str(artifact), ] ) audit_exit = main(["audit-model", str(card), "--out", str(tmp_path / "audit")]) assert create_exit == 0 assert audit_exit == 0 assert (tmp_path / "audit" / "model-audit.json").exists() def test_cli_create_tesseract_model_card_with_backend_kwargs(tmp_path: Path) -> None: card = tmp_path / "model.json" create_exit = main( [ "create-model-card", "--out", str(card), "--model-id", "ocrtech-tess-cli", "--backend", "tesseract", "--base-model", "tesseract-ocr", "--backend-kwarg", "language=nep+eng", "--backend-kwarg", "psm=7", ] ) payload = json.loads(card.read_text(encoding="utf-8")) assert create_exit == 0 assert payload["backend"] == "tesseract" assert payload["backend_kwargs"] == {"language": "nep+eng", "psm": 7} def test_cli_create_model_card_records_fallback_model_card(tmp_path: Path) -> None: fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") card = tmp_path / "model.json" create_exit = main( [ "create-model-card", "--out", str(card), "--model-id", "ocrtech-hybrid-cli", "--backend", "sidecar", "--base-model", "fixture", "--fallback-model-card", str(fallback), ] ) payload = json.loads(card.read_text(encoding="utf-8")) assert create_exit == 0 assert payload["provenance"]["fallback_model_card"] == { "path": str(fallback), "sha256": sha256_file(fallback), } def test_cli_summarize_benchmark(tmp_path: Path, capsys) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": "a.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["english"], "metrics": {"cer": 0.10}, "error": None, }, { "baseline": "tesseract", "input_path": "a.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["english"], "metrics": {"cer": 0.20}, "error": None, }, ] ), encoding="utf-8", ) exit_code = main( [ "summarize-benchmark", "--benchmark-report", str(report), "--out", str(tmp_path / "summary"), "--candidate", "candidate", "--baseline", "tesseract", "--metric", "cer", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "summary.json" in captured.out assert (tmp_path / "summary" / "summary.json").exists() assert (tmp_path / "summary" / "summary.md").exists() def test_cli_benchmark_loads_inputs_from_manifest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys) -> None: image = tmp_path / "image.png" image.write_bytes(b"image") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="s1", dataset="eval", split="eval", image_path="image.png", text="नेपाल", sha256=sha256_file(image), metadata={"reference_path": "refs/s1.json"}, ) ], manifest, ) captured_args: dict[str, object] = {} def fake_run_benchmark(inputs, output_dir, **kwargs): captured_args["inputs"] = inputs captured_args["output_dir"] = output_dir captured_args.update(kwargs) return [] monkeypatch.setattr("ocrtech.cli.run_benchmark", fake_run_benchmark) exit_code = main( [ "benchmark", "--inputs-from-manifest", str(manifest), "--out", str(tmp_path / "bench"), "--baselines", "candidate,tesseract", "--fallback-model-config", str(tmp_path / "fallback.json"), "--low-confidence-threshold", "0.71", ] ) stdout = capsys.readouterr().out assert exit_code == 0 assert captured_args["inputs"] == [str(image)] assert captured_args["eval_manifest"] == str(manifest) assert captured_args["baselines"] == ["candidate", "tesseract"] assert captured_args["fallback_model_config"] == str(tmp_path / "fallback.json") assert captured_args["low_confidence_threshold"] == 0.71 assert "runs=0" in stdout def test_cli_paper_benchmark_table(tmp_path: Path, capsys) -> None: first = tmp_path / "first-summary.json" second = tmp_path / "second-summary.json" first.write_text( json.dumps( { "report_path": "outputs/first/report.json", "candidate": "candidate", "baselines": ["tesseract"], "metrics": ["cer"], "aggregate_metrics": { "candidate": {"all": {"sample_count": 5, "cer": 0.22, "table_cell_f1": 1.0, "latency_seconds": 4.2}}, "tesseract": {"all": {"sample_count": 5, "cer": 0.73, "table_cell_f1": 0.0, "latency_seconds": 0.2}}, }, "paired_metrics": [], } ), encoding="utf-8", ) second.write_text( json.dumps( { "report_path": "outputs/second/report.json", "candidate": "candidate", "baselines": ["surya"], "metrics": ["cer"], "aggregate_metrics": { "candidate": {"all": {"sample_count": 5, "cer": 0.22, "reading_order_pair_accuracy": 0.78, "latency_seconds": 4.2}}, "surya": {"all": {"sample_count": 5, "cer": 0.19, "reading_order_pair_accuracy": 0.75, "latency_seconds": 3.8}}, }, "paired_metrics": [], } ), encoding="utf-8", ) exit_code = main( [ "paper-benchmark-table", "--summary", str(first), "--summary", str(second), "--out", str(tmp_path / "paper-table"), "--system", "candidate", "--system", "surya", "--system", "tesseract", "--label", "candidate=Candidate", "--source-preference", "candidate=" + str(first), "--metric", "cer", "--metric", "table_cell_f1", "--metric", "reading_order_pair_accuracy", "--metric", "latency_seconds", "--title", "Blur Comparison", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "paper-table.json" in captured.out assert (tmp_path / "paper-table" / "paper-table.json").exists() assert (tmp_path / "paper-table" / "paper-table.md").exists() def test_cli_benchmark_oracle(tmp_path: Path, capsys) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ { "baseline": "surya", "input_path": "s1.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["english"], "metrics": {"cer": 0.10}, "error": None, }, { "baseline": "tesseract", "input_path": "s1.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["english"], "metrics": {"cer": 0.20}, "error": None, }, ] ), encoding="utf-8", ) exit_code = main( [ "benchmark-oracle", "--benchmark-report", str(report), "--out", str(tmp_path / "oracle"), "--system", "surya", "--system", "tesseract", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "oracle-analysis.json" in captured.out assert (tmp_path / "oracle" / "oracle-analysis.json").exists() assert (tmp_path / "oracle" / "oracle-choices.json").exists() assert (tmp_path / "oracle" / "oracle-summary" / "summary.json").exists() def test_cli_benchmark_error_analysis(tmp_path: Path, capsys) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ { "baseline": "surya", "input_path": "s1.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.30}, "error": None, }, { "baseline": "tesseract", "input_path": "s1.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.20}, "error": None, }, ] ), encoding="utf-8", ) exit_code = main( [ "benchmark-error-analysis", "--benchmark-report", str(report), "--out", str(tmp_path / "analysis"), "--system", "surya", "--baseline", "tesseract", "--top-n", "1", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "error-analysis.json" in captured.out assert (tmp_path / "analysis" / "error-analysis.json").exists() assert (tmp_path / "analysis" / "top-failures.json").exists() def test_cli_export_failure_manifest(tmp_path: Path, capsys) -> None: image = tmp_path / "sample.png" image.write_bytes(b"image") manifest = tmp_path / "source.jsonl" write_manifest( [ ManifestEntry( sample_id="s1", dataset="fixture", split="eval", image_path=str(image), text="नेपाल", metadata={"slices": ["figure"]}, ) ], manifest, ) failures = tmp_path / "top-failures.json" failures.write_text(json.dumps([{"sample_id": "s1", "cer": 0.5, "badness": 0.5, "worst_gap": None, "comparisons": {}}]), encoding="utf-8") exit_code = main( [ "export-failure-manifest", "--source-manifest", str(manifest), "--top-failures", str(failures), "--out", str(tmp_path / "failure.jsonl"), "--split", "hard_eval", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "selected=1" in captured.out entries = load_manifest(tmp_path / "failure.jsonl") assert entries[0].split == "hard_eval" assert entries[0].metadata["failure_analysis"]["metric_value"] == 0.5 def test_cli_create_quality_select_model_card(tmp_path: Path) -> None: card = tmp_path / "model.json" create_exit = main( [ "create-model-card", "--out", str(card), "--model-id", "ocrtech-quality-router", "--backend", "quality_select_composite", "--base-model", "tesseract-ocr", "--backend-kwarg", "primary_engine=tesseract", "--backend-kwarg", "secondary_engine=tesseract", "--backend-kwarg", 'primary_engine_kwargs={"language":"nep+eng","psm":6}', "--backend-kwarg", 'secondary_engine_kwargs={"language":"nep+eng","psm":13}', "--backend-kwarg", "secondary_quality_margin=0.02", ] ) payload = json.loads(card.read_text(encoding="utf-8")) assert create_exit == 0 assert payload["backend"] == "quality_select_composite" assert payload["backend_kwargs"]["secondary_quality_margin"] == 0.02 assert payload["backend_kwargs"]["secondary_engine_kwargs"]["psm"] == 13 def test_cli_create_script_select_model_card(tmp_path: Path) -> None: card = tmp_path / "model.json" create_exit = main( [ "create-model-card", "--out", str(card), "--model-id", "ocrtech-script-router", "--backend", "script_select_composite", "--base-model", "tesseract+surya", "--backend-kwarg", "primary_engine=tesseract", "--backend-kwarg", "secondary_engine=surya", "--backend-kwarg", "primary_script_threshold=0.2", ] ) payload = json.loads(card.read_text(encoding="utf-8")) assert create_exit == 0 assert payload["backend"] == "script_select_composite" assert payload["backend_kwargs"]["primary_script_threshold"] == 0.2 def test_cli_create_line_align_model_card(tmp_path: Path) -> None: card = tmp_path / "model.json" create_exit = main( [ "create-model-card", "--out", str(card), "--model-id", "ocrtech-line-align", "--backend", "line_align_composite", "--base-model", "surya+tesseract", "--backend-kwarg", "primary_engine=surya", "--backend-kwarg", "secondary_engine=tesseract", "--backend-kwarg", "append_unmatched_primary=true", ] ) payload = json.loads(card.read_text(encoding="utf-8")) assert create_exit == 0 assert payload["backend"] == "line_align_composite" assert payload["backend_kwargs"]["primary_engine"] == "surya" assert payload["backend_kwargs"]["secondary_engine"] == "tesseract" assert payload["backend_kwargs"]["append_unmatched_primary"] is True def test_cli_calibrate_quality_router(tmp_path: Path, capsys) -> None: source = tmp_path / "sample.txt" source.write_text("नेपाल\n", encoding="utf-8") manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल", metadata={"slices": ["nepali"]})], manifest, ) benchmark_report = tmp_path / "report.json" benchmark_report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.2, "quality_score": 0.9}, "error": None, }, { "baseline": "tesseract", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.4, "quality_score": 0.7}, "error": None, }, ] ), encoding="utf-8", ) experiment_report = tmp_path / "experiment.json" experiment_report.write_text( json.dumps( { "experiment_id": "calibration", "output_dir": str(tmp_path / "exp"), "eval_manifest": str(manifest), "references_dir": None, "baselines": ["candidate", "tesseract"], "input_count": 1, "datasets": ["fixture"], "slice_counts": {"nepali": 1}, "benchmark_report": str(benchmark_report), "benchmark_status_counts": {"ok": 2}, "validation_report": None, "validation_status": None, "environment": {}, } ), encoding="utf-8", ) exit_code = main( [ "calibrate-quality-router", "--experiment-report", str(experiment_report), "--out", str(tmp_path / "calibration"), "--model-id", "ocrtech-quality-router", "--primary-engine", "tesseract", "--secondary-engine", "tesseract", "--primary-engine-kwarg", "psm=6", "--secondary-engine-kwarg", "psm=13", "--threshold-start", "0.0", "--threshold-end", "0.1", "--threshold-step", "0.05", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "calibration.json" in captured.out assert (tmp_path / "calibration" / "model-card.json").exists() def test_cli_calibrate_script_router(tmp_path: Path, capsys) -> None: source = tmp_path / "sample.txt" source.write_text("नेपाल\n", encoding="utf-8") manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल", metadata={"slices": ["nepali"]})], manifest, ) experiment_dir = tmp_path / "exp" (experiment_dir / "benchmark" / "tesseract" / source.stem).mkdir(parents=True) (experiment_dir / "benchmark" / "tesseract" / source.stem / "document.md").write_text("नेपाल सरकार\n", encoding="utf-8") benchmark_report = tmp_path / "report.json" benchmark_report.write_text( json.dumps( [ { "baseline": "tesseract", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.4}, "error": None, }, { "baseline": "surya", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.2}, "error": None, }, ] ), encoding="utf-8", ) experiment_report = tmp_path / "experiment.json" experiment_report.write_text( json.dumps( { "experiment_id": "script-calibration", "output_dir": str(experiment_dir), "eval_manifest": str(manifest), "references_dir": None, "baselines": ["surya", "tesseract"], "input_count": 1, "datasets": ["fixture"], "slice_counts": {"nepali": 1}, "benchmark_report": str(benchmark_report), "benchmark_status_counts": {"ok": 2}, "validation_report": None, "validation_status": None, "environment": {}, } ), encoding="utf-8", ) exit_code = main( [ "calibrate-script-router", "--experiment-report", str(experiment_report), "--out", str(tmp_path / "calibration"), "--model-id", "ocrtech-script-router", "--primary-engine", "tesseract", "--secondary-engine", "surya", "--threshold-start", "0.0", "--threshold-end", "0.2", "--threshold-step", "0.2", "--routing-granularity", "page", "--secondary-structure-backfill", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "calibration.json" in captured.out assert (tmp_path / "calibration" / "model-card.json").exists() def test_cli_calibrate_tesseract_psm_ensemble(tmp_path: Path, capsys) -> None: source = tmp_path / "sample.txt" source.write_text("नेपाल\n", encoding="utf-8") manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल", metadata={"slices": ["nepali"]})], manifest, ) def write_experiment(name: str, benchmark_rows: list[dict[str, object]]) -> Path: benchmark = tmp_path / f"{name}-report.json" benchmark.write_text(json.dumps(benchmark_rows), encoding="utf-8") experiment = tmp_path / f"{name}-experiment.json" experiment.write_text( json.dumps( { "experiment_id": name, "output_dir": str(tmp_path / name), "eval_manifest": str(manifest), "references_dir": None, "baselines": ["candidate", "tesseract"], "input_count": 1, "datasets": ["fixture"], "slice_counts": {"nepali": 1}, "benchmark_report": str(benchmark), "benchmark_status_counts": {"ok": len(benchmark_rows)}, "validation_report": None, "validation_status": None, "environment": {}, } ), encoding="utf-8", ) return experiment primary = write_experiment("primary", [{"baseline": "tesseract", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.4, "quality_score": 0.7}, "error": None}]) psm7 = write_experiment("psm7", [{"baseline": "candidate", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.3, "quality_score": 0.6}, "error": None}]) psm13 = write_experiment("psm13", [{"baseline": "candidate", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.2, "quality_score": 0.9}, "error": None}]) exit_code = main( [ "calibrate-tesseract-psm-ensemble", "--primary-experiment-report", str(primary), "--variant-experiment", f"psm7={psm7}", "--variant-experiment", f"psm13={psm13}", "--out", str(tmp_path / "ensemble"), "--model-id", "ocrtech-ensemble", "--bias-start", "-0.04", "--bias-end", "0.04", "--bias-step", "0.04", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "calibration.json" in captured.out assert (tmp_path / "ensemble" / "model-card.json").exists() def test_cli_export_and_package_recognizer(tmp_path: Path) -> None: training_config = tmp_path / "paddle_recognizer.yml" training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8") export_exit = main( [ "export-recognizer", "--training-config", str(training_config), "--checkpoint", str(tmp_path / "checkpoints" / "best_accuracy"), "--out", str(tmp_path / "export"), ] ) inference_dir = tmp_path / "inference" inference_dir.mkdir() (inference_dir / "inference.pdmodel").write_bytes(b"model") (inference_dir / "inference.pdiparams").write_bytes(b"params") dictionary = tmp_path / "dict.txt" dictionary.write_text("क\nA\n", encoding="utf-8") package_exit = main( [ "package-recognizer", "--inference-dir", str(inference_dir), "--dictionary", str(dictionary), "--out", str(tmp_path / "package"), "--model-id", "ocrtech-cli-package", ] ) assert export_exit == 0 assert package_exit == 0 assert (tmp_path / "export" / "export-recognizer.json").exists() assert (tmp_path / "package" / "model-card.json").exists()