Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_cli.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 57.8 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_cli.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_cli.py
-
curl -L -o test_cli.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_cli.py
57.8 kB
from __future__ import annotations
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from ocrtech.cli import main
from ocrtech.experiments import run_experiment
from ocrtech.models import write_model_card
from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest
def test_cli_prepare_data_with_skip_invalid(tmp_path: Path, capsys) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"line")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline.png,नेपाल\nline.png,\n", encoding="utf-8")
exit_code = main(
[
"prepare-data",
str(labels),
"--out",
str(tmp_path / "manifests"),
"--dataset-name",
"local",
"--skip-invalid",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "local-train.jsonl" in captured.out
assert "local-train-rejects.jsonl" in captured.out
assert "prepared=1 rejected=1" in captured.out
def test_cli_prepare_data_can_stamp_slices(tmp_path: Path, capsys) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"line")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8")
exit_code = main(
[
"prepare-data",
str(labels),
"--out",
str(tmp_path / "manifests"),
"--dataset-name",
"local",
"--slice",
"nepali",
"--slice",
"real",
]
)
entries = [json.loads(line) for line in (tmp_path / "manifests" / "local-train.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
captured = capsys.readouterr()
assert exit_code == 0
assert entries[0]["metadata"]["slices"] == ["nepali", "real"]
assert "local-train.jsonl" in captured.out
def test_cli_parse_doc_sidecar_text(tmp_path: Path, capsys) -> None:
source = tmp_path / "sample.txt"
source.write_text("नेपाल सरकार\n\nhello world\n", encoding="utf-8")
exit_code = main(["parse-doc", str(source), "--out", str(tmp_path / "out"), "--engine", "sidecar"])
captured = capsys.readouterr()
assert exit_code == 0
assert "document.json" in captured.out
assert (tmp_path / "out" / "document.md").exists()
def test_cli_parse_doc_passes_fallback_model_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
captured: dict[str, object] = {}
source = tmp_path / "sample.png"
source.write_bytes(b"image")
class DocumentStub:
pages: list[object] = []
tables: list[object] = []
figures: list[object] = []
def fake_parse_document(*args: object, **kwargs: object) -> DocumentStub:
captured["args"] = args
captured["kwargs"] = kwargs
return DocumentStub()
monkeypatch.setattr("ocrtech.cli.parse_document", fake_parse_document)
exit_code = main(
[
"parse-doc",
str(source),
"--out",
str(tmp_path / "out"),
"--engine",
"sidecar",
"--fallback-model-config",
str(tmp_path / "fallback.json"),
]
)
assert exit_code == 0
assert captured["kwargs"]["fallback_model_config"] == str(tmp_path / "fallback.json")
def test_cli_validate_claim_returns_success_only_when_validated(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "ours",
"input_path": "a.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.01},
"error": None,
},
{
"baseline": "tesseract",
"input_path": "a.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.10},
"error": None,
},
]
),
encoding="utf-8",
)
config = tmp_path / "config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 1,
"required_slices": {"nepali": 1},
"gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower", "slice": "nepali"}],
}
),
encoding="utf-8",
)
exit_code = main(["validate-claim", "--benchmark-report", str(report), "--config", str(config), "--out", str(tmp_path / "validation")])
assert exit_code == 0
assert (tmp_path / "validation" / "validation.json").exists()
def test_cli_recognizer_admission_decision_exit_codes(tmp_path: Path) -> None:
admitted = tmp_path / "admitted.json"
admitted.write_text('{"claim_status":"validated","passed":true,"sample_count":3}\n', encoding="utf-8")
rejected = tmp_path / "rejected.json"
rejected.write_text('{"claim_status":"not_validated","passed":false,"sample_count":3}\n', encoding="utf-8")
admitted_exit = main(
[
"recognizer-admission-decision",
"--validation-report",
str(admitted),
"--out",
str(tmp_path / "admitted-decision"),
]
)
rejected_exit = main(
[
"recognizer-admission-decision",
"--validation-report",
str(rejected),
"--out",
str(tmp_path / "rejected-decision"),
]
)
assert admitted_exit == 0
assert rejected_exit == 1
assert (tmp_path / "admitted-decision" / "recognizer-admission-decision.json").exists()
assert (tmp_path / "rejected-decision" / "recognizer-admission-decision.md").exists()
def test_cli_recognizer_failure_review_stops_family(tmp_path: Path) -> None:
args = ["recognizer-failure-review", "--out", str(tmp_path / "review")]
for index in range(3):
card = tmp_path / f"model-{index}.json"
card.write_text(
json.dumps(
{
"model_id": f"candidate-{index}",
"backend": "paddleocr",
"base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec",
"artifacts": [],
"backend_kwargs": {},
"metrics": {},
}
),
encoding="utf-8",
)
decision = tmp_path / f"decision-{index}.json"
decision.write_text(
json.dumps(
{
"decision": "reject",
"claim_status": "not_validated",
"passed": False,
"sample_count": 50,
"failed_gates": [
{
"baseline": "tesseract",
"metric": "cer",
"slices": ["nepali"],
"pairs": 50,
"win_count": 0,
"loss_count": 50,
"tie_count": 0,
}
],
}
),
encoding="utf-8",
)
args.extend(["--admission-decision", str(decision), "--model-card", str(card)])
exit_code = main(args)
review = json.loads((tmp_path / "review" / "recognizer-failure-review.json").read_text(encoding="utf-8"))
assert exit_code == 1
assert review["families"][0]["recommendation"] == "stop_family"
assert (tmp_path / "review" / "recognizer-failure-review.md").exists()
def test_cli_create_and_audit_eval_pack(tmp_path: Path) -> None:
pack_dir = tmp_path / "pack"
create_exit = main(["create-eval-pack", "--out", str(pack_dir), "--input-format", "text"])
manifest = pack_dir / "manifests" / "ocrtech-eval-pack.jsonl"
assert create_exit == 0
assert manifest.exists()
audit_exit = main(["audit-eval-pack", str(manifest), "--out", str(tmp_path / "audit")])
assert audit_exit == 0
assert (tmp_path / "audit" / "eval-pack-audit.json").exists()
def test_cli_audit_and_split_manifest(tmp_path: Path) -> None:
manifest = tmp_path / "combined.jsonl"
entries = []
for index in range(6):
image = tmp_path / f"line-{index}.png"
image.write_bytes(f"image-{index}".encode("utf-8"))
image_hash = sha256_file(image)
text = "नेपाल" if index % 2 == 0 else "English"
text_hash = sha256_text(text)
entries.append(
ManifestEntry(
sample_id=f"sample-{index}",
dataset="local",
split="all",
image_path=str(image),
text=text,
sha256=image_hash,
metadata={"slices": ["nepali" if index % 2 == 0 else "english"], "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
)
write_manifest(entries, manifest)
audit_exit = main(["audit-manifest", str(manifest), "--out", str(tmp_path / "audit"), "--require-slice", "nepali"])
split_exit = main(["split-manifest", str(manifest), "--out", str(tmp_path / "split"), "--eval-ratio", "0.33", "--seed", "1"])
assert audit_exit == 0
assert split_exit == 0
assert (tmp_path / "audit" / "manifest-audit.json").exists()
assert (tmp_path / "split" / "train.jsonl").exists()
assert (tmp_path / "split" / "eval.jsonl").exists()
def test_cli_merge_manifests(tmp_path: Path) -> None:
first = tmp_path / "first.jsonl"
second = tmp_path / "second.jsonl"
write_manifest(_entries(tmp_path, start=0, count=2), first)
write_manifest(_entries(tmp_path, start=2, count=2), second)
merge_exit = main(["merge-manifests", str(first), str(second), "--out", str(tmp_path / "combined.jsonl")])
assert merge_exit == 0
assert (tmp_path / "combined.jsonl").exists()
assert (tmp_path / "combined-merge.json").exists()
assert (tmp_path / "combined-merge.md").exists()
def test_cli_filter_manifest(tmp_path: Path, capsys) -> None:
manifest = tmp_path / "combined.jsonl"
write_manifest(_entries(tmp_path, start=0, count=4), manifest)
exit_code = main(
[
"filter-manifest",
str(manifest),
"--out",
str(tmp_path / "filtered.jsonl"),
"--sample-id",
"merge-sample-0",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "selected=1 source=4" in captured.out
assert (tmp_path / "filtered.jsonl").exists()
assert (tmp_path / "filtered-filter.json").exists()
assert (tmp_path / "filtered-filter.md").exists()
def test_cli_rebalance_manifest(tmp_path: Path, capsys) -> None:
manifest = tmp_path / "combined.jsonl"
write_manifest(_entries(tmp_path, start=0, count=5), manifest)
exit_code = main(
[
"rebalance-manifest",
str(manifest),
"--out",
str(tmp_path / "rebalanced.jsonl"),
"--slice",
"english",
"--slice",
"nepali",
"--target-count",
"4",
"--seed",
"7",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "rebalanced.jsonl" in captured.out
assert "rebalanced-rebalance.json" in captured.out
assert "original=5 rebalanced=8 target=4" in captured.out
assert (tmp_path / "rebalanced.jsonl").exists()
def test_cli_audit_recognizer_corpus(tmp_path: Path, capsys) -> None:
train = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
image_train_en = tmp_path / "train-en.png"
image_train_ne = tmp_path / "train-ne.png"
image_train_mix = tmp_path / "train-mix.png"
image_eval_mix = tmp_path / "eval-mix.png"
for path in [image_train_en, image_train_ne, image_train_mix, image_eval_mix]:
path.write_bytes(path.name.encode("utf-8"))
write_manifest(
[
ManifestEntry(sample_id="tr-en", dataset="fixture", split="train", image_path=str(image_train_en), text="Invoice", sha256=sha256_file(image_train_en), metadata={"slices": ["english", "real"]}),
ManifestEntry(sample_id="tr-ne", dataset="fixture", split="train", image_path=str(image_train_ne), text="नेपाल", sha256=sha256_file(image_train_ne), metadata={"slices": ["nepali", "synthetic"]}),
ManifestEntry(sample_id="tr-mix", dataset="fixture", split="train", image_path=str(image_train_mix), text="नाम Ram", sha256=sha256_file(image_train_mix), metadata={"slices": ["mixed_script", "real", "synthetic"]}),
],
train,
)
write_manifest(
[
ManifestEntry(sample_id="ev-mix", dataset="fixture", split="eval", image_path=str(image_eval_mix), text="बिल No", sha256=sha256_file(image_eval_mix), metadata={"slices": ["mixed_script", "hard_eval"]}),
],
eval_manifest,
)
exit_code = main(
[
"audit-recognizer-corpus",
"--train-manifest",
str(train),
"--eval-manifest",
str(eval_manifest),
"--out",
str(tmp_path / "audit"),
"--min-train-samples",
"3",
"--min-eval-samples",
"1",
"--min-train-english",
"2",
"--min-train-devanagari",
"2",
"--min-train-mixed",
"1",
"--min-eval-english",
"1",
"--min-eval-devanagari",
"1",
"--min-eval-mixed",
"1",
"--min-train-latin-only",
"1",
"--min-train-devanagari-only",
"1",
"--min-eval-latin-only",
"0",
"--min-eval-devanagari-only",
"0",
"--min-train-real",
"2",
"--min-train-synthetic",
"2",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "recognizer-corpus-audit.json" in captured.out
assert "passed=True train=3 eval=1" in captured.out
assert (tmp_path / "audit" / "recognizer-corpus-audit.md").exists()
def test_cli_build_recognizer_corpus(tmp_path: Path, capsys) -> None:
source = tmp_path / "source.jsonl"
images = {
"en": tmp_path / "en.png",
"ne": tmp_path / "ne.png",
"mix": tmp_path / "mix.png",
}
for path in images.values():
path.write_bytes(path.name.encode("utf-8"))
write_manifest(
[
ManifestEntry(sample_id="en", dataset="fixture", split="train", image_path=str(images["en"]), text="Invoice", sha256=sha256_file(images["en"]), metadata={"slices": ["english", "real"]}),
ManifestEntry(sample_id="ne", dataset="fixture", split="train", image_path=str(images["ne"]), text="नेपाल", sha256=sha256_file(images["ne"]), metadata={"slices": ["nepali", "real"]}),
ManifestEntry(sample_id="mix", dataset="fixture", split="train", image_path=str(images["mix"]), text="नाम Ram", sha256=sha256_file(images["mix"]), metadata={"slices": ["mixed_script", "synthetic"]}),
],
source,
)
exit_code = main(
[
"build-recognizer-corpus",
str(source),
"--out",
str(tmp_path / "built.jsonl"),
"--target-latin-only",
"2",
"--target-devanagari-only",
"2",
"--target-mixed",
"2",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "source=3" in captured.out
assert (tmp_path / "built.jsonl").exists()
assert (tmp_path / "built-recognizer-corpus-build.json").exists()
def test_cli_materialize_references(tmp_path: Path, capsys) -> None:
source = tmp_path / "sample.png"
source.write_bytes(b"png")
manifest = tmp_path / "eval.jsonl"
image_hash = sha256_file(source)
text_hash = sha256_text("hello world")
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=image_hash,
metadata={"text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
],
manifest,
)
exit_code = main(
[
"materialize-references",
str(manifest),
"--out",
str(tmp_path / "references"),
"--rewritten-manifest",
str(tmp_path / "eval-with-refs.jsonl"),
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "reference-pack.json" in captured.out
assert (tmp_path / "references" / "sample-1" / "document.md").exists()
def test_cli_normalize_manifest_images(tmp_path: Path, capsys, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "sample.img"
source.write_bytes(b"raw image bytes")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="hello world",
sha256=sha256_file(source),
)
],
manifest,
)
class FakeImage:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb) -> None:
return None
def save(self, target: Path, format: str) -> None:
assert format == "PNG"
target.write_bytes(b"converted png bytes")
monkeypatch.setitem(sys.modules, "PIL", SimpleNamespace(Image=SimpleNamespace(open=lambda path: FakeImage())))
exit_code = main(["normalize-manifest-images", str(manifest), "--out", str(tmp_path / "normalized")])
captured = capsys.readouterr()
assert exit_code == 0
assert "manifest-images-normalized.jsonl" in captured.out
assert "converted=1" in captured.out
entries = load_manifest(tmp_path / "normalized" / "manifest-images-normalized.jsonl")
assert Path(entries[0].image_path).suffix == ".png"
assert (tmp_path / "normalized" / "image-normalization.json").exists()
def test_cli_train_recognizer_hf_backend(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
monkeypatch.setattr("ocrtech.cli.write_hf_recognizer_recipe", lambda *args, **kwargs: tmp_path / "run" / "hf_recognizer_config.json")
exit_code = main(
[
"train-recognizer",
"--backend",
"hf-vision-encoder-decoder",
"--train-manifest",
"train.jsonl",
"--eval-manifest",
"eval.jsonl",
"--out",
str(tmp_path / "run"),
]
)
assert exit_code == 0
def test_cli_bundle_recognizer_training_passes_corpus_audit_options(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
captured: dict[str, object] = {}
class FakeBundle:
archive_path = str(tmp_path / "bundle.tar.gz")
manifest_path = str(tmp_path / "bundle.json")
readme_path = str(tmp_path / "bundle.md")
file_count = 3
total_bytes = 42
def fake_bundle(*args, **kwargs):
captured["args"] = args
captured["kwargs"] = kwargs
return FakeBundle()
monkeypatch.setattr("ocrtech.cli.bundle_recognizer_training", fake_bundle)
exit_code = main(
[
"bundle-recognizer-training",
"--recipe-dir",
"runs/recipe",
"--train-manifest",
"data/train.jsonl",
"--eval-manifest",
"data/eval.jsonl",
"--out",
str(tmp_path / "bundle"),
"--require-corpus-audit",
"--min-train-samples",
"11",
"--min-eval-samples",
"7",
"--min-train-english",
"5",
"--min-train-latin-only",
"3",
"--failure-review",
str(tmp_path / "review.json"),
]
)
assert exit_code == 0
kwargs = captured["kwargs"]
assert isinstance(kwargs, dict)
assert kwargs["require_corpus_audit"] is True
assert kwargs["min_train_samples"] == 11
assert kwargs["min_eval_samples"] == 7
assert kwargs["min_train_english"] == 5
assert kwargs["min_train_latin_only"] == 3
assert kwargs["failure_review"] == str(tmp_path / "review.json")
def test_cli_generate_correction_pairs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
output = tmp_path / "pairs.jsonl"
monkeypatch.setattr("ocrtech.cli.generate_correction_pairs", lambda *args, **kwargs: output)
exit_code = main(
[
"generate-correction-pairs",
"--manifest",
"train.jsonl",
"--out",
str(output),
"--engine",
"tesseract",
]
)
assert exit_code == 0
def test_cli_train_corrector_hf_backend(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
def fake_recipe(*args, **kwargs):
pairs = tmp_path / "corrector" / "correction_pairs.jsonl"
pairs.parent.mkdir(parents=True, exist_ok=True)
pairs.write_text('{"noisy_text":"गलत","clean_text":"सही"}\n{"noisy_text":"A","clean_text":"A"}\n', encoding="utf-8")
return tmp_path / "corrector" / "corrector_config.json"
monkeypatch.setattr("ocrtech.cli.write_corrector_recipe", fake_recipe)
monkeypatch.setattr("ocrtech.cli.write_hf_text_corrector_recipe", lambda *args, **kwargs: tmp_path / "corrector" / "hf_text_corrector_config.json")
exit_code = main(
[
"train-corrector",
"--pairs-source",
"pairs.jsonl",
"--out",
str(tmp_path / "corrector"),
"--backend",
"hf-seq2seq",
]
)
assert exit_code == 0
def test_cli_run_experiment_without_validation(tmp_path: Path) -> None:
pack_dir = tmp_path / "pack"
create_exit = main(["create-eval-pack", "--out", str(pack_dir), "--input-format", "text"])
manifest = pack_dir / "manifests" / "ocrtech-eval-pack.jsonl"
experiment_exit = main(
[
"run-experiment",
"--eval-manifest",
str(manifest),
"--out",
str(tmp_path / "experiment"),
"--baselines",
"ours",
"--skip-validation",
]
)
assert create_exit == 0
assert experiment_exit == 0
assert (tmp_path / "experiment" / "experiment.json").exists()
assert (tmp_path / "experiment" / "benchmark" / "report.json").exists()
def test_cli_preflight_claim_with_smoke_pack(tmp_path: Path) -> None:
pack_dir = tmp_path / "pack"
create_exit = main(["create-eval-pack", "--out", str(pack_dir), "--input-format", "text"])
manifest = pack_dir / "manifests" / "ocrtech-eval-pack.jsonl"
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-cli-fallback", backend="sidecar", base_model="fixture")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-cli-sidecar",
backend="sidecar",
base_model="fixture",
fallback_model_card=fallback,
)
preflight_exit = main(
[
"preflight-claim",
"--eval-manifest",
str(manifest),
"--references-dir",
str(pack_dir / "references"),
"--baselines",
"ours",
"--candidate-model-config",
str(card),
"--fallback-model-config",
str(fallback),
"--allow-smoke-eval-pack",
"--out",
str(tmp_path / "preflight"),
]
)
assert create_exit == 0
assert preflight_exit == 0
assert (tmp_path / "preflight" / "preflight.json").exists()
payload = json.loads((tmp_path / "preflight" / "preflight.json").read_text(encoding="utf-8"))
fallback_check = next(check for check in payload["checks"] if check["name"] == "fallback-model")
assert fallback_check["status"] == "pass"
provenance_check = next(check for check in payload["checks"] if check["name"] == "candidate-fallback-provenance")
assert provenance_check["status"] == "pass"
def test_cli_review_claim(tmp_path: Path) -> None:
model = write_model_card(tmp_path / "model.json", model_id="ocrtech-cli-sidecar", backend="sidecar", base_model="fixture")
source = tmp_path / "sample.txt"
source.write_text("hello world\n", encoding="utf-8")
source.with_suffix(".ref.txt").write_text("hello world\n", encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
manifest.write_text(
'{"sample_id":"s1","dataset":"ocrtech-eval-pack","split":"eval","image_path":"'
+ str(source)
+ '","text":"hello world","sha256":null,"metadata":{"slices":["english"],"reference_path":"'
+ str(source.with_suffix(".ref.txt"))
+ '"}}\n',
encoding="utf-8",
)
validation_config = tmp_path / "validation.json"
validation_config.write_text(
json.dumps(
{
"candidate": "candidate",
"min_samples": 1,
"required_slices": {"english": 1},
"gates": [{"baseline": "candidate", "metric": "quality_score", "direction": "higher", "require_ci_above_zero": False}],
}
),
encoding="utf-8",
)
experiment = run_experiment(
manifest,
tmp_path / "experiment",
baselines=["candidate"],
candidate_model_config=model,
run_validation=True,
validation_config=validation_config,
)
review_config = tmp_path / "claim-review.json"
review_config.write_text(
json.dumps(
{
"required_experiments": [
{
"name": "internal-pack",
"datasets": ["ocrtech-eval-pack"],
"require_validation_status": "validated",
"require_baselines": ["candidate"],
"required_slices": ["english"],
}
]
}
),
encoding="utf-8",
)
review_exit = main(
[
"review-claim",
"--experiment",
str(Path(experiment.output_dir) / "experiment.json"),
"--config",
str(review_config),
"--model-config",
str(model),
"--out",
str(tmp_path / "review"),
]
)
assert review_exit == 0
assert (tmp_path / "review" / "claim-review.json").exists()
def test_cli_audit_remote_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
captured: dict[str, object] = {}
def fake_audit_remote_host(host, out, **kwargs):
captured.update(kwargs)
return type(
"RemoteAuditResult",
(),
{"passed": True, "host": host},
)()
monkeypatch.setattr(
"ocrtech.cli.audit_remote_host",
fake_audit_remote_host,
)
exit_code = main(
[
"audit-remote-host",
"--host",
"100.97.238.103",
"--user",
"k2",
"--require-paddle-training",
"--out",
str(tmp_path / "remote"),
]
)
assert exit_code == 0
assert captured["require_paddle_training"] is True
def test_cli_bootstrap_remote_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
monkeypatch.setattr(
"ocrtech.cli.bootstrap_remote_host",
lambda host, out, **kwargs: type(
"RemoteBootstrapResult",
(),
{"passed": True, "workdir": "/Users/k2/ocr-tech", "venv_dir": "/Users/k2/ocr-tech/.venv"},
)(),
)
exit_code = main(
[
"bootstrap-remote-host",
"--host",
"100.97.238.103",
"--user",
"k2",
"--workdir",
"~/ocr-tech",
"--recreate-venv",
"--out",
str(tmp_path / "bootstrap"),
]
)
assert exit_code == 0
def test_cli_sync_remote_workspace(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
monkeypatch.setattr(
"ocrtech.cli.sync_remote_workspace",
lambda local_root, out, **kwargs: type(
"RemoteSyncResult",
(),
{"passed": True, "remote_workdir": "/Users/k2/ocr-tech"},
)(),
)
exit_code = main(
[
"sync-remote-workspace",
"--host",
"100.97.238.103",
"--user",
"k2",
"--remote-workdir",
"~/ocr-tech",
"--out",
str(tmp_path / "sync"),
]
)
assert exit_code == 0
def test_cli_package_hf_recognizer(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
captured: dict[str, object] = {}
def fake_package_hf_recognizer_model(*args: object, **kwargs: object) -> Path:
captured["args"] = args
captured["kwargs"] = kwargs
return tmp_path / "package" / "model-card.json"
monkeypatch.setattr("ocrtech.cli.package_hf_recognizer_model", fake_package_hf_recognizer_model)
exit_code = main(
[
"package-hf-recognizer",
"--model-dir",
"runs/hf/final-model",
"--out",
str(tmp_path / "package"),
"--model-id",
"ocrtech-hf",
"--base-model",
"microsoft/trocr-small-printed",
"--admission-validation-report",
"outputs/validation-next-recognizer-hard50/validation.json",
"--training-summary",
"runs/hf/recognizer-training-summary.json",
]
)
assert exit_code == 0
assert captured["kwargs"]["admission_validation_report"] == "outputs/validation-next-recognizer-hard50/validation.json"
assert captured["kwargs"]["training_summary"] == "runs/hf/recognizer-training-summary.json"
def _entries(tmp_path: Path, *, start: int, count: int) -> list[ManifestEntry]:
entries: list[ManifestEntry] = []
for index in range(start, start + count):
image = tmp_path / f"merge-line-{index}.png"
image.write_bytes(f"merge-image-{index}".encode("utf-8"))
image_hash = sha256_file(image)
text = "नेपाल" if index % 2 == 0 else "English"
text_hash = sha256_text(text)
entries.append(
ManifestEntry(
sample_id=f"merge-sample-{index}",
dataset="local",
split="all",
image_path=str(image),
text=text,
sha256=image_hash,
metadata={"slices": ["nepali" if index % 2 == 0 else "english"], "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
)
return entries
def test_cli_parse_python_version_rejects_invalid_format() -> None:
exit_code = main(["audit-remote-host", "--host", "100.97.238.103", "--min-python", "3"])
assert exit_code == 2
def test_cli_create_and_audit_model_card(tmp_path: Path) -> None:
artifact = tmp_path / "model.pdmodel"
artifact.write_bytes(b"model")
card = tmp_path / "model.json"
create_exit = main(
[
"create-model-card",
"--out",
str(card),
"--model-id",
"ocrtech-cli",
"--backend",
"sidecar",
"--base-model",
"fixture",
"--artifact",
str(artifact),
]
)
audit_exit = main(["audit-model", str(card), "--out", str(tmp_path / "audit")])
assert create_exit == 0
assert audit_exit == 0
assert (tmp_path / "audit" / "model-audit.json").exists()
def test_cli_create_tesseract_model_card_with_backend_kwargs(tmp_path: Path) -> None:
card = tmp_path / "model.json"
create_exit = main(
[
"create-model-card",
"--out",
str(card),
"--model-id",
"ocrtech-tess-cli",
"--backend",
"tesseract",
"--base-model",
"tesseract-ocr",
"--backend-kwarg",
"language=nep+eng",
"--backend-kwarg",
"psm=7",
]
)
payload = json.loads(card.read_text(encoding="utf-8"))
assert create_exit == 0
assert payload["backend"] == "tesseract"
assert payload["backend_kwargs"] == {"language": "nep+eng", "psm": 7}
def test_cli_create_model_card_records_fallback_model_card(tmp_path: Path) -> None:
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
card = tmp_path / "model.json"
create_exit = main(
[
"create-model-card",
"--out",
str(card),
"--model-id",
"ocrtech-hybrid-cli",
"--backend",
"sidecar",
"--base-model",
"fixture",
"--fallback-model-card",
str(fallback),
]
)
payload = json.loads(card.read_text(encoding="utf-8"))
assert create_exit == 0
assert payload["provenance"]["fallback_model_card"] == {
"path": str(fallback),
"sha256": sha256_file(fallback),
}
def test_cli_summarize_benchmark(tmp_path: Path, capsys) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": "a.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["english"],
"metrics": {"cer": 0.10},
"error": None,
},
{
"baseline": "tesseract",
"input_path": "a.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["english"],
"metrics": {"cer": 0.20},
"error": None,
},
]
),
encoding="utf-8",
)
exit_code = main(
[
"summarize-benchmark",
"--benchmark-report",
str(report),
"--out",
str(tmp_path / "summary"),
"--candidate",
"candidate",
"--baseline",
"tesseract",
"--metric",
"cer",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "summary.json" in captured.out
assert (tmp_path / "summary" / "summary.json").exists()
assert (tmp_path / "summary" / "summary.md").exists()
def test_cli_benchmark_loads_inputs_from_manifest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys) -> None:
image = tmp_path / "image.png"
image.write_bytes(b"image")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="s1",
dataset="eval",
split="eval",
image_path="image.png",
text="नेपाल",
sha256=sha256_file(image),
metadata={"reference_path": "refs/s1.json"},
)
],
manifest,
)
captured_args: dict[str, object] = {}
def fake_run_benchmark(inputs, output_dir, **kwargs):
captured_args["inputs"] = inputs
captured_args["output_dir"] = output_dir
captured_args.update(kwargs)
return []
monkeypatch.setattr("ocrtech.cli.run_benchmark", fake_run_benchmark)
exit_code = main(
[
"benchmark",
"--inputs-from-manifest",
str(manifest),
"--out",
str(tmp_path / "bench"),
"--baselines",
"candidate,tesseract",
"--fallback-model-config",
str(tmp_path / "fallback.json"),
"--low-confidence-threshold",
"0.71",
]
)
stdout = capsys.readouterr().out
assert exit_code == 0
assert captured_args["inputs"] == [str(image)]
assert captured_args["eval_manifest"] == str(manifest)
assert captured_args["baselines"] == ["candidate", "tesseract"]
assert captured_args["fallback_model_config"] == str(tmp_path / "fallback.json")
assert captured_args["low_confidence_threshold"] == 0.71
assert "runs=0" in stdout
def test_cli_paper_benchmark_table(tmp_path: Path, capsys) -> None:
first = tmp_path / "first-summary.json"
second = tmp_path / "second-summary.json"
first.write_text(
json.dumps(
{
"report_path": "outputs/first/report.json",
"candidate": "candidate",
"baselines": ["tesseract"],
"metrics": ["cer"],
"aggregate_metrics": {
"candidate": {"all": {"sample_count": 5, "cer": 0.22, "table_cell_f1": 1.0, "latency_seconds": 4.2}},
"tesseract": {"all": {"sample_count": 5, "cer": 0.73, "table_cell_f1": 0.0, "latency_seconds": 0.2}},
},
"paired_metrics": [],
}
),
encoding="utf-8",
)
second.write_text(
json.dumps(
{
"report_path": "outputs/second/report.json",
"candidate": "candidate",
"baselines": ["surya"],
"metrics": ["cer"],
"aggregate_metrics": {
"candidate": {"all": {"sample_count": 5, "cer": 0.22, "reading_order_pair_accuracy": 0.78, "latency_seconds": 4.2}},
"surya": {"all": {"sample_count": 5, "cer": 0.19, "reading_order_pair_accuracy": 0.75, "latency_seconds": 3.8}},
},
"paired_metrics": [],
}
),
encoding="utf-8",
)
exit_code = main(
[
"paper-benchmark-table",
"--summary",
str(first),
"--summary",
str(second),
"--out",
str(tmp_path / "paper-table"),
"--system",
"candidate",
"--system",
"surya",
"--system",
"tesseract",
"--label",
"candidate=Candidate",
"--source-preference",
"candidate=" + str(first),
"--metric",
"cer",
"--metric",
"table_cell_f1",
"--metric",
"reading_order_pair_accuracy",
"--metric",
"latency_seconds",
"--title",
"Blur Comparison",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "paper-table.json" in captured.out
assert (tmp_path / "paper-table" / "paper-table.json").exists()
assert (tmp_path / "paper-table" / "paper-table.md").exists()
def test_cli_benchmark_oracle(tmp_path: Path, capsys) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "surya",
"input_path": "s1.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["english"],
"metrics": {"cer": 0.10},
"error": None,
},
{
"baseline": "tesseract",
"input_path": "s1.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["english"],
"metrics": {"cer": 0.20},
"error": None,
},
]
),
encoding="utf-8",
)
exit_code = main(
[
"benchmark-oracle",
"--benchmark-report",
str(report),
"--out",
str(tmp_path / "oracle"),
"--system",
"surya",
"--system",
"tesseract",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "oracle-analysis.json" in captured.out
assert (tmp_path / "oracle" / "oracle-analysis.json").exists()
assert (tmp_path / "oracle" / "oracle-choices.json").exists()
assert (tmp_path / "oracle" / "oracle-summary" / "summary.json").exists()
def test_cli_benchmark_error_analysis(tmp_path: Path, capsys) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "surya",
"input_path": "s1.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.30},
"error": None,
},
{
"baseline": "tesseract",
"input_path": "s1.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.20},
"error": None,
},
]
),
encoding="utf-8",
)
exit_code = main(
[
"benchmark-error-analysis",
"--benchmark-report",
str(report),
"--out",
str(tmp_path / "analysis"),
"--system",
"surya",
"--baseline",
"tesseract",
"--top-n",
"1",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "error-analysis.json" in captured.out
assert (tmp_path / "analysis" / "error-analysis.json").exists()
assert (tmp_path / "analysis" / "top-failures.json").exists()
def test_cli_export_failure_manifest(tmp_path: Path, capsys) -> None:
image = tmp_path / "sample.png"
image.write_bytes(b"image")
manifest = tmp_path / "source.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="s1",
dataset="fixture",
split="eval",
image_path=str(image),
text="नेपाल",
metadata={"slices": ["figure"]},
)
],
manifest,
)
failures = tmp_path / "top-failures.json"
failures.write_text(json.dumps([{"sample_id": "s1", "cer": 0.5, "badness": 0.5, "worst_gap": None, "comparisons": {}}]), encoding="utf-8")
exit_code = main(
[
"export-failure-manifest",
"--source-manifest",
str(manifest),
"--top-failures",
str(failures),
"--out",
str(tmp_path / "failure.jsonl"),
"--split",
"hard_eval",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "selected=1" in captured.out
entries = load_manifest(tmp_path / "failure.jsonl")
assert entries[0].split == "hard_eval"
assert entries[0].metadata["failure_analysis"]["metric_value"] == 0.5
def test_cli_create_quality_select_model_card(tmp_path: Path) -> None:
card = tmp_path / "model.json"
create_exit = main(
[
"create-model-card",
"--out",
str(card),
"--model-id",
"ocrtech-quality-router",
"--backend",
"quality_select_composite",
"--base-model",
"tesseract-ocr",
"--backend-kwarg",
"primary_engine=tesseract",
"--backend-kwarg",
"secondary_engine=tesseract",
"--backend-kwarg",
'primary_engine_kwargs={"language":"nep+eng","psm":6}',
"--backend-kwarg",
'secondary_engine_kwargs={"language":"nep+eng","psm":13}',
"--backend-kwarg",
"secondary_quality_margin=0.02",
]
)
payload = json.loads(card.read_text(encoding="utf-8"))
assert create_exit == 0
assert payload["backend"] == "quality_select_composite"
assert payload["backend_kwargs"]["secondary_quality_margin"] == 0.02
assert payload["backend_kwargs"]["secondary_engine_kwargs"]["psm"] == 13
def test_cli_create_script_select_model_card(tmp_path: Path) -> None:
card = tmp_path / "model.json"
create_exit = main(
[
"create-model-card",
"--out",
str(card),
"--model-id",
"ocrtech-script-router",
"--backend",
"script_select_composite",
"--base-model",
"tesseract+surya",
"--backend-kwarg",
"primary_engine=tesseract",
"--backend-kwarg",
"secondary_engine=surya",
"--backend-kwarg",
"primary_script_threshold=0.2",
]
)
payload = json.loads(card.read_text(encoding="utf-8"))
assert create_exit == 0
assert payload["backend"] == "script_select_composite"
assert payload["backend_kwargs"]["primary_script_threshold"] == 0.2
def test_cli_create_line_align_model_card(tmp_path: Path) -> None:
card = tmp_path / "model.json"
create_exit = main(
[
"create-model-card",
"--out",
str(card),
"--model-id",
"ocrtech-line-align",
"--backend",
"line_align_composite",
"--base-model",
"surya+tesseract",
"--backend-kwarg",
"primary_engine=surya",
"--backend-kwarg",
"secondary_engine=tesseract",
"--backend-kwarg",
"append_unmatched_primary=true",
]
)
payload = json.loads(card.read_text(encoding="utf-8"))
assert create_exit == 0
assert payload["backend"] == "line_align_composite"
assert payload["backend_kwargs"]["primary_engine"] == "surya"
assert payload["backend_kwargs"]["secondary_engine"] == "tesseract"
assert payload["backend_kwargs"]["append_unmatched_primary"] is True
def test_cli_calibrate_quality_router(tmp_path: Path, capsys) -> None:
source = tmp_path / "sample.txt"
source.write_text("नेपाल\n", encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल", metadata={"slices": ["nepali"]})],
manifest,
)
benchmark_report = tmp_path / "report.json"
benchmark_report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": str(source),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.2, "quality_score": 0.9},
"error": None,
},
{
"baseline": "tesseract",
"input_path": str(source),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.4, "quality_score": 0.7},
"error": None,
},
]
),
encoding="utf-8",
)
experiment_report = tmp_path / "experiment.json"
experiment_report.write_text(
json.dumps(
{
"experiment_id": "calibration",
"output_dir": str(tmp_path / "exp"),
"eval_manifest": str(manifest),
"references_dir": None,
"baselines": ["candidate", "tesseract"],
"input_count": 1,
"datasets": ["fixture"],
"slice_counts": {"nepali": 1},
"benchmark_report": str(benchmark_report),
"benchmark_status_counts": {"ok": 2},
"validation_report": None,
"validation_status": None,
"environment": {},
}
),
encoding="utf-8",
)
exit_code = main(
[
"calibrate-quality-router",
"--experiment-report",
str(experiment_report),
"--out",
str(tmp_path / "calibration"),
"--model-id",
"ocrtech-quality-router",
"--primary-engine",
"tesseract",
"--secondary-engine",
"tesseract",
"--primary-engine-kwarg",
"psm=6",
"--secondary-engine-kwarg",
"psm=13",
"--threshold-start",
"0.0",
"--threshold-end",
"0.1",
"--threshold-step",
"0.05",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "calibration.json" in captured.out
assert (tmp_path / "calibration" / "model-card.json").exists()
def test_cli_calibrate_script_router(tmp_path: Path, capsys) -> None:
source = tmp_path / "sample.txt"
source.write_text("नेपाल\n", encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल", metadata={"slices": ["nepali"]})],
manifest,
)
experiment_dir = tmp_path / "exp"
(experiment_dir / "benchmark" / "tesseract" / source.stem).mkdir(parents=True)
(experiment_dir / "benchmark" / "tesseract" / source.stem / "document.md").write_text("नेपाल सरकार\n", encoding="utf-8")
benchmark_report = tmp_path / "report.json"
benchmark_report.write_text(
json.dumps(
[
{
"baseline": "tesseract",
"input_path": str(source),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.4},
"error": None,
},
{
"baseline": "surya",
"input_path": str(source),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.2},
"error": None,
},
]
),
encoding="utf-8",
)
experiment_report = tmp_path / "experiment.json"
experiment_report.write_text(
json.dumps(
{
"experiment_id": "script-calibration",
"output_dir": str(experiment_dir),
"eval_manifest": str(manifest),
"references_dir": None,
"baselines": ["surya", "tesseract"],
"input_count": 1,
"datasets": ["fixture"],
"slice_counts": {"nepali": 1},
"benchmark_report": str(benchmark_report),
"benchmark_status_counts": {"ok": 2},
"validation_report": None,
"validation_status": None,
"environment": {},
}
),
encoding="utf-8",
)
exit_code = main(
[
"calibrate-script-router",
"--experiment-report",
str(experiment_report),
"--out",
str(tmp_path / "calibration"),
"--model-id",
"ocrtech-script-router",
"--primary-engine",
"tesseract",
"--secondary-engine",
"surya",
"--threshold-start",
"0.0",
"--threshold-end",
"0.2",
"--threshold-step",
"0.2",
"--routing-granularity",
"page",
"--secondary-structure-backfill",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "calibration.json" in captured.out
assert (tmp_path / "calibration" / "model-card.json").exists()
def test_cli_calibrate_tesseract_psm_ensemble(tmp_path: Path, capsys) -> None:
source = tmp_path / "sample.txt"
source.write_text("नेपाल\n", encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल", metadata={"slices": ["nepali"]})],
manifest,
)
def write_experiment(name: str, benchmark_rows: list[dict[str, object]]) -> Path:
benchmark = tmp_path / f"{name}-report.json"
benchmark.write_text(json.dumps(benchmark_rows), encoding="utf-8")
experiment = tmp_path / f"{name}-experiment.json"
experiment.write_text(
json.dumps(
{
"experiment_id": name,
"output_dir": str(tmp_path / name),
"eval_manifest": str(manifest),
"references_dir": None,
"baselines": ["candidate", "tesseract"],
"input_count": 1,
"datasets": ["fixture"],
"slice_counts": {"nepali": 1},
"benchmark_report": str(benchmark),
"benchmark_status_counts": {"ok": len(benchmark_rows)},
"validation_report": None,
"validation_status": None,
"environment": {},
}
),
encoding="utf-8",
)
return experiment
primary = write_experiment("primary", [{"baseline": "tesseract", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.4, "quality_score": 0.7}, "error": None}])
psm7 = write_experiment("psm7", [{"baseline": "candidate", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.3, "quality_score": 0.6}, "error": None}])
psm13 = write_experiment("psm13", [{"baseline": "candidate", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.2, "quality_score": 0.9}, "error": None}])
exit_code = main(
[
"calibrate-tesseract-psm-ensemble",
"--primary-experiment-report",
str(primary),
"--variant-experiment",
f"psm7={psm7}",
"--variant-experiment",
f"psm13={psm13}",
"--out",
str(tmp_path / "ensemble"),
"--model-id",
"ocrtech-ensemble",
"--bias-start",
"-0.04",
"--bias-end",
"0.04",
"--bias-step",
"0.04",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "calibration.json" in captured.out
assert (tmp_path / "ensemble" / "model-card.json").exists()
def test_cli_export_and_package_recognizer(tmp_path: Path) -> None:
training_config = tmp_path / "paddle_recognizer.yml"
training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8")
export_exit = main(
[
"export-recognizer",
"--training-config",
str(training_config),
"--checkpoint",
str(tmp_path / "checkpoints" / "best_accuracy"),
"--out",
str(tmp_path / "export"),
]
)
inference_dir = tmp_path / "inference"
inference_dir.mkdir()
(inference_dir / "inference.pdmodel").write_bytes(b"model")
(inference_dir / "inference.pdiparams").write_bytes(b"params")
dictionary = tmp_path / "dict.txt"
dictionary.write_text("क\nA\n", encoding="utf-8")
package_exit = main(
[
"package-recognizer",
"--inference-dir",
str(inference_dir),
"--dictionary",
str(dictionary),
"--out",
str(tmp_path / "package"),
"--model-id",
"ocrtech-cli-package",
]
)
assert export_exit == 0
assert package_exit == 0
assert (tmp_path / "export" / "export-recognizer.json").exists()
assert (tmp_path / "package" / "model-card.json").exists()
from __future__ import annotations
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from ocrtech.cli import main
from ocrtech.experiments import run_experiment
from ocrtech.models import write_model_card
from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest
def test_cli_prepare_data_with_skip_invalid(tmp_path: Path, capsys) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"line")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline.png,नेपाल\nline.png,\n", encoding="utf-8")
exit_code = main(
[
"prepare-data",
str(labels),
"--out",
str(tmp_path / "manifests"),
"--dataset-name",
"local",
"--skip-invalid",
]
)
captured = capsys.readouterr()
assert exit_code == 0
assert "local-train.jsonl" in captured.out
assert "local-train-rejects.jsonl" in captured.out
assert "prepared=1 rejected=1" in captured.out
def test_cli_prepare_data_can_stamp_slices(tmp_path: Path, capsys) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"line")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8")
exit_code = main(
[
"prepare-data",
str(labels),
"--out",
str(tmp_path / "manifests"),
"--dataset-name",
"local",
"--slice",
"nepali",
"--slice",
"real",
]
)
entries = [json.loads(line) for line in (tmp_path / "manifests" / "local-train.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
captured = capsys.readouterr()
assert exit_code == 0
assert entries[0]["metadata"]["slices"] == ["nepali", "real"]
assert "local-train.jsonl" in captured.out
def test_cli_parse_doc_sidecar_text(tmp_path: Path, capsys) -> None:
source = tmp_path / "sample.txt"
source.write_text("नेपाल सरकार\n\nhello world\n", encoding="utf-8")
exit_code = main(["parse-doc", str(source), "--out", str(tmp_path / "out"), "--engine", "sidecar"])
captured = capsys.readouterr()
assert exit_code == 0
assert "document.json" in captured.out
assert (tmp_path / "out" / "document.md").exists()
def test_cli_parse_doc_passes_fallback_model_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
captured: dict[str, object] = {}
source = tmp_path / "sample.png"
source.write_bytes(b"image")
class DocumentStub:
pages: list[object] = []
tables: list[object] = []
figures: list[object] = []
def fake_parse_document(*args: object, **kwargs: object) -> DocumentStub:
captured["args"] = args
captured["kwargs"] = kwargs
return DocumentStub()
monkeypatch.setattr("ocrtech.cli.parse_document", fake_parse_document)
exit_code = main(
[
"parse-doc",