from __future__ import annotations import json import multiprocessing import os import signal import sys import time from pathlib import Path from types import SimpleNamespace import pytest from ocrtech.benchmark import derive_table_postprocess_benchmark, rescore_benchmark_report, run_benchmark, summarize_benchmark_report from ocrtech.calibration import calibrate_quality_router, calibrate_script_router, calibrate_tesseract_psm_ensemble from ocrtech.errors import DataValidationError, ValidationError from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest from ocrtech.models import ModelCard, audit_model_card, model_admission_status from ocrtech.schemas import BBox, Document, Page, Table, TableCell, TextLine from ocrtech.training import ( _hf_text_corrector_metrics_factory, bundle_recognizer_training, finalize_hf_recognizer_run, generate_correction_pairs, package_hf_text_corrector_model, package_hf_recognizer_model, package_paddleocr_model, run_hf_recognizer_training, write_corrector_recipe, write_hf_text_corrector_recipe, write_hf_recognizer_recipe, write_recognizer_export_recipe, write_recognizer_recipe, ) def test_train_recognizer_writes_paddle_recipe(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल English", metadata={"slices": ["nepali", "english"]}, ) ], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")], eval_manifest, ) config = write_recognizer_recipe( train_manifest, eval_manifest, tmp_path / "run", train_batch_size=4, eval_batch_size=6, learning_rate=0.0003, epochs=2.5, ) assert config.exists() assert (tmp_path / "run" / "labels" / "train.txt").exists() config_text = config.read_text(encoding="utf-8") assert "Architecture:" in config_text assert "algorithm: SVTR_LCNet" in config_text assert "Loss:" in config_text assert "PostProcess:" in config_text assert "Metric:" in config_text assert "character_dict_path" in config_text assert "batch_size_per_card: 4" in config_text assert "batch_size_per_card: 6" in config_text assert "name: Cosine" in config_text assert "learning_rate: 0.0003" in config_text assert "epoch_num: 2.5" in config_text dictionary = (tmp_path / "run" / "nep_eng_dict.txt").read_text(encoding="utf-8") assert "क" in dictionary assert "E" in dictionary summary = json.loads((tmp_path / "run" / "recognizer-training-summary.json").read_text(encoding="utf-8")) assert summary["backend"] == "paddleocr" assert summary["manifests"]["train"]["sample_count"] == 1 assert summary["manifests"]["train"]["slice_counts"] == {"english": 1, "nepali": 1} assert summary["manifests"]["train"]["contains_ascii"] is True assert summary["manifests"]["train"]["contains_devanagari"] is True assert summary["artifacts"]["dictionary"]["contains_ascii"] is True assert summary["artifacts"]["dictionary"]["contains_devanagari"] is True assert summary["artifacts"]["config"]["sha256"] def test_train_recognizer_writes_integral_epochs_as_integer(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="नेपाल")], eval_manifest, ) config = write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", epochs=20.0) config_text = config.read_text(encoding="utf-8") assert "epoch_num: 20\n" in config_text assert "epoch_num: 20.0" not in config_text def test_train_recognizer_accepts_custom_dictionary_covering_labels(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल A")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="क B")], eval_manifest, ) dictionary = tmp_path / "dict.txt" dictionary.write_text("\n".join(["न", "े", "प", "ा", "ल", " ", "A", "क", "B"]) + "\n", encoding="utf-8") config = write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", dictionary_path=dictionary) assert config.exists() summary = json.loads((tmp_path / "run" / "recognizer-training-summary.json").read_text(encoding="utf-8")) assert summary["artifacts"]["dictionary"]["path"] == str(dictionary) def test_train_recognizer_rejects_custom_dictionary_missing_label_characters(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल A")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="क B")], eval_manifest, ) dictionary = tmp_path / "dict.txt" dictionary.write_text("\n".join(["न", "े", "प", "ा", "ल", " ", "क"]) + "\n", encoding="utf-8") with pytest.raises(DataValidationError, match="dictionary_path is missing label characters"): write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", dictionary_path=dictionary) def test_train_recognizer_writes_low_learning_rate_as_yaml_float(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="नेपाल")], eval_manifest, ) config = write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", learning_rate=0.00005) config_text = config.read_text(encoding="utf-8") assert "learning_rate: 0.00005" in config_text assert "learning_rate: 5e-05" not in config_text def test_train_recognizer_rejects_stopped_family_when_failure_review_supplied(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="नेपाल")], eval_manifest, ) with pytest.raises(ValidationError, match="recognizer family is stopped"): write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", failure_review=_stopped_ppocrv3_failure_review(tmp_path)) def test_train_recognizer_paddle_labels_are_one_line_per_sample(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="पहिलो\tभाग\nदोस्रो भाग", ) ], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")], eval_manifest, ) write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run") train_lines = (tmp_path / "run" / "labels" / "train.txt").read_text(encoding="utf-8").splitlines() assert len(train_lines) == 1 assert train_lines[0].endswith("\tपहिलो भाग दोस्रो भाग") def test_bundle_recognizer_training_packages_recipe_manifests_and_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: train_image = tmp_path / "data" / "train.png" eval_image = tmp_path / "data" / "eval.png" train_image.parent.mkdir() train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "data" / "train.jsonl" eval_manifest = tmp_path / "data" / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path="data/train.png", text="नेपाल")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path="data/eval.png", text="English")], eval_manifest, ) monkeypatch.chdir(tmp_path) write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe") bundle = bundle_recognizer_training( "runs/recipe", "runs/bundle", train_manifest="data/train.jsonl", eval_manifest="data/eval.jsonl", ) assert bundle.file_count >= 7 archive = Path(bundle.archive_path) extract_dir = tmp_path / "extract" extract_dir.mkdir() import tarfile with tarfile.open(archive, "r:gz") as handle: handle.extractall(extract_dir) assert (extract_dir / "runs" / "recipe" / "paddle_recognizer.yml").exists() summary_path = extract_dir / "runs" / "recipe" / "recognizer-training-summary.json" assert summary_path.exists() train_label = extract_dir / "runs" / "recipe" / "labels" / "train.txt" assert train_label.exists() assert (extract_dir / "data" / "train.jsonl").exists() rewritten = json.loads((extract_dir / "data" / "train.jsonl").read_text(encoding="utf-8").splitlines()[0]) rewritten_image = extract_dir / rewritten["image_path"] assert rewritten_image.read_bytes() == b"train" assert train_label.read_text(encoding="utf-8").split("\t", 1)[0] == rewritten["image_path"] manifest = json.loads(Path(bundle.manifest_path).read_text(encoding="utf-8")) assert manifest["recipe_backend"] == "paddleocr" assert manifest["run_command"] == "python PaddleOCR/tools/train.py -c runs/recipe/paddle_recognizer.yml" assert manifest["training_summary"]["present"] is True assert manifest["training_summary"]["path"] == "runs/recipe/recognizer-training-summary.json" assert manifest["training_summary"]["sha256"] assert manifest["recognizer_corpus_audit"] == {"required": False, "present": False} assert "python PaddleOCR/tools/train.py -c runs/recipe/paddle_recognizer.yml" in Path(bundle.readme_path).read_text(encoding="utf-8") def test_bundle_recognizer_training_writes_hf_run_command(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: train_image = tmp_path / "data" / "train.png" eval_image = tmp_path / "data" / "eval.png" train_image.parent.mkdir() train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "data" / "train.jsonl" eval_manifest = tmp_path / "data" / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path="data/train.png", text="नेपाल English")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path="data/eval.png", text="काठमाडौं")], eval_manifest, ) monkeypatch.chdir(tmp_path) write_hf_recognizer_recipe( "data/train.jsonl", "data/eval.jsonl", "runs/hf-recipe", base_model="microsoft/trocr-small-printed", learning_rate=3e-5, num_train_epochs=4, ) bundle = bundle_recognizer_training( "runs/hf-recipe", "runs/hf-bundle", train_manifest="data/train.jsonl", eval_manifest="data/eval.jsonl", ) manifest = json.loads(Path(bundle.manifest_path).read_text(encoding="utf-8")) assert manifest["recipe_backend"] == "hf_vision_encoder_decoder" assert "--backend hf-vision-encoder-decoder" in manifest["run_command"] assert "--base-model microsoft/trocr-small-printed" in manifest["run_command"] assert "--learning-rate 3e-05" in manifest["run_command"] assert "--epochs 4" in manifest["run_command"] readme = Path(bundle.readme_path).read_text(encoding="utf-8") assert manifest["run_command"] in readme def test_bundle_recognizer_training_can_require_passing_corpus_audit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: train_image_en = tmp_path / "data" / "train-en.png" train_image_ne = tmp_path / "data" / "train-ne.png" train_image_mix = tmp_path / "data" / "train-mix.png" eval_image = tmp_path / "data" / "eval-mix.png" train_image_en.parent.mkdir() for path in [train_image_en, train_image_ne, train_image_mix, eval_image]: path.write_bytes(path.name.encode("utf-8")) train_manifest = tmp_path / "data" / "train.jsonl" eval_manifest = tmp_path / "data" / "eval.jsonl" write_manifest( [ ManifestEntry(sample_id="train-en", dataset="local", split="train", image_path="data/train-en.png", text="Invoice", metadata={"slices": ["english", "real"]}), ManifestEntry(sample_id="train-ne", dataset="local", split="train", image_path="data/train-ne.png", text="नेपाल", metadata={"slices": ["nepali", "synthetic"]}), ManifestEntry(sample_id="train-mix", dataset="local", split="train", image_path="data/train-mix.png", text="नाम Ram", metadata={"slices": ["mixed_script", "real", "synthetic"]}), ], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-mix", dataset="local", split="eval", image_path="data/eval-mix.png", text="बिल No", metadata={"slices": ["mixed_script", "hard_eval"]})], eval_manifest, ) monkeypatch.chdir(tmp_path) write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe") bundle = bundle_recognizer_training( "runs/recipe", "runs/bundle", train_manifest="data/train.jsonl", eval_manifest="data/eval.jsonl", require_corpus_audit=True, min_train_samples=3, min_eval_samples=1, min_train_english=2, min_train_devanagari=2, min_train_mixed=1, min_eval_english=1, min_eval_devanagari=1, min_eval_mixed=1, min_train_latin_only=1, min_train_devanagari_only=1, min_eval_latin_only=0, min_eval_devanagari_only=0, min_train_real=2, min_train_synthetic=2, ) manifest = json.loads(Path(bundle.manifest_path).read_text(encoding="utf-8")) assert manifest["recognizer_corpus_audit"] == { "required": True, "present": True, "passed": True, "path": "recognizer-corpus-audit/recognizer-corpus-audit.json", } assert (tmp_path / "runs" / "bundle" / "recognizer-corpus-audit" / "recognizer-corpus-audit.json").exists() def test_bundle_recognizer_training_rejects_stopped_family_review(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: train_image = tmp_path / "data" / "train.png" eval_image = tmp_path / "data" / "eval.png" train_image.parent.mkdir() train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "data" / "train.jsonl" eval_manifest = tmp_path / "data" / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path="data/train.png", text="नेपाल")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path="data/eval.png", text="English")], eval_manifest, ) monkeypatch.chdir(tmp_path) write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe") with pytest.raises(ValidationError, match="recognizer family is stopped"): bundle_recognizer_training( "runs/recipe", "runs/bundle", train_manifest="data/train.jsonl", eval_manifest="data/eval.jsonl", failure_review=_stopped_ppocrv3_failure_review(tmp_path), ) def test_bundle_recognizer_training_rejects_failed_corpus_audit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: train_image = tmp_path / "data" / "train.png" eval_image = tmp_path / "data" / "eval.png" train_image.parent.mkdir() train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "data" / "train.jsonl" eval_manifest = tmp_path / "data" / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-ne", dataset="local", split="train", image_path="data/train.png", text="नेपाल", metadata={"slices": ["nepali", "real"]})], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-ne", dataset="local", split="eval", image_path="data/eval.png", text="सरकार", metadata={"slices": ["nepali"]})], eval_manifest, ) monkeypatch.chdir(tmp_path) write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe") with pytest.raises(DataValidationError, match="recognizer corpus audit failed"): bundle_recognizer_training( "runs/recipe", "runs/bundle", train_manifest="data/train.jsonl", eval_manifest="data/eval.jsonl", require_corpus_audit=True, min_train_samples=1, min_eval_samples=1, min_train_english=1, min_train_devanagari=1, min_train_mixed=1, min_eval_english=1, min_eval_devanagari=1, min_eval_mixed=0, min_train_latin_only=1, min_train_devanagari_only=1, min_eval_latin_only=0, min_eval_devanagari_only=1, min_train_real=1, min_train_synthetic=0, ) def test_train_hf_recognizer_writes_recipe(tmp_path: Path) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल English")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")], eval_manifest, ) config = write_hf_recognizer_recipe( train_manifest, eval_manifest, tmp_path / "run", base_model="microsoft/trocr-small-printed", learning_rate=3e-5, num_train_epochs=4, ) assert config.exists() payload = json.loads(config.read_text(encoding="utf-8")) assert payload["base_model"] == "microsoft/trocr-small-printed" assert payload["learning_rate"] == 3e-5 assert payload["num_train_epochs"] == 4 assert payload["train_manifest"] == str(train_manifest) readme = (tmp_path / "run" / "README.md").read_text(encoding="utf-8") assert "--base-model microsoft/trocr-small-printed" in readme assert "--learning-rate 3e-05" in readme assert "--epochs 4" in readme summary = json.loads((tmp_path / "run" / "recognizer-training-summary.json").read_text(encoding="utf-8")) assert summary["backend"] == "hf_vision_encoder_decoder" assert summary["base_model"] == "microsoft/trocr-small-printed" assert summary["manifests"]["train"]["contains_ascii"] is True assert summary["manifests"]["train"]["contains_devanagari"] is True assert summary["artifacts"]["dictionary"]["contains_ascii"] is True assert summary["artifacts"]["dictionary"]["contains_devanagari"] is True assert summary["artifacts"]["config"]["sha256"] def test_run_hf_recognizer_training_loads_processor_without_fast_tokenizer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: train_image = tmp_path / "train.png" eval_image = tmp_path / "eval.png" train_image.write_bytes(b"train") eval_image.write_bytes(b"eval") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल English")], train_manifest, ) write_manifest( [ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")], eval_manifest, ) config = write_hf_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", base_model="microsoft/trocr-small-printed") calls: list[tuple[str, object]] = [] class FakeTorch: class cuda: @staticmethod def is_available() -> bool: return False class backends: class mps: @staticmethod def is_available() -> bool: return False class FakeTokenizer: cls_token_id = 0 bos_token_id = 1 pad_token_id = 2 class FakeProcessor: def __init__(self) -> None: self.tokenizer = FakeTokenizer() @classmethod def from_pretrained(cls, source: str, **kwargs): calls.append(("processor", kwargs)) return cls() def save_pretrained(self, output_dir: str) -> None: Path(output_dir).mkdir(parents=True, exist_ok=True) Path(output_dir, "preprocessor_config.json").write_text("{}", encoding="utf-8") class FakeGenerationConfig: max_length = 0 class FakeModel: def __init__(self) -> None: self.config = SimpleNamespace(decoder=SimpleNamespace()) self.generation_config = FakeGenerationConfig() @classmethod def from_pretrained(cls, source: str): calls.append(("model", source)) return cls() class FakeArgs: def __init__(self, **kwargs): self.kwargs = kwargs class FakeTrainer: def __init__(self, **kwargs): self.kwargs = kwargs def train(self) -> None: return None def save_model(self, output_dir: str) -> None: Path(output_dir).mkdir(parents=True, exist_ok=True) Path(output_dir, "config.json").write_text("{}", encoding="utf-8") def evaluate(self, max_length: int): return {"eval_cer": 0.1, "max_length": max_length} class FakeImageModule: class Image: pass monkeypatch.setitem( sys.modules, "transformers", SimpleNamespace( Seq2SeqTrainer=FakeTrainer, Seq2SeqTrainingArguments=FakeArgs, TrOCRProcessor=FakeProcessor, VisionEncoderDecoderModel=FakeModel, ), ) monkeypatch.setitem(sys.modules, "torch", FakeTorch) monkeypatch.setitem(sys.modules, "PIL", FakeImageModule) final_model_dir = run_hf_recognizer_training(config) assert calls[0] == ("processor", {"backend": "pil"}) assert final_model_dir.exists() assert final_model_dir.parent.joinpath("training_metrics.json").exists() assert json.loads((tmp_path / "run" / "training_metrics.json").read_text(encoding="utf-8"))["eval_cer"] == 0.1 def test_train_corrector_writes_synthetic_pairs(tmp_path: Path) -> None: image = tmp_path / "line.png" image.write_bytes(b"line") manifest = tmp_path / "train.jsonl" write_manifest( [ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(image), text="This is a longer clean line for corruption.")], manifest, ) config = write_corrector_recipe(tmp_path / "corrector", clean_manifest=manifest, seed=1) assert config.exists() assert (tmp_path / "corrector" / "correction_pairs.jsonl").exists() def test_train_corrector_writes_hf_pairs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: fake_module = SimpleNamespace(load_dataset=lambda dataset_id, split, streaming=False: [{"ocr": "गलत पाठ", "correct": "सही पाठ", "font": "Kalimati"}]) monkeypatch.setitem(sys.modules, "datasets", fake_module) config = write_corrector_recipe(tmp_path / "corrector", hf_dataset="cfilt/RoundTripOCR-nepali", hf_split="train") pairs = (tmp_path / "corrector" / "correction_pairs.jsonl").read_text(encoding="utf-8") assert config.exists() assert json.loads(pairs.splitlines()[0])["clean_text"] == "सही पाठ" def test_generate_correction_pairs_uses_engine_output(tmp_path: Path) -> None: source = tmp_path / "page.txt" source.write_text("गलत पाठ\n", encoding="utf-8") manifest = tmp_path / "train.jsonl" write_manifest( [ManifestEntry(sample_id="s1", dataset="fixture", split="train", image_path=str(source), text="सही पाठ")], manifest, ) pairs = generate_correction_pairs(manifest, tmp_path / "pairs.jsonl", engine_name="sidecar") rows = [json.loads(line) for line in pairs.read_text(encoding="utf-8").splitlines() if line.strip()] assert pairs.exists() assert rows[0]["noisy_text"] == "गलत पाठ" assert rows[0]["clean_text"] == "सही पाठ" def test_write_hf_text_corrector_recipe_splits_pairs(tmp_path: Path) -> None: pairs_path = tmp_path / "pairs.jsonl" pairs_path.write_text( "\n".join( json.dumps({"noisy_text": f"गलत {index}", "clean_text": f"सही {index}"}, ensure_ascii=False) for index in range(6) ) + "\n", encoding="utf-8", ) config = write_hf_text_corrector_recipe(pairs_path, tmp_path / "corrector", eval_ratio=0.33, seed=5) payload = json.loads(config.read_text(encoding="utf-8")) assert config.exists() assert payload["base_model"] == "google/byt5-small" assert (tmp_path / "corrector" / "train_pairs.jsonl").exists() assert (tmp_path / "corrector" / "eval_pairs.jsonl").exists() def test_hf_text_corrector_metrics_sanitizes_invalid_prediction_ids() -> None: class FakeTokenizer: pad_token_id = 0 vocab_size = 8 def batch_decode(self, rows, skip_special_tokens: bool = True): return [" ".join(str(token) for token in row if not skip_special_tokens or token != self.pad_token_id) for row in rows] metrics_fn = _hf_text_corrector_metrics_factory(FakeTokenizer()) eval_pred = SimpleNamespace( predictions=[[3, 1_114_112, -7, 2]], label_ids=[[3, -100, 2]], ) metrics = metrics_fn(eval_pred) assert metrics["cer"] == 0 assert metrics["wer"] == 0 assert metrics["exact_line_accuracy"] == 1 def test_package_hf_text_corrector_model_writes_auditable_model_card(tmp_path: Path) -> None: model_dir = tmp_path / "final-model" model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") (model_dir / "tokenizer.json").write_text("{}", encoding="utf-8") card = package_hf_text_corrector_model( model_dir, tmp_path / "package", model_id="ocrtech-hf-corrector", base_model="google/byt5-small", ) audit = audit_model_card(card) assert card.exists() assert audit.passed assert (tmp_path / "package" / "artifacts" / "model" / "config.json").exists() def test_export_recognizer_writes_export_recipe(tmp_path: Path) -> None: training_config = tmp_path / "paddle_recognizer.yml" training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8") export_spec = write_recognizer_export_recipe(training_config, tmp_path / "checkpoints" / "best_accuracy", tmp_path / "export") assert export_spec.exists() text = export_spec.read_text(encoding="utf-8") assert "export_model.py" in text assert "Global.save_inference_dir" in text def test_export_recognizer_accepts_paddle_checkpoint_prefix_with_pdparams(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: training_config = tmp_path / "paddle_recognizer.yml" training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8") checkpoint = tmp_path / "checkpoints" / "best_accuracy" checkpoint.parent.mkdir() checkpoint.with_name("best_accuracy.pdparams").write_bytes(b"params") monkeypatch.setattr("ocrtech.training.subprocess.run", lambda *args, **kwargs: SimpleNamespace(returncode=0)) export_spec = write_recognizer_export_recipe(training_config, checkpoint, tmp_path / "export", run=True) payload = json.loads(export_spec.read_text(encoding="utf-8")) assert f"Global.pretrained_model={checkpoint}" in payload["command"] def test_export_recognizer_normalizes_pdparams_checkpoint_path(tmp_path: Path) -> None: training_config = tmp_path / "paddle_recognizer.yml" training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8") checkpoint_file = tmp_path / "checkpoints" / "best_accuracy.pdparams" checkpoint_file.parent.mkdir() checkpoint_file.write_bytes(b"params") export_spec = write_recognizer_export_recipe(training_config, checkpoint_file, tmp_path / "export") payload = json.loads(export_spec.read_text(encoding="utf-8")) assert f"Global.pretrained_model={checkpoint_file.with_suffix('')}" in payload["command"] def test_package_paddleocr_model_writes_auditable_model_card(tmp_path: Path) -> None: inference_dir = tmp_path / "inference" inference_dir.mkdir() (inference_dir / "inference.pdmodel").write_bytes(b"model") (inference_dir / "inference.pdiparams").write_bytes(b"params") (inference_dir / "inference.pdiparams.info").write_bytes(b"info") dictionary = tmp_path / "nep_eng_dict.txt" dictionary.write_text("क\nA\n", encoding="utf-8") training_summary = tmp_path / "recognizer-training-summary.json" training_summary.write_text('{"backend": "paddleocr"}\n', encoding="utf-8") card = package_paddleocr_model( inference_dir, tmp_path / "package", model_id="ocrtech-packaged", dictionary_path=dictionary, training_summary=training_summary, ) audit = audit_model_card(card) assert card.exists() assert audit.passed payload = json.loads(card.read_text(encoding="utf-8")) assert payload["backend_kwargs"]["lang"] == "ne" assert "rec_model_dir" not in payload["backend_kwargs"] assert "rec_char_dict_path" not in payload["backend_kwargs"] assert payload["backend_kwargs"]["text_recognition_model_name"] == "devanagari_PP-OCRv3_mobile_rec" assert payload["backend_kwargs"]["text_recognition_model_dir"] == "artifacts" assert payload["provenance"]["training_summary"] == { "path": str(training_summary), "sha256": sha256_file(training_summary), } assert all(artifact["sha256"] for artifact in payload["artifacts"]) assert (tmp_path / "package" / "artifacts" / "inference.pdmodel").exists() assert (tmp_path / "package" / "artifacts" / "nep_eng_dict.txt").exists() def test_package_paddleocr_model_records_admission_validation(tmp_path: Path) -> None: inference_dir = tmp_path / "inference" inference_dir.mkdir() (inference_dir / "inference.pdmodel").write_bytes(b"model") (inference_dir / "inference.pdiparams").write_bytes(b"params") (inference_dir / "inference.pdiparams.info").write_bytes(b"info") dictionary = tmp_path / "nep_eng_dict.txt" dictionary.write_text("क\nA\n", encoding="utf-8") validation = tmp_path / "validation.json" card_path = tmp_path / "package" / "model-card.json" validation.write_text( json.dumps( { "claim_status": "validated", "passed": True, "provenance": {"candidate_model_config": {"path": str(card_path)}}, "gates": [_passing_validation_gate()], } ) + "\n", encoding="utf-8", ) card = package_paddleocr_model( inference_dir, tmp_path / "package", model_id="ocrtech-packaged-admitted", dictionary_path=dictionary, admission_validation_report=validation, ) status = model_admission_status(ModelCard.from_path(card), card) assert status.passed def test_package_hf_recognizer_model_writes_auditable_model_card(tmp_path: Path) -> None: model_dir = tmp_path / "final-model" model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") (model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8") training_summary = tmp_path / "recognizer-training-summary.json" training_summary.write_text('{"backend": "hf-vision-encoder-decoder"}\n', encoding="utf-8") card = package_hf_recognizer_model( model_dir, tmp_path / "package", model_id="ocrtech-hf-packaged", base_model="microsoft/trocr-small-printed", training_summary=training_summary, ) audit = audit_model_card(card) assert card.exists() assert audit.passed payload = json.loads(card.read_text(encoding="utf-8")) assert payload["provenance"]["training_summary"]["sha256"] == sha256_file(training_summary) assert (tmp_path / "package" / "artifacts" / "model" / "config.json").exists() def test_package_hf_recognizer_model_records_admission_validation(tmp_path: Path) -> None: model_dir = tmp_path / "final-model" model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") (model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8") validation = tmp_path / "validation.json" card_path = tmp_path / "package" / "model-card.json" validation.write_text( json.dumps( { "claim_status": "validated", "passed": True, "provenance": {"candidate_model_config": {"path": str(card_path)}}, "gates": [_passing_validation_gate()], } ) + "\n", encoding="utf-8", ) card = package_hf_recognizer_model( model_dir, tmp_path / "package", model_id="ocrtech-hf-packaged-admitted", base_model="microsoft/trocr-small-printed", admission_validation_report=validation, ) status = model_admission_status(ModelCard.from_path(card), card) assert status.passed def test_package_hf_recognizer_model_uses_card_relative_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) model_dir = Path("final-model") model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") (model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8") card = package_hf_recognizer_model(model_dir, Path("package"), model_id="ocrtech-hf-relative", base_model="microsoft/trocr-small-printed") payload = json.loads(card.read_text(encoding="utf-8")) audit = audit_model_card(card) assert payload["artifacts"][0]["path"] == "artifacts/model" assert payload["backend_kwargs"]["model_dir"] == "artifacts/model" assert audit.passed def test_finalize_hf_recognizer_run_requires_finished_artifacts(tmp_path: Path) -> None: run_dir = tmp_path / "run" run_dir.mkdir() with pytest.raises(DataValidationError, match="missing final model directory"): finalize_hf_recognizer_run( run_dir, model_id="ocrtech-hf-mixed-v2-full", base_model="microsoft/trocr-small-printed", ) (run_dir / "final-model").mkdir() with pytest.raises(DataValidationError, match="missing training metrics report"): finalize_hf_recognizer_run( run_dir, model_id="ocrtech-hf-mixed-v2-full", base_model="microsoft/trocr-small-printed", ) def test_finalize_hf_recognizer_run_packages_audits_and_writes_report(tmp_path: Path) -> None: run_dir = tmp_path / "run" model_dir = run_dir / "final-model" model_dir.mkdir(parents=True) (model_dir / "config.json").write_text("{}", encoding="utf-8") (model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8") (run_dir / "training_metrics.json").write_text(json.dumps({"eval_cer": 0.1}, indent=2), encoding="utf-8") training_summary = run_dir / "recognizer-training-summary.json" training_summary.write_text('{"backend": "hf-vision-encoder-decoder"}\n', encoding="utf-8") train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" image = tmp_path / "page.png" image.write_bytes(b"image") write_manifest([ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(image), text="नेपाल")], train_manifest) write_manifest([ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(image), text="English")], eval_manifest) result = finalize_hf_recognizer_run( run_dir, output_dir=tmp_path / "finalized", model_id="ocrtech-hf-mixed-v2-full", base_model="microsoft/trocr-small-printed", train_manifest=train_manifest, eval_manifest=eval_manifest, training_summary=training_summary, ) assert result.audit_passed assert Path(result.model_card_path).exists() assert Path(result.audit_path).exists() assert Path(result.report_json_path).exists() assert Path(result.report_md_path).exists() payload = json.loads(Path(result.report_json_path).read_text(encoding="utf-8")) assert payload["audit_passed"] is True model_card = json.loads(Path(result.model_card_path).read_text(encoding="utf-8")) assert model_card["metrics"]["report"]["path"] == str(run_dir / "training_metrics.json") assert model_card["provenance"]["training_summary"] == { "path": str(training_summary), "sha256": sha256_file(training_summary), } def test_benchmark_writes_report(tmp_path: Path) -> None: source = tmp_path / "sample.txt" source.write_text("hello world\n", encoding="utf-8") source.with_suffix(".ref.txt").write_text("hello world\n", encoding="utf-8") results = run_benchmark([source], tmp_path / "bench", baselines=["ours"]) assert len(results) == 1 assert results[0].status == "ok" assert results[0].metrics["cer"] == 0 assert (tmp_path / "bench" / "report.json").exists() def test_rescore_benchmark_recomputes_metrics_from_existing_documents(tmp_path: Path) -> None: source = tmp_path / "page.png" source.write_bytes(b"image") reference = tmp_path / "page.ref.json" reference.write_text(json.dumps({"tables": [[["Name", "Value"], ["A", "1"]]]}), encoding="utf-8") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="", metadata={"reference_path": str(reference), "slices": ["table"]}, ) ], manifest, ) benchmark_dir = tmp_path / "bench" output_dir = benchmark_dir / "candidate" / source.stem output_dir.mkdir(parents=True) document = Document( source_path=str(source), pages=[Page(page_index=0)], tables=[ Table(table_id="p0-t0", page_index=0, bbox=BBox(0, 0, 20, 20), cells=[TableCell(row=0, col=0, text="wrong")]), Table( table_id="p0-t1", page_index=0, bbox=BBox(0, 30, 100, 40), cells=[ TableCell(row=0, col=0, text="Name"), TableCell(row=0, col=1, text="Value"), TableCell(row=1, col=0, text="A"), TableCell(row=1, col=1, text="1"), ], ), ], ) (output_dir / "document.json").write_text(json.dumps(document.to_dict(), indent=2), encoding="utf-8") report = benchmark_dir / "report.json" report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "sample-1", "slices": ["table"], "metrics": {"table_cell_f1": 0.0}, "error": None, } ] ), encoding="utf-8", ) results = rescore_benchmark_report(report, tmp_path / "rescored", eval_manifest=manifest) assert results[0].status == "ok" assert results[0].metrics["table_cell_f1"] == 1.0 assert results[0].metrics["table_exact_match"] == 1.0 assert (tmp_path / "rescored" / "report.json").exists() assert (tmp_path / "rescored" / "candidate" / source.stem / "document.json").exists() def test_derive_table_postprocess_benchmark_scores_candidate_from_same_baseline_document(tmp_path: Path) -> None: source = tmp_path / "page.png" source.write_bytes(b"image") reference = tmp_path / "page.ref.json" reference.write_text(json.dumps({"tables": [[["Name", "Value"], ["A", "1"]]]}), encoding="utf-8") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path=str(source), text="", metadata={"reference_path": str(reference), "slices": ["table"]}, ) ], manifest, ) benchmark_dir = tmp_path / "bench" output_dir = benchmark_dir / "surya" / source.stem output_dir.mkdir(parents=True) document = Document( source_path=str(source), pages=[Page(page_index=0)], tables=[ Table( table_id="p0-t0", page_index=0, bbox=BBox(0, 0, 100, 40), cells=[ TableCell(row=0, col=0, text="—"), TableCell(row=0, col=1, text="Name"), TableCell(row=0, col=2, text="Value"), TableCell(row=0, col=3, text="Value"), TableCell(row=1, col=0, text="—"), TableCell(row=1, col=1, text="A"), TableCell(row=1, col=2, text="1"), TableCell(row=1, col=3, text="1"), ], ) ], ) (output_dir / "document.json").write_text(json.dumps(document.to_dict(), indent=2), encoding="utf-8") report = benchmark_dir / "report.json" report.write_text( json.dumps( [ { "baseline": "surya", "input_path": str(source), "status": "ok", "latency_seconds": 1.0, "sample_id": "sample-1", "slices": ["table"], "metrics": {"table_cell_f1": 0.0}, "error": None, } ] ), encoding="utf-8", ) results = derive_table_postprocess_benchmark(report, tmp_path / "derived", source_baseline="surya", eval_manifest=manifest) by_baseline = {result.baseline: result for result in results} candidate_document = json.loads((tmp_path / "derived" / "candidate" / source.stem / "document.json").read_text(encoding="utf-8")) assert by_baseline["surya"].metrics["table_cell_f1"] < 1.0 assert by_baseline["candidate"].metrics["table_cell_f1"] == 1.0 assert by_baseline["candidate"].metrics["table_exact_match"] == 1.0 assert candidate_document["metadata"]["derived_table_postprocess"]["source_baseline"] == "surya" assert candidate_document["metadata"]["derived_table_postprocess"]["normalized_table_grid_dropped_columns"] == 2 def test_benchmark_external_command_materializes_document_from_markdown_stdout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"png") source.with_suffix(".ref.json").write_text( json.dumps( { "text": "नेपाल सरकार", "tables": [[["नाम", "उमेर"], ["राम", "३२"]]], }, ensure_ascii=False, ), encoding="utf-8", ) script = tmp_path / "emit_markdown.py" script.write_text( "import argparse\n" "parser = argparse.ArgumentParser()\n" "parser.add_argument('--input')\n" "parser.add_argument('--out')\n" "parser.parse_args()\n" "print('# नेपाल सरकार\\n\\n| नाम | उमेर |\\n| --- | --- |\\n| राम | ३२ |')\n", encoding="utf-8", ) monkeypatch.setenv("OCRTECH_GLM_OCR_CMD", f"{sys.executable} {script} --input {{input}} --out {{out}}") results = run_benchmark([source], tmp_path / "bench", baselines=["glm-ocr"]) document_path = tmp_path / "bench" / "glm-ocr" / "page" / "document.json" assert results[0].status == "ok" assert results[0].metrics["table_cell_f1"] == 1.0 assert document_path.exists() payload = json.loads(document_path.read_text(encoding="utf-8")) assert payload["tables"][0]["cells"][0]["text"] == "नाम" assert (tmp_path / "bench" / "report.md").exists() def test_benchmark_reuses_candidate_engine_per_baseline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: first = tmp_path / "a.txt" second = tmp_path / "b.txt" first.write_text("नेपाल\n", encoding="utf-8") second.write_text("English\n", encoding="utf-8") create_calls: list[tuple[str, object]] = [] class FakeEngine: def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine( text=input_path.read_text(encoding="utf-8").strip(), bbox=BBox(0, 0, 10, 10), confidence=0.99, page_index=0, line_id="p0-l0", ) ], ) ] ) def fake_create_engine(name: str, **kwargs): create_calls.append((name, kwargs.get("model_config"))) return FakeEngine() monkeypatch.setattr("ocrtech.benchmark.create_engine", fake_create_engine) results = run_benchmark([first, second], tmp_path / "bench", baselines=["candidate"], candidate_model_config="model.json") assert len(results) == 2 assert all(result.status == "ok" for result in results) assert len(create_calls) == 1 assert create_calls[0][0] == "candidate" def test_benchmark_candidate_uses_fallback_parser_when_configured(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: source = tmp_path / "page.txt" source.write_text("नेपाल\n", encoding="utf-8") source.with_suffix(".ref.txt").write_text("नेपाल\n", encoding="utf-8") primary_model = tmp_path / "primary.json" fallback_model = tmp_path / "fallback.json" primary_model.write_text("{}", encoding="utf-8") fallback_model.write_text("{}", encoding="utf-8") calls: list[dict[str, object]] = [] def fake_parse_document(input_path: Path, output_dir: Path, **kwargs: object) -> Document: calls.append({"input_path": input_path, "output_dir": output_dir, **kwargs}) output_dir.mkdir(parents=True) document = Document( source_path=str(input_path), pages=[ Page( page_index=0, text_lines=[ TextLine( text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=0.99, page_index=0, line_id="p0-l0", ) ], ) ], metadata={ "quality": {"quality_score": 1.0, "metrics": {"line_count": 1, "low_confidence_line_count": 0}}, "fallback": { "triggered": True, "outcome": "success", "primary_average_confidence": 0.42, "threshold": 0.77, }, }, ) (output_dir / "document.json").write_text(json.dumps(document.to_dict()), encoding="utf-8") return document monkeypatch.setattr("ocrtech.benchmark.parse_document", fake_parse_document) results = run_benchmark( [source], tmp_path / "bench", baselines=["candidate"], candidate_model_config=primary_model, fallback_model_config=fallback_model, low_confidence_threshold=0.77, ) assert results[0].status == "ok" assert results[0].metrics["fallback_triggered"] == 1.0 assert results[0].metrics["fallback_succeeded"] == 1.0 report_md = (tmp_path / "bench" / "report.md").read_text(encoding="utf-8") assert "fallback_failed" in report_md assert "| candidate |" in report_md assert "| 1 | 0 |" in report_md assert calls[0]["engine_name"] == "candidate" assert calls[0]["model_config"] == primary_model assert calls[0]["fallback_model_config"] == fallback_model assert calls[0]["low_confidence_threshold"] == 0.77 def test_benchmark_checkpoints_report_after_each_sample(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: first = tmp_path / "first.txt" second = tmp_path / "second.txt" first.write_text("one\n", encoding="utf-8") second.write_text("two\n", encoding="utf-8") first.with_suffix(".ref.txt").write_text("one\n", encoding="utf-8") second.with_suffix(".ref.txt").write_text("two\n", encoding="utf-8") output_dir = tmp_path / "bench" class FakeEngine: calls = 0 def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine self.calls += 1 if self.calls == 2: checkpoint = json.loads((output_dir / "report.json").read_text(encoding="utf-8")) assert len(checkpoint) == 1 assert checkpoint[0]["input_path"] == str(first) return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine( text=input_path.read_text(encoding="utf-8").strip(), bbox=BBox(0, 0, 10, 10), confidence=0.99, page_index=0, line_id="p0-l0", ) ], ) ] ) monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine()) results = run_benchmark([first, second], output_dir, baselines=["candidate"], candidate_model_config="model.json") assert len(results) == 2 final_report = json.loads((output_dir / "report.json").read_text(encoding="utf-8")) assert len(final_report) == 2 def test_benchmark_resume_existing_scores_existing_document_without_rerun(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: first = tmp_path / "first.txt" second = tmp_path / "second.txt" first.write_text("one\n", encoding="utf-8") second.write_text("two\n", encoding="utf-8") first.with_suffix(".ref.txt").write_text("one\n", encoding="utf-8") second.with_suffix(".ref.txt").write_text("two\n", encoding="utf-8") output_dir = tmp_path / "bench" existing_dir = output_dir / "candidate" / first.stem existing_dir.mkdir(parents=True) existing_document = Document( source_path=str(first), pages=[ Page( page_index=0, text_lines=[ TextLine( text="one", bbox=BBox(0, 0, 10, 10), confidence=0.99, page_index=0, line_id="p0-l0", ) ], ) ], ) (existing_dir / "document.json").write_text(json.dumps(existing_document.to_dict(), ensure_ascii=False), encoding="utf-8") engine_inputs: list[Path] = [] class FakeEngine: def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput engine_inputs.append(input_path) return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine( text=input_path.read_text(encoding="utf-8").strip(), bbox=BBox(0, 0, 10, 10), confidence=0.99, page_index=0, line_id="p0-l0", ) ], ) ] ) monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine()) results = run_benchmark( [first, second], output_dir, baselines=["candidate"], candidate_model_config="model.json", resume_existing=True, ) assert [result.status for result in results] == ["ok", "ok"] assert results[0].latency_seconds == 0.0 assert engine_inputs == [second] report = json.loads((output_dir / "report.json").read_text(encoding="utf-8")) assert [row["input_path"] for row in report] == [str(first), str(second)] def test_benchmark_sample_timeout_records_error_and_continues(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: slow = tmp_path / "slow.txt" fast = tmp_path / "fast.txt" slow.write_text("slow\n", encoding="utf-8") fast.write_text("fast\n", encoding="utf-8") slow.with_suffix(".ref.txt").write_text("slow\n", encoding="utf-8") fast.with_suffix(".ref.txt").write_text("fast\n", encoding="utf-8") class FakeEngine: def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput if input_path == slow: time.sleep(2) return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine( text=input_path.read_text(encoding="utf-8").strip(), bbox=BBox(0, 0, 10, 10), confidence=0.99, page_index=0, line_id="p0-l0", ) ], ) ] ) monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine()) results = run_benchmark( [slow, fast], tmp_path / "bench", baselines=["candidate"], candidate_model_config="model.json", sample_timeout_seconds=1, ) assert results[0].status == "error" assert "TimeoutError" in str(results[0].error) assert results[1].status == "ok" def test_benchmark_sample_timeout_uses_process_watchdog_when_available(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: if "fork" not in multiprocessing.get_all_start_methods(): pytest.skip("process watchdog requires fork start method") source = tmp_path / "slow.txt" source.write_text("slow\n", encoding="utf-8") source.with_suffix(".ref.txt").write_text("slow\n", encoding="utf-8") def fail_if_signal_timer_is_used(*args, **kwargs): raise AssertionError("signal timer should not be used when fork watchdog is available") class FakeEngine: def recognize(self, input_path: Path): time.sleep(2) monkeypatch.setattr(signal, "setitimer", fail_if_signal_timer_is_used) monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine()) results = run_benchmark( [source], tmp_path / "bench", baselines=["candidate"], candidate_model_config="model.json", sample_timeout_seconds=1, ) assert results[0].status == "error" assert "TimeoutError" in str(results[0].error) def test_benchmark_timeout_builds_engine_inside_watchdog_worker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: if "fork" not in multiprocessing.get_all_start_methods(): pytest.skip("process watchdog requires fork start method") source = tmp_path / "sample.txt" source.write_text("नेपाल\n", encoding="utf-8") source.with_suffix(".ref.txt").write_text("नेपाल\n", encoding="utf-8") pid_log = tmp_path / "engine-pids.txt" parent_pid = os.getpid() class FakeEngine: def __init__(self) -> None: with pid_log.open("a", encoding="utf-8") as handle: handle.write(f"{os.getpid()}\n") def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine( text=input_path.read_text(encoding="utf-8").strip(), bbox=BBox(0, 0, 10, 10), confidence=0.99, page_index=0, line_id="p0-l0", ) ], ) ] ) monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine()) results = run_benchmark( [source], tmp_path / "bench", baselines=["candidate"], candidate_model_config="model.json", sample_timeout_seconds=5, ) assert results[0].status == "ok" engine_pids = [int(line) for line in pid_log.read_text(encoding="utf-8").splitlines()] assert engine_pids assert parent_pid not in engine_pids def test_benchmark_supports_surya_baseline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: source = tmp_path / "sample.txt" source.write_text("नेपाल\n", encoding="utf-8") source.with_suffix(".ref.txt").write_text("नेपाल\n", encoding="utf-8") create_calls: list[str] = [] class FakeEngine: def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=0.88, page_index=0, line_id="p0-l0")], metadata={"surya_mode": "block"}, ) ], metadata={"surya_mode": "block"}, ) def fake_create_engine(name: str, **kwargs): create_calls.append(name) return FakeEngine() monkeypatch.setattr("ocrtech.benchmark.create_engine", fake_create_engine) results = run_benchmark([source], tmp_path / "bench", baselines=["surya"]) assert len(results) == 1 assert results[0].status == "ok" assert results[0].metrics["cer"] == 0 assert create_calls == ["surya"] def test_summarize_benchmark_report_writes_paired_and_aggregate_metrics(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": "a.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["english"], "metrics": {"cer": 0.10, "quality_score": 0.90}, "error": None, }, { "baseline": "candidate", "input_path": "b.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s2", "slices": ["nepali"], "metrics": {"cer": 0.40, "quality_score": 0.80}, "error": None, }, { "baseline": "tesseract", "input_path": "a.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["english"], "metrics": {"cer": 0.20, "quality_score": 0.70}, "error": None, }, { "baseline": "tesseract", "input_path": "b.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s2", "slices": ["nepali"], "metrics": {"cer": 0.30, "quality_score": 0.75}, "error": None, }, ], indent=2, ), encoding="utf-8", ) summary = summarize_benchmark_report(report, tmp_path / "summary", candidate="candidate", baselines=["tesseract"], metrics=["cer"]) assert (tmp_path / "summary" / "summary.json").exists() assert (tmp_path / "summary" / "summary.md").exists() assert summary.aggregate_metrics["candidate"]["all"]["sample_count"] == 2 assert summary.aggregate_metrics["candidate"]["all"]["latency_seconds"] == pytest.approx(1.0) english = next(item for item in summary.paired_metrics if item.baseline == "tesseract" and item.slice_name == "english") nepali = next(item for item in summary.paired_metrics if item.baseline == "tesseract" and item.slice_name == "nepali") assert english.absolute_improvement == pytest.approx(0.10) assert english.win_count == 1 assert nepali.absolute_improvement == pytest.approx(-0.10) assert nepali.loss_count == 1 def test_calibrate_quality_router_writes_model_card_with_calibration_provenance(tmp_path: Path) -> None: eval_manifest = tmp_path / "eval.jsonl" eval_image = tmp_path / "sample.txt" eval_image.write_text("नेपाल\n", encoding="utf-8") write_manifest( [ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(eval_image), text="नेपाल", metadata={"slices": ["nepali"]})], eval_manifest, ) benchmark_report = tmp_path / "report.json" benchmark_report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": str(eval_image), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.20, "quality_score": 0.95}, "error": None, }, { "baseline": "tesseract", "input_path": str(eval_image), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.40, "quality_score": 0.70}, "error": None, }, ], indent=2, ), encoding="utf-8", ) experiment_report = tmp_path / "experiment.json" experiment_report.write_text( json.dumps( { "experiment_id": "calibration", "output_dir": str(tmp_path / "exp"), "eval_manifest": str(eval_manifest), "references_dir": None, "baselines": ["candidate", "tesseract"], "input_count": 1, "datasets": ["fixture"], "slice_counts": {"nepali": 1}, "benchmark_report": str(benchmark_report), "benchmark_status_counts": {"ok": 2}, "validation_report": None, "validation_status": None, "environment": {}, }, indent=2, ), encoding="utf-8", ) calibration, model_path = calibrate_quality_router( experiment_report, tmp_path / "calibrated", model_id="ocrtech-quality-router", primary_engine="tesseract", secondary_engine="tesseract", primary_engine_kwargs={"psm": 6}, secondary_engine_kwargs={"psm": 13}, threshold_start=0.0, threshold_end=0.1, threshold_step=0.05, ) payload = json.loads(model_path.read_text(encoding="utf-8")) assert (tmp_path / "calibrated" / "calibration.json").exists() assert calibration.selected_threshold == 0.1 assert payload["backend"] == "quality_select_composite" assert payload["provenance"]["calibration"]["eval_manifest"]["path"] == str(eval_manifest) def test_calibrate_tesseract_psm_ensemble_writes_ranked_model_card(tmp_path: Path) -> None: eval_manifest = tmp_path / "eval.jsonl" sample = tmp_path / "sample.txt" sample.write_text("नेपाल\n", encoding="utf-8") write_manifest( [ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(sample), text="नेपाल", metadata={"slices": ["nepali"]})], eval_manifest, ) def write_experiment(path: Path, benchmark_path: Path, rows: list[dict[str, object]]) -> Path: benchmark_path.write_text(json.dumps(rows, indent=2), encoding="utf-8") payload = { "experiment_id": path.stem, "output_dir": str(path.parent / path.stem), "eval_manifest": str(eval_manifest), "references_dir": None, "baselines": ["candidate", "tesseract"], "input_count": 1, "datasets": ["fixture"], "slice_counts": {"nepali": 1}, "benchmark_report": str(benchmark_path), "benchmark_status_counts": {"ok": len(rows)}, "validation_report": None, "validation_status": None, "environment": {}, } path.write_text(json.dumps(payload, indent=2), encoding="utf-8") return path primary = write_experiment( tmp_path / "primary-experiment.json", tmp_path / "primary-report.json", [ {"baseline": "tesseract", "input_path": str(sample), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.4, "quality_score": 0.7}, "error": None} ], ) psm7 = write_experiment( tmp_path / "psm7-experiment.json", tmp_path / "psm7-report.json", [ {"baseline": "candidate", "input_path": str(sample), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.3, "quality_score": 0.6}, "error": None} ], ) psm13 = write_experiment( tmp_path / "psm13-experiment.json", tmp_path / "psm13-report.json", [ {"baseline": "candidate", "input_path": str(sample), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.2, "quality_score": 0.9}, "error": None} ], ) calibration, model_path = calibrate_tesseract_psm_ensemble( primary, tmp_path / "ensemble", model_id="ocrtech-psm-ensemble", variant_experiment_reports={"psm7": psm7, "psm13": psm13}, bias_start=-0.04, bias_end=0.04, bias_step=0.04, ) payload = json.loads(model_path.read_text(encoding="utf-8")) assert (tmp_path / "ensemble" / "calibration.json").exists() assert payload["backend"] == "quality_ranked_ensemble" assert payload["backend_kwargs"]["alternative_engines"][0]["engine"] == "tesseract" assert calibration.selected_biases["psm13"] >= calibration.selected_biases["psm7"] def test_calibrate_script_router_writes_model_card(tmp_path: Path) -> None: eval_manifest = tmp_path / "eval.jsonl" sample_nep = tmp_path / "nep.png" sample_eng = tmp_path / "eng.png" sample_nep.write_bytes(b"nep") sample_eng.write_bytes(b"eng") write_manifest( [ ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(sample_nep), text="नेपाल", metadata={"slices": ["nepali"]}), ManifestEntry(sample_id="s2", dataset="fixture", split="eval", image_path=str(sample_eng), text="hello", metadata={"slices": ["english"]}), ], eval_manifest, ) output_dir = tmp_path / "experiment" (output_dir / "benchmark" / "tesseract" / sample_nep.stem).mkdir(parents=True) (output_dir / "benchmark" / "tesseract" / sample_eng.stem).mkdir(parents=True) (output_dir / "benchmark" / "tesseract" / sample_nep.stem / "document.md").write_text("नेपाल सरकार\n", encoding="utf-8") (output_dir / "benchmark" / "tesseract" / sample_eng.stem / "document.md").write_text("hello world\n", encoding="utf-8") benchmark_report = tmp_path / "report.json" benchmark_report.write_text( json.dumps( [ {"baseline": "tesseract", "input_path": str(sample_nep), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.40}, "error": None}, {"baseline": "surya", "input_path": str(sample_nep), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.20}, "error": None}, {"baseline": "tesseract", "input_path": str(sample_eng), "status": "ok", "latency_seconds": 1.0, "sample_id": "s2", "slices": ["english"], "metrics": {"cer": 0.10}, "error": None}, {"baseline": "surya", "input_path": str(sample_eng), "status": "ok", "latency_seconds": 1.0, "sample_id": "s2", "slices": ["english"], "metrics": {"cer": 0.50}, "error": None}, ], indent=2, ), encoding="utf-8", ) experiment_report = tmp_path / "experiment.json" experiment_report.write_text( json.dumps( { "experiment_id": "script-calibration", "output_dir": str(output_dir), "eval_manifest": str(eval_manifest), "references_dir": None, "baselines": ["surya", "tesseract"], "input_count": 2, "datasets": ["fixture"], "slice_counts": {"nepali": 1, "english": 1}, "benchmark_report": str(benchmark_report), "benchmark_status_counts": {"ok": 4}, "validation_report": None, "validation_status": None, "environment": {}, }, indent=2, ), encoding="utf-8", ) calibration, model_path = calibrate_script_router( experiment_report, tmp_path / "script-router", model_id="ocrtech-script-router", primary_engine="tesseract", secondary_engine="surya", threshold_start=0.0, threshold_end=0.5, threshold_step=0.25, routing_granularity="page", secondary_structure_backfill=True, ) payload = json.loads(model_path.read_text(encoding="utf-8")) assert (tmp_path / "script-router" / "calibration.json").exists() assert calibration.selected_threshold in {0.0, 0.25} assert payload["backend"] == "script_select_composite" assert payload["backend_kwargs"]["primary_script_threshold"] == calibration.selected_threshold assert payload["backend_kwargs"]["routing_granularity"] == "page" assert payload["backend_kwargs"]["secondary_structure_backfill"] is True def test_calibrate_script_router_supports_latin_script(tmp_path: Path) -> None: output_dir = tmp_path / "outputs" (output_dir / "benchmark" / "surya" / "doc-nepali").mkdir(parents=True) (output_dir / "benchmark" / "surya" / "doc-english").mkdir(parents=True) (output_dir / "benchmark" / "surya" / "doc-nepali" / "document.md").write_text("नेपाल सरकार\n", encoding="utf-8") (output_dir / "benchmark" / "surya" / "doc-english" / "document.md").write_text("Invoice Total\n", encoding="utf-8") benchmark_report = output_dir / "benchmark" / "report.json" benchmark_report.write_text( json.dumps( [ {"baseline": "surya", "input_path": "doc-nepali.png", "sample_id": "doc-nepali", "status": "ok", "metrics": {"cer": 0.1}}, {"baseline": "tesseract", "input_path": "doc-nepali.png", "sample_id": "doc-nepali", "status": "ok", "metrics": {"cer": 0.4}}, {"baseline": "surya", "input_path": "doc-english.png", "sample_id": "doc-english", "status": "ok", "metrics": {"cer": 0.4}}, {"baseline": "tesseract", "input_path": "doc-english.png", "sample_id": "doc-english", "status": "ok", "metrics": {"cer": 0.1}}, ], indent=2, ), encoding="utf-8", ) eval_manifest = tmp_path / "eval.jsonl" eval_manifest.write_text( "\n".join( [ json.dumps({"sample_id": "doc-nepali", "dataset": "fixture", "split": "eval", "image_path": "doc-nepali.png", "text": "नेपाल सरकार"}), json.dumps({"sample_id": "doc-english", "dataset": "fixture", "split": "eval", "image_path": "doc-english.png", "text": "Invoice Total"}), ] ) + "\n", encoding="utf-8", ) experiment_report = tmp_path / "experiment.json" experiment_report.write_text( json.dumps( { "experiment_id": "script-calibration-latin", "output_dir": str(output_dir), "eval_manifest": str(eval_manifest), "references_dir": None, "baselines": ["surya", "tesseract"], "input_count": 2, "datasets": ["fixture"], "slice_counts": {"nepali": 1, "english": 1}, "benchmark_report": str(benchmark_report), "benchmark_status_counts": {"ok": 4}, "validation_report": None, "validation_status": None, "environment": {}, }, indent=2, ), encoding="utf-8", ) calibration, model_path = calibrate_script_router( experiment_report, tmp_path / "script-router-latin", model_id="ocrtech-script-router-latin", primary_engine="surya", secondary_engine="tesseract", primary_baseline="surya", secondary_baseline="tesseract", script="latin", threshold_start=0.0, threshold_end=0.5, threshold_step=0.25, ) payload = json.loads(model_path.read_text(encoding="utf-8")) assert calibration.selected_threshold in {0.0, 0.25} assert payload["backend_kwargs"]["script"] == "latin" assert payload["backend_kwargs"]["primary_engine"] == "surya" assert payload["backend_kwargs"]["secondary_engine"] == "tesseract" def _stopped_ppocrv3_failure_review(tmp_path: Path) -> Path: review = tmp_path / "recognizer-failure-review.json" review.write_text( json.dumps( { "families": [ { "backend": "paddleocr", "base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", "family_key": "paddleocr::PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", "recommendation": "stop_family", "rejected_count": 4, "total_wins": 0, "total_losses": 545, "total_ties": 7, } ] } ) + "\n", encoding="utf-8", ) return review def _passing_validation_gate() -> dict[str, object]: return { "passed": True, "pairs": 2, "candidate_mean": 0.1, "baseline_mean": 0.2, "absolute_improvement": 0.1, "relative_improvement": 0.5, "improvement_ci_low": 0.01, "win_count": 2, "loss_count": 0, "tie_count": 0, "pair_win_rate": 1.0, "sign_test_p_value": 0.0, "gate": { "baseline": "tesseract", "metric": "cer", "min_pairs": 2, "min_pair_win_rate": 0.5, "require_ci_above_zero": True, }, }