Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_training_benchmark.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 79.9 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_training_benchmark.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_training_benchmark.py
-
curl -L -o test_training_benchmark.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_training_benchmark.py
79.9 kB
from __future__ import annotations
import json
import multiprocessing
import os
import signal
import sys
import time
from pathlib import Path
from types import SimpleNamespace
import pytest
from ocrtech.benchmark import derive_table_postprocess_benchmark, rescore_benchmark_report, run_benchmark, summarize_benchmark_report
from ocrtech.calibration import calibrate_quality_router, calibrate_script_router, calibrate_tesseract_psm_ensemble
from ocrtech.errors import DataValidationError, ValidationError
from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest
from ocrtech.models import ModelCard, audit_model_card, model_admission_status
from ocrtech.schemas import BBox, Document, Page, Table, TableCell, TextLine
from ocrtech.training import (
_hf_text_corrector_metrics_factory,
bundle_recognizer_training,
finalize_hf_recognizer_run,
generate_correction_pairs,
package_hf_text_corrector_model,
package_hf_recognizer_model,
package_paddleocr_model,
run_hf_recognizer_training,
write_corrector_recipe,
write_hf_text_corrector_recipe,
write_hf_recognizer_recipe,
write_recognizer_export_recipe,
write_recognizer_recipe,
)
def test_train_recognizer_writes_paddle_recipe(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="train-1",
dataset="local",
split="train",
image_path=str(train_image),
text="नेपाल English",
metadata={"slices": ["nepali", "english"]},
)
],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")],
eval_manifest,
)
config = write_recognizer_recipe(
train_manifest,
eval_manifest,
tmp_path / "run",
train_batch_size=4,
eval_batch_size=6,
learning_rate=0.0003,
epochs=2.5,
)
assert config.exists()
assert (tmp_path / "run" / "labels" / "train.txt").exists()
config_text = config.read_text(encoding="utf-8")
assert "Architecture:" in config_text
assert "algorithm: SVTR_LCNet" in config_text
assert "Loss:" in config_text
assert "PostProcess:" in config_text
assert "Metric:" in config_text
assert "character_dict_path" in config_text
assert "batch_size_per_card: 4" in config_text
assert "batch_size_per_card: 6" in config_text
assert "name: Cosine" in config_text
assert "learning_rate: 0.0003" in config_text
assert "epoch_num: 2.5" in config_text
dictionary = (tmp_path / "run" / "nep_eng_dict.txt").read_text(encoding="utf-8")
assert "क" in dictionary
assert "E" in dictionary
summary = json.loads((tmp_path / "run" / "recognizer-training-summary.json").read_text(encoding="utf-8"))
assert summary["backend"] == "paddleocr"
assert summary["manifests"]["train"]["sample_count"] == 1
assert summary["manifests"]["train"]["slice_counts"] == {"english": 1, "nepali": 1}
assert summary["manifests"]["train"]["contains_ascii"] is True
assert summary["manifests"]["train"]["contains_devanagari"] is True
assert summary["artifacts"]["dictionary"]["contains_ascii"] is True
assert summary["artifacts"]["dictionary"]["contains_devanagari"] is True
assert summary["artifacts"]["config"]["sha256"]
def test_train_recognizer_writes_integral_epochs_as_integer(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="नेपाल")],
eval_manifest,
)
config = write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", epochs=20.0)
config_text = config.read_text(encoding="utf-8")
assert "epoch_num: 20\n" in config_text
assert "epoch_num: 20.0" not in config_text
def test_train_recognizer_accepts_custom_dictionary_covering_labels(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल A")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="क B")],
eval_manifest,
)
dictionary = tmp_path / "dict.txt"
dictionary.write_text("\n".join(["न", "े", "प", "ा", "ल", " ", "A", "क", "B"]) + "\n", encoding="utf-8")
config = write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", dictionary_path=dictionary)
assert config.exists()
summary = json.loads((tmp_path / "run" / "recognizer-training-summary.json").read_text(encoding="utf-8"))
assert summary["artifacts"]["dictionary"]["path"] == str(dictionary)
def test_train_recognizer_rejects_custom_dictionary_missing_label_characters(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल A")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="क B")],
eval_manifest,
)
dictionary = tmp_path / "dict.txt"
dictionary.write_text("\n".join(["न", "े", "प", "ा", "ल", " ", "क"]) + "\n", encoding="utf-8")
with pytest.raises(DataValidationError, match="dictionary_path is missing label characters"):
write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", dictionary_path=dictionary)
def test_train_recognizer_writes_low_learning_rate_as_yaml_float(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="नेपाल")],
eval_manifest,
)
config = write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", learning_rate=0.00005)
config_text = config.read_text(encoding="utf-8")
assert "learning_rate: 0.00005" in config_text
assert "learning_rate: 5e-05" not in config_text
def test_train_recognizer_rejects_stopped_family_when_failure_review_supplied(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="नेपाल")],
eval_manifest,
)
with pytest.raises(ValidationError, match="recognizer family is stopped"):
write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", failure_review=_stopped_ppocrv3_failure_review(tmp_path))
def test_train_recognizer_paddle_labels_are_one_line_per_sample(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="train-1",
dataset="local",
split="train",
image_path=str(train_image),
text="पहिलो\tभाग\nदोस्रो भाग",
)
],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")],
eval_manifest,
)
write_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run")
train_lines = (tmp_path / "run" / "labels" / "train.txt").read_text(encoding="utf-8").splitlines()
assert len(train_lines) == 1
assert train_lines[0].endswith("\tपहिलो भाग दोस्रो भाग")
def test_bundle_recognizer_training_packages_recipe_manifests_and_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
train_image = tmp_path / "data" / "train.png"
eval_image = tmp_path / "data" / "eval.png"
train_image.parent.mkdir()
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "data" / "train.jsonl"
eval_manifest = tmp_path / "data" / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path="data/train.png", text="नेपाल")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path="data/eval.png", text="English")],
eval_manifest,
)
monkeypatch.chdir(tmp_path)
write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe")
bundle = bundle_recognizer_training(
"runs/recipe",
"runs/bundle",
train_manifest="data/train.jsonl",
eval_manifest="data/eval.jsonl",
)
assert bundle.file_count >= 7
archive = Path(bundle.archive_path)
extract_dir = tmp_path / "extract"
extract_dir.mkdir()
import tarfile
with tarfile.open(archive, "r:gz") as handle:
handle.extractall(extract_dir)
assert (extract_dir / "runs" / "recipe" / "paddle_recognizer.yml").exists()
summary_path = extract_dir / "runs" / "recipe" / "recognizer-training-summary.json"
assert summary_path.exists()
train_label = extract_dir / "runs" / "recipe" / "labels" / "train.txt"
assert train_label.exists()
assert (extract_dir / "data" / "train.jsonl").exists()
rewritten = json.loads((extract_dir / "data" / "train.jsonl").read_text(encoding="utf-8").splitlines()[0])
rewritten_image = extract_dir / rewritten["image_path"]
assert rewritten_image.read_bytes() == b"train"
assert train_label.read_text(encoding="utf-8").split("\t", 1)[0] == rewritten["image_path"]
manifest = json.loads(Path(bundle.manifest_path).read_text(encoding="utf-8"))
assert manifest["recipe_backend"] == "paddleocr"
assert manifest["run_command"] == "python PaddleOCR/tools/train.py -c runs/recipe/paddle_recognizer.yml"
assert manifest["training_summary"]["present"] is True
assert manifest["training_summary"]["path"] == "runs/recipe/recognizer-training-summary.json"
assert manifest["training_summary"]["sha256"]
assert manifest["recognizer_corpus_audit"] == {"required": False, "present": False}
assert "python PaddleOCR/tools/train.py -c runs/recipe/paddle_recognizer.yml" in Path(bundle.readme_path).read_text(encoding="utf-8")
def test_bundle_recognizer_training_writes_hf_run_command(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
train_image = tmp_path / "data" / "train.png"
eval_image = tmp_path / "data" / "eval.png"
train_image.parent.mkdir()
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "data" / "train.jsonl"
eval_manifest = tmp_path / "data" / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path="data/train.png", text="नेपाल English")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path="data/eval.png", text="काठमाडौं")],
eval_manifest,
)
monkeypatch.chdir(tmp_path)
write_hf_recognizer_recipe(
"data/train.jsonl",
"data/eval.jsonl",
"runs/hf-recipe",
base_model="microsoft/trocr-small-printed",
learning_rate=3e-5,
num_train_epochs=4,
)
bundle = bundle_recognizer_training(
"runs/hf-recipe",
"runs/hf-bundle",
train_manifest="data/train.jsonl",
eval_manifest="data/eval.jsonl",
)
manifest = json.loads(Path(bundle.manifest_path).read_text(encoding="utf-8"))
assert manifest["recipe_backend"] == "hf_vision_encoder_decoder"
assert "--backend hf-vision-encoder-decoder" in manifest["run_command"]
assert "--base-model microsoft/trocr-small-printed" in manifest["run_command"]
assert "--learning-rate 3e-05" in manifest["run_command"]
assert "--epochs 4" in manifest["run_command"]
readme = Path(bundle.readme_path).read_text(encoding="utf-8")
assert manifest["run_command"] in readme
def test_bundle_recognizer_training_can_require_passing_corpus_audit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
train_image_en = tmp_path / "data" / "train-en.png"
train_image_ne = tmp_path / "data" / "train-ne.png"
train_image_mix = tmp_path / "data" / "train-mix.png"
eval_image = tmp_path / "data" / "eval-mix.png"
train_image_en.parent.mkdir()
for path in [train_image_en, train_image_ne, train_image_mix, eval_image]:
path.write_bytes(path.name.encode("utf-8"))
train_manifest = tmp_path / "data" / "train.jsonl"
eval_manifest = tmp_path / "data" / "eval.jsonl"
write_manifest(
[
ManifestEntry(sample_id="train-en", dataset="local", split="train", image_path="data/train-en.png", text="Invoice", metadata={"slices": ["english", "real"]}),
ManifestEntry(sample_id="train-ne", dataset="local", split="train", image_path="data/train-ne.png", text="नेपाल", metadata={"slices": ["nepali", "synthetic"]}),
ManifestEntry(sample_id="train-mix", dataset="local", split="train", image_path="data/train-mix.png", text="नाम Ram", metadata={"slices": ["mixed_script", "real", "synthetic"]}),
],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-mix", dataset="local", split="eval", image_path="data/eval-mix.png", text="बिल No", metadata={"slices": ["mixed_script", "hard_eval"]})],
eval_manifest,
)
monkeypatch.chdir(tmp_path)
write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe")
bundle = bundle_recognizer_training(
"runs/recipe",
"runs/bundle",
train_manifest="data/train.jsonl",
eval_manifest="data/eval.jsonl",
require_corpus_audit=True,
min_train_samples=3,
min_eval_samples=1,
min_train_english=2,
min_train_devanagari=2,
min_train_mixed=1,
min_eval_english=1,
min_eval_devanagari=1,
min_eval_mixed=1,
min_train_latin_only=1,
min_train_devanagari_only=1,
min_eval_latin_only=0,
min_eval_devanagari_only=0,
min_train_real=2,
min_train_synthetic=2,
)
manifest = json.loads(Path(bundle.manifest_path).read_text(encoding="utf-8"))
assert manifest["recognizer_corpus_audit"] == {
"required": True,
"present": True,
"passed": True,
"path": "recognizer-corpus-audit/recognizer-corpus-audit.json",
}
assert (tmp_path / "runs" / "bundle" / "recognizer-corpus-audit" / "recognizer-corpus-audit.json").exists()
def test_bundle_recognizer_training_rejects_stopped_family_review(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
train_image = tmp_path / "data" / "train.png"
eval_image = tmp_path / "data" / "eval.png"
train_image.parent.mkdir()
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "data" / "train.jsonl"
eval_manifest = tmp_path / "data" / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path="data/train.png", text="नेपाल")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path="data/eval.png", text="English")],
eval_manifest,
)
monkeypatch.chdir(tmp_path)
write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe")
with pytest.raises(ValidationError, match="recognizer family is stopped"):
bundle_recognizer_training(
"runs/recipe",
"runs/bundle",
train_manifest="data/train.jsonl",
eval_manifest="data/eval.jsonl",
failure_review=_stopped_ppocrv3_failure_review(tmp_path),
)
def test_bundle_recognizer_training_rejects_failed_corpus_audit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
train_image = tmp_path / "data" / "train.png"
eval_image = tmp_path / "data" / "eval.png"
train_image.parent.mkdir()
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "data" / "train.jsonl"
eval_manifest = tmp_path / "data" / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-ne", dataset="local", split="train", image_path="data/train.png", text="नेपाल", metadata={"slices": ["nepali", "real"]})],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-ne", dataset="local", split="eval", image_path="data/eval.png", text="सरकार", metadata={"slices": ["nepali"]})],
eval_manifest,
)
monkeypatch.chdir(tmp_path)
write_recognizer_recipe("data/train.jsonl", "data/eval.jsonl", "runs/recipe")
with pytest.raises(DataValidationError, match="recognizer corpus audit failed"):
bundle_recognizer_training(
"runs/recipe",
"runs/bundle",
train_manifest="data/train.jsonl",
eval_manifest="data/eval.jsonl",
require_corpus_audit=True,
min_train_samples=1,
min_eval_samples=1,
min_train_english=1,
min_train_devanagari=1,
min_train_mixed=1,
min_eval_english=1,
min_eval_devanagari=1,
min_eval_mixed=0,
min_train_latin_only=1,
min_train_devanagari_only=1,
min_eval_latin_only=0,
min_eval_devanagari_only=1,
min_train_real=1,
min_train_synthetic=0,
)
def test_train_hf_recognizer_writes_recipe(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल English")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")],
eval_manifest,
)
config = write_hf_recognizer_recipe(
train_manifest,
eval_manifest,
tmp_path / "run",
base_model="microsoft/trocr-small-printed",
learning_rate=3e-5,
num_train_epochs=4,
)
assert config.exists()
payload = json.loads(config.read_text(encoding="utf-8"))
assert payload["base_model"] == "microsoft/trocr-small-printed"
assert payload["learning_rate"] == 3e-5
assert payload["num_train_epochs"] == 4
assert payload["train_manifest"] == str(train_manifest)
readme = (tmp_path / "run" / "README.md").read_text(encoding="utf-8")
assert "--base-model microsoft/trocr-small-printed" in readme
assert "--learning-rate 3e-05" in readme
assert "--epochs 4" in readme
summary = json.loads((tmp_path / "run" / "recognizer-training-summary.json").read_text(encoding="utf-8"))
assert summary["backend"] == "hf_vision_encoder_decoder"
assert summary["base_model"] == "microsoft/trocr-small-printed"
assert summary["manifests"]["train"]["contains_ascii"] is True
assert summary["manifests"]["train"]["contains_devanagari"] is True
assert summary["artifacts"]["dictionary"]["contains_ascii"] is True
assert summary["artifacts"]["dictionary"]["contains_devanagari"] is True
assert summary["artifacts"]["config"]["sha256"]
def test_run_hf_recognizer_training_loads_processor_without_fast_tokenizer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(train_image), text="नेपाल English")],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")],
eval_manifest,
)
config = write_hf_recognizer_recipe(train_manifest, eval_manifest, tmp_path / "run", base_model="microsoft/trocr-small-printed")
calls: list[tuple[str, object]] = []
class FakeTorch:
class cuda:
@staticmethod
def is_available() -> bool:
return False
class backends:
class mps:
@staticmethod
def is_available() -> bool:
return False
class FakeTokenizer:
cls_token_id = 0
bos_token_id = 1
pad_token_id = 2
class FakeProcessor:
def __init__(self) -> None:
self.tokenizer = FakeTokenizer()
@classmethod
def from_pretrained(cls, source: str, **kwargs):
calls.append(("processor", kwargs))
return cls()
def save_pretrained(self, output_dir: str) -> None:
Path(output_dir).mkdir(parents=True, exist_ok=True)
Path(output_dir, "preprocessor_config.json").write_text("{}", encoding="utf-8")
class FakeGenerationConfig:
max_length = 0
class FakeModel:
def __init__(self) -> None:
self.config = SimpleNamespace(decoder=SimpleNamespace())
self.generation_config = FakeGenerationConfig()
@classmethod
def from_pretrained(cls, source: str):
calls.append(("model", source))
return cls()
class FakeArgs:
def __init__(self, **kwargs):
self.kwargs = kwargs
class FakeTrainer:
def __init__(self, **kwargs):
self.kwargs = kwargs
def train(self) -> None:
return None
def save_model(self, output_dir: str) -> None:
Path(output_dir).mkdir(parents=True, exist_ok=True)
Path(output_dir, "config.json").write_text("{}", encoding="utf-8")
def evaluate(self, max_length: int):
return {"eval_cer": 0.1, "max_length": max_length}
class FakeImageModule:
class Image:
pass
monkeypatch.setitem(
sys.modules,
"transformers",
SimpleNamespace(
Seq2SeqTrainer=FakeTrainer,
Seq2SeqTrainingArguments=FakeArgs,
TrOCRProcessor=FakeProcessor,
VisionEncoderDecoderModel=FakeModel,
),
)
monkeypatch.setitem(sys.modules, "torch", FakeTorch)
monkeypatch.setitem(sys.modules, "PIL", FakeImageModule)
final_model_dir = run_hf_recognizer_training(config)
assert calls[0] == ("processor", {"backend": "pil"})
assert final_model_dir.exists()
assert final_model_dir.parent.joinpath("training_metrics.json").exists()
assert json.loads((tmp_path / "run" / "training_metrics.json").read_text(encoding="utf-8"))["eval_cer"] == 0.1
def test_train_corrector_writes_synthetic_pairs(tmp_path: Path) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"line")
manifest = tmp_path / "train.jsonl"
write_manifest(
[ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(image), text="This is a longer clean line for corruption.")],
manifest,
)
config = write_corrector_recipe(tmp_path / "corrector", clean_manifest=manifest, seed=1)
assert config.exists()
assert (tmp_path / "corrector" / "correction_pairs.jsonl").exists()
def test_train_corrector_writes_hf_pairs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
fake_module = SimpleNamespace(load_dataset=lambda dataset_id, split, streaming=False: [{"ocr": "गलत पाठ", "correct": "सही पाठ", "font": "Kalimati"}])
monkeypatch.setitem(sys.modules, "datasets", fake_module)
config = write_corrector_recipe(tmp_path / "corrector", hf_dataset="cfilt/RoundTripOCR-nepali", hf_split="train")
pairs = (tmp_path / "corrector" / "correction_pairs.jsonl").read_text(encoding="utf-8")
assert config.exists()
assert json.loads(pairs.splitlines()[0])["clean_text"] == "सही पाठ"
def test_generate_correction_pairs_uses_engine_output(tmp_path: Path) -> None:
source = tmp_path / "page.txt"
source.write_text("गलत पाठ\n", encoding="utf-8")
manifest = tmp_path / "train.jsonl"
write_manifest(
[ManifestEntry(sample_id="s1", dataset="fixture", split="train", image_path=str(source), text="सही पाठ")],
manifest,
)
pairs = generate_correction_pairs(manifest, tmp_path / "pairs.jsonl", engine_name="sidecar")
rows = [json.loads(line) for line in pairs.read_text(encoding="utf-8").splitlines() if line.strip()]
assert pairs.exists()
assert rows[0]["noisy_text"] == "गलत पाठ"
assert rows[0]["clean_text"] == "सही पाठ"
def test_write_hf_text_corrector_recipe_splits_pairs(tmp_path: Path) -> None:
pairs_path = tmp_path / "pairs.jsonl"
pairs_path.write_text(
"\n".join(
json.dumps({"noisy_text": f"गलत {index}", "clean_text": f"सही {index}"}, ensure_ascii=False)
for index in range(6)
)
+ "\n",
encoding="utf-8",
)
config = write_hf_text_corrector_recipe(pairs_path, tmp_path / "corrector", eval_ratio=0.33, seed=5)
payload = json.loads(config.read_text(encoding="utf-8"))
assert config.exists()
assert payload["base_model"] == "google/byt5-small"
assert (tmp_path / "corrector" / "train_pairs.jsonl").exists()
assert (tmp_path / "corrector" / "eval_pairs.jsonl").exists()
def test_hf_text_corrector_metrics_sanitizes_invalid_prediction_ids() -> None:
class FakeTokenizer:
pad_token_id = 0
vocab_size = 8
def batch_decode(self, rows, skip_special_tokens: bool = True):
return [" ".join(str(token) for token in row if not skip_special_tokens or token != self.pad_token_id) for row in rows]
metrics_fn = _hf_text_corrector_metrics_factory(FakeTokenizer())
eval_pred = SimpleNamespace(
predictions=[[3, 1_114_112, -7, 2]],
label_ids=[[3, -100, 2]],
)
metrics = metrics_fn(eval_pred)
assert metrics["cer"] == 0
assert metrics["wer"] == 0
assert metrics["exact_line_accuracy"] == 1
def test_package_hf_text_corrector_model_writes_auditable_model_card(tmp_path: Path) -> None:
model_dir = tmp_path / "final-model"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
(model_dir / "tokenizer.json").write_text("{}", encoding="utf-8")
card = package_hf_text_corrector_model(
model_dir,
tmp_path / "package",
model_id="ocrtech-hf-corrector",
base_model="google/byt5-small",
)
audit = audit_model_card(card)
assert card.exists()
assert audit.passed
assert (tmp_path / "package" / "artifacts" / "model" / "config.json").exists()
def test_export_recognizer_writes_export_recipe(tmp_path: Path) -> None:
training_config = tmp_path / "paddle_recognizer.yml"
training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8")
export_spec = write_recognizer_export_recipe(training_config, tmp_path / "checkpoints" / "best_accuracy", tmp_path / "export")
assert export_spec.exists()
text = export_spec.read_text(encoding="utf-8")
assert "export_model.py" in text
assert "Global.save_inference_dir" in text
def test_export_recognizer_accepts_paddle_checkpoint_prefix_with_pdparams(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
training_config = tmp_path / "paddle_recognizer.yml"
training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8")
checkpoint = tmp_path / "checkpoints" / "best_accuracy"
checkpoint.parent.mkdir()
checkpoint.with_name("best_accuracy.pdparams").write_bytes(b"params")
monkeypatch.setattr("ocrtech.training.subprocess.run", lambda *args, **kwargs: SimpleNamespace(returncode=0))
export_spec = write_recognizer_export_recipe(training_config, checkpoint, tmp_path / "export", run=True)
payload = json.loads(export_spec.read_text(encoding="utf-8"))
assert f"Global.pretrained_model={checkpoint}" in payload["command"]
def test_export_recognizer_normalizes_pdparams_checkpoint_path(tmp_path: Path) -> None:
training_config = tmp_path / "paddle_recognizer.yml"
training_config.write_text("Global:\n save_model_dir: checkpoints\n", encoding="utf-8")
checkpoint_file = tmp_path / "checkpoints" / "best_accuracy.pdparams"
checkpoint_file.parent.mkdir()
checkpoint_file.write_bytes(b"params")
export_spec = write_recognizer_export_recipe(training_config, checkpoint_file, tmp_path / "export")
payload = json.loads(export_spec.read_text(encoding="utf-8"))
assert f"Global.pretrained_model={checkpoint_file.with_suffix('')}" in payload["command"]
def test_package_paddleocr_model_writes_auditable_model_card(tmp_path: Path) -> None:
inference_dir = tmp_path / "inference"
inference_dir.mkdir()
(inference_dir / "inference.pdmodel").write_bytes(b"model")
(inference_dir / "inference.pdiparams").write_bytes(b"params")
(inference_dir / "inference.pdiparams.info").write_bytes(b"info")
dictionary = tmp_path / "nep_eng_dict.txt"
dictionary.write_text("क\nA\n", encoding="utf-8")
training_summary = tmp_path / "recognizer-training-summary.json"
training_summary.write_text('{"backend": "paddleocr"}\n', encoding="utf-8")
card = package_paddleocr_model(
inference_dir,
tmp_path / "package",
model_id="ocrtech-packaged",
dictionary_path=dictionary,
training_summary=training_summary,
)
audit = audit_model_card(card)
assert card.exists()
assert audit.passed
payload = json.loads(card.read_text(encoding="utf-8"))
assert payload["backend_kwargs"]["lang"] == "ne"
assert "rec_model_dir" not in payload["backend_kwargs"]
assert "rec_char_dict_path" not in payload["backend_kwargs"]
assert payload["backend_kwargs"]["text_recognition_model_name"] == "devanagari_PP-OCRv3_mobile_rec"
assert payload["backend_kwargs"]["text_recognition_model_dir"] == "artifacts"
assert payload["provenance"]["training_summary"] == {
"path": str(training_summary),
"sha256": sha256_file(training_summary),
}
assert all(artifact["sha256"] for artifact in payload["artifacts"])
assert (tmp_path / "package" / "artifacts" / "inference.pdmodel").exists()
assert (tmp_path / "package" / "artifacts" / "nep_eng_dict.txt").exists()
def test_package_paddleocr_model_records_admission_validation(tmp_path: Path) -> None:
inference_dir = tmp_path / "inference"
inference_dir.mkdir()
(inference_dir / "inference.pdmodel").write_bytes(b"model")
(inference_dir / "inference.pdiparams").write_bytes(b"params")
(inference_dir / "inference.pdiparams.info").write_bytes(b"info")
dictionary = tmp_path / "nep_eng_dict.txt"
dictionary.write_text("क\nA\n", encoding="utf-8")
validation = tmp_path / "validation.json"
card_path = tmp_path / "package" / "model-card.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(card_path)}},
"gates": [_passing_validation_gate()],
}
)
+ "\n",
encoding="utf-8",
)
card = package_paddleocr_model(
inference_dir,
tmp_path / "package",
model_id="ocrtech-packaged-admitted",
dictionary_path=dictionary,
admission_validation_report=validation,
)
status = model_admission_status(ModelCard.from_path(card), card)
assert status.passed
def test_package_hf_recognizer_model_writes_auditable_model_card(tmp_path: Path) -> None:
model_dir = tmp_path / "final-model"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
(model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8")
training_summary = tmp_path / "recognizer-training-summary.json"
training_summary.write_text('{"backend": "hf-vision-encoder-decoder"}\n', encoding="utf-8")
card = package_hf_recognizer_model(
model_dir,
tmp_path / "package",
model_id="ocrtech-hf-packaged",
base_model="microsoft/trocr-small-printed",
training_summary=training_summary,
)
audit = audit_model_card(card)
assert card.exists()
assert audit.passed
payload = json.loads(card.read_text(encoding="utf-8"))
assert payload["provenance"]["training_summary"]["sha256"] == sha256_file(training_summary)
assert (tmp_path / "package" / "artifacts" / "model" / "config.json").exists()
def test_package_hf_recognizer_model_records_admission_validation(tmp_path: Path) -> None:
model_dir = tmp_path / "final-model"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
(model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8")
validation = tmp_path / "validation.json"
card_path = tmp_path / "package" / "model-card.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(card_path)}},
"gates": [_passing_validation_gate()],
}
)
+ "\n",
encoding="utf-8",
)
card = package_hf_recognizer_model(
model_dir,
tmp_path / "package",
model_id="ocrtech-hf-packaged-admitted",
base_model="microsoft/trocr-small-printed",
admission_validation_report=validation,
)
status = model_admission_status(ModelCard.from_path(card), card)
assert status.passed
def test_package_hf_recognizer_model_uses_card_relative_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.chdir(tmp_path)
model_dir = Path("final-model")
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
(model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8")
card = package_hf_recognizer_model(model_dir, Path("package"), model_id="ocrtech-hf-relative", base_model="microsoft/trocr-small-printed")
payload = json.loads(card.read_text(encoding="utf-8"))
audit = audit_model_card(card)
assert payload["artifacts"][0]["path"] == "artifacts/model"
assert payload["backend_kwargs"]["model_dir"] == "artifacts/model"
assert audit.passed
def test_finalize_hf_recognizer_run_requires_finished_artifacts(tmp_path: Path) -> None:
run_dir = tmp_path / "run"
run_dir.mkdir()
with pytest.raises(DataValidationError, match="missing final model directory"):
finalize_hf_recognizer_run(
run_dir,
model_id="ocrtech-hf-mixed-v2-full",
base_model="microsoft/trocr-small-printed",
)
(run_dir / "final-model").mkdir()
with pytest.raises(DataValidationError, match="missing training metrics report"):
finalize_hf_recognizer_run(
run_dir,
model_id="ocrtech-hf-mixed-v2-full",
base_model="microsoft/trocr-small-printed",
)
def test_finalize_hf_recognizer_run_packages_audits_and_writes_report(tmp_path: Path) -> None:
run_dir = tmp_path / "run"
model_dir = run_dir / "final-model"
model_dir.mkdir(parents=True)
(model_dir / "config.json").write_text("{}", encoding="utf-8")
(model_dir / "preprocessor_config.json").write_text("{}", encoding="utf-8")
(run_dir / "training_metrics.json").write_text(json.dumps({"eval_cer": 0.1}, indent=2), encoding="utf-8")
training_summary = run_dir / "recognizer-training-summary.json"
training_summary.write_text('{"backend": "hf-vision-encoder-decoder"}\n', encoding="utf-8")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
image = tmp_path / "page.png"
image.write_bytes(b"image")
write_manifest([ManifestEntry(sample_id="train-1", dataset="local", split="train", image_path=str(image), text="नेपाल")], train_manifest)
write_manifest([ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(image), text="English")], eval_manifest)
result = finalize_hf_recognizer_run(
run_dir,
output_dir=tmp_path / "finalized",
model_id="ocrtech-hf-mixed-v2-full",
base_model="microsoft/trocr-small-printed",
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
)
assert result.audit_passed
assert Path(result.model_card_path).exists()
assert Path(result.audit_path).exists()
assert Path(result.report_json_path).exists()
assert Path(result.report_md_path).exists()
payload = json.loads(Path(result.report_json_path).read_text(encoding="utf-8"))
assert payload["audit_passed"] is True
model_card = json.loads(Path(result.model_card_path).read_text(encoding="utf-8"))
assert model_card["metrics"]["report"]["path"] == str(run_dir / "training_metrics.json")
assert model_card["provenance"]["training_summary"] == {
"path": str(training_summary),
"sha256": sha256_file(training_summary),
}
def test_benchmark_writes_report(tmp_path: Path) -> None:
source = tmp_path / "sample.txt"
source.write_text("hello world\n", encoding="utf-8")
source.with_suffix(".ref.txt").write_text("hello world\n", encoding="utf-8")
results = run_benchmark([source], tmp_path / "bench", baselines=["ours"])
assert len(results) == 1
assert results[0].status == "ok"
assert results[0].metrics["cer"] == 0
assert (tmp_path / "bench" / "report.json").exists()
def test_rescore_benchmark_recomputes_metrics_from_existing_documents(tmp_path: Path) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"image")
reference = tmp_path / "page.ref.json"
reference.write_text(json.dumps({"tables": [[["Name", "Value"], ["A", "1"]]]}), encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="",
metadata={"reference_path": str(reference), "slices": ["table"]},
)
],
manifest,
)
benchmark_dir = tmp_path / "bench"
output_dir = benchmark_dir / "candidate" / source.stem
output_dir.mkdir(parents=True)
document = Document(
source_path=str(source),
pages=[Page(page_index=0)],
tables=[
Table(table_id="p0-t0", page_index=0, bbox=BBox(0, 0, 20, 20), cells=[TableCell(row=0, col=0, text="wrong")]),
Table(
table_id="p0-t1",
page_index=0,
bbox=BBox(0, 30, 100, 40),
cells=[
TableCell(row=0, col=0, text="Name"),
TableCell(row=0, col=1, text="Value"),
TableCell(row=1, col=0, text="A"),
TableCell(row=1, col=1, text="1"),
],
),
],
)
(output_dir / "document.json").write_text(json.dumps(document.to_dict(), indent=2), encoding="utf-8")
report = benchmark_dir / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": str(source),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "sample-1",
"slices": ["table"],
"metrics": {"table_cell_f1": 0.0},
"error": None,
}
]
),
encoding="utf-8",
)
results = rescore_benchmark_report(report, tmp_path / "rescored", eval_manifest=manifest)
assert results[0].status == "ok"
assert results[0].metrics["table_cell_f1"] == 1.0
assert results[0].metrics["table_exact_match"] == 1.0
assert (tmp_path / "rescored" / "report.json").exists()
assert (tmp_path / "rescored" / "candidate" / source.stem / "document.json").exists()
def test_derive_table_postprocess_benchmark_scores_candidate_from_same_baseline_document(tmp_path: Path) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"image")
reference = tmp_path / "page.ref.json"
reference.write_text(json.dumps({"tables": [[["Name", "Value"], ["A", "1"]]]}), encoding="utf-8")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path=str(source),
text="",
metadata={"reference_path": str(reference), "slices": ["table"]},
)
],
manifest,
)
benchmark_dir = tmp_path / "bench"
output_dir = benchmark_dir / "surya" / source.stem
output_dir.mkdir(parents=True)
document = Document(
source_path=str(source),
pages=[Page(page_index=0)],
tables=[
Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(0, 0, 100, 40),
cells=[
TableCell(row=0, col=0, text="—"),
TableCell(row=0, col=1, text="Name"),
TableCell(row=0, col=2, text="Value"),
TableCell(row=0, col=3, text="Value"),
TableCell(row=1, col=0, text="—"),
TableCell(row=1, col=1, text="A"),
TableCell(row=1, col=2, text="1"),
TableCell(row=1, col=3, text="1"),
],
)
],
)
(output_dir / "document.json").write_text(json.dumps(document.to_dict(), indent=2), encoding="utf-8")
report = benchmark_dir / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "surya",
"input_path": str(source),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "sample-1",
"slices": ["table"],
"metrics": {"table_cell_f1": 0.0},
"error": None,
}
]
),
encoding="utf-8",
)
results = derive_table_postprocess_benchmark(report, tmp_path / "derived", source_baseline="surya", eval_manifest=manifest)
by_baseline = {result.baseline: result for result in results}
candidate_document = json.loads((tmp_path / "derived" / "candidate" / source.stem / "document.json").read_text(encoding="utf-8"))
assert by_baseline["surya"].metrics["table_cell_f1"] < 1.0
assert by_baseline["candidate"].metrics["table_cell_f1"] == 1.0
assert by_baseline["candidate"].metrics["table_exact_match"] == 1.0
assert candidate_document["metadata"]["derived_table_postprocess"]["source_baseline"] == "surya"
assert candidate_document["metadata"]["derived_table_postprocess"]["normalized_table_grid_dropped_columns"] == 2
def test_benchmark_external_command_materializes_document_from_markdown_stdout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"png")
source.with_suffix(".ref.json").write_text(
json.dumps(
{
"text": "नेपाल सरकार",
"tables": [[["नाम", "उमेर"], ["राम", "३२"]]],
},
ensure_ascii=False,
),
encoding="utf-8",
)
script = tmp_path / "emit_markdown.py"
script.write_text(
"import argparse\n"
"parser = argparse.ArgumentParser()\n"
"parser.add_argument('--input')\n"
"parser.add_argument('--out')\n"
"parser.parse_args()\n"
"print('# नेपाल सरकार\\n\\n| नाम | उमेर |\\n| --- | --- |\\n| राम | ३२ |')\n",
encoding="utf-8",
)
monkeypatch.setenv("OCRTECH_GLM_OCR_CMD", f"{sys.executable} {script} --input {{input}} --out {{out}}")
results = run_benchmark([source], tmp_path / "bench", baselines=["glm-ocr"])
document_path = tmp_path / "bench" / "glm-ocr" / "page" / "document.json"
assert results[0].status == "ok"
assert results[0].metrics["table_cell_f1"] == 1.0
assert document_path.exists()
payload = json.loads(document_path.read_text(encoding="utf-8"))
assert payload["tables"][0]["cells"][0]["text"] == "नाम"
assert (tmp_path / "bench" / "report.md").exists()
def test_benchmark_reuses_candidate_engine_per_baseline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
first = tmp_path / "a.txt"
second = tmp_path / "b.txt"
first.write_text("नेपाल\n", encoding="utf-8")
second.write_text("English\n", encoding="utf-8")
create_calls: list[tuple[str, object]] = []
class FakeEngine:
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(
text=input_path.read_text(encoding="utf-8").strip(),
bbox=BBox(0, 0, 10, 10),
confidence=0.99,
page_index=0,
line_id="p0-l0",
)
],
)
]
)
def fake_create_engine(name: str, **kwargs):
create_calls.append((name, kwargs.get("model_config")))
return FakeEngine()
monkeypatch.setattr("ocrtech.benchmark.create_engine", fake_create_engine)
results = run_benchmark([first, second], tmp_path / "bench", baselines=["candidate"], candidate_model_config="model.json")
assert len(results) == 2
assert all(result.status == "ok" for result in results)
assert len(create_calls) == 1
assert create_calls[0][0] == "candidate"
def test_benchmark_candidate_uses_fallback_parser_when_configured(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.txt"
source.write_text("नेपाल\n", encoding="utf-8")
source.with_suffix(".ref.txt").write_text("नेपाल\n", encoding="utf-8")
primary_model = tmp_path / "primary.json"
fallback_model = tmp_path / "fallback.json"
primary_model.write_text("{}", encoding="utf-8")
fallback_model.write_text("{}", encoding="utf-8")
calls: list[dict[str, object]] = []
def fake_parse_document(input_path: Path, output_dir: Path, **kwargs: object) -> Document:
calls.append({"input_path": input_path, "output_dir": output_dir, **kwargs})
output_dir.mkdir(parents=True)
document = Document(
source_path=str(input_path),
pages=[
Page(
page_index=0,
text_lines=[
TextLine(
text="नेपाल",
bbox=BBox(0, 0, 10, 10),
confidence=0.99,
page_index=0,
line_id="p0-l0",
)
],
)
],
metadata={
"quality": {"quality_score": 1.0, "metrics": {"line_count": 1, "low_confidence_line_count": 0}},
"fallback": {
"triggered": True,
"outcome": "success",
"primary_average_confidence": 0.42,
"threshold": 0.77,
},
},
)
(output_dir / "document.json").write_text(json.dumps(document.to_dict()), encoding="utf-8")
return document
monkeypatch.setattr("ocrtech.benchmark.parse_document", fake_parse_document)
results = run_benchmark(
[source],
tmp_path / "bench",
baselines=["candidate"],
candidate_model_config=primary_model,
fallback_model_config=fallback_model,
low_confidence_threshold=0.77,
)
assert results[0].status == "ok"
assert results[0].metrics["fallback_triggered"] == 1.0
assert results[0].metrics["fallback_succeeded"] == 1.0
report_md = (tmp_path / "bench" / "report.md").read_text(encoding="utf-8")
assert "fallback_failed" in report_md
assert "| candidate |" in report_md
assert "| 1 | 0 |" in report_md
assert calls[0]["engine_name"] == "candidate"
assert calls[0]["model_config"] == primary_model
assert calls[0]["fallback_model_config"] == fallback_model
assert calls[0]["low_confidence_threshold"] == 0.77
def test_benchmark_checkpoints_report_after_each_sample(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
first = tmp_path / "first.txt"
second = tmp_path / "second.txt"
first.write_text("one\n", encoding="utf-8")
second.write_text("two\n", encoding="utf-8")
first.with_suffix(".ref.txt").write_text("one\n", encoding="utf-8")
second.with_suffix(".ref.txt").write_text("two\n", encoding="utf-8")
output_dir = tmp_path / "bench"
class FakeEngine:
calls = 0
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
self.calls += 1
if self.calls == 2:
checkpoint = json.loads((output_dir / "report.json").read_text(encoding="utf-8"))
assert len(checkpoint) == 1
assert checkpoint[0]["input_path"] == str(first)
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(
text=input_path.read_text(encoding="utf-8").strip(),
bbox=BBox(0, 0, 10, 10),
confidence=0.99,
page_index=0,
line_id="p0-l0",
)
],
)
]
)
monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine())
results = run_benchmark([first, second], output_dir, baselines=["candidate"], candidate_model_config="model.json")
assert len(results) == 2
final_report = json.loads((output_dir / "report.json").read_text(encoding="utf-8"))
assert len(final_report) == 2
def test_benchmark_resume_existing_scores_existing_document_without_rerun(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
first = tmp_path / "first.txt"
second = tmp_path / "second.txt"
first.write_text("one\n", encoding="utf-8")
second.write_text("two\n", encoding="utf-8")
first.with_suffix(".ref.txt").write_text("one\n", encoding="utf-8")
second.with_suffix(".ref.txt").write_text("two\n", encoding="utf-8")
output_dir = tmp_path / "bench"
existing_dir = output_dir / "candidate" / first.stem
existing_dir.mkdir(parents=True)
existing_document = Document(
source_path=str(first),
pages=[
Page(
page_index=0,
text_lines=[
TextLine(
text="one",
bbox=BBox(0, 0, 10, 10),
confidence=0.99,
page_index=0,
line_id="p0-l0",
)
],
)
],
)
(existing_dir / "document.json").write_text(json.dumps(existing_document.to_dict(), ensure_ascii=False), encoding="utf-8")
engine_inputs: list[Path] = []
class FakeEngine:
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
engine_inputs.append(input_path)
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(
text=input_path.read_text(encoding="utf-8").strip(),
bbox=BBox(0, 0, 10, 10),
confidence=0.99,
page_index=0,
line_id="p0-l0",
)
],
)
]
)
monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine())
results = run_benchmark(
[first, second],
output_dir,
baselines=["candidate"],
candidate_model_config="model.json",
resume_existing=True,
)
assert [result.status for result in results] == ["ok", "ok"]
assert results[0].latency_seconds == 0.0
assert engine_inputs == [second]
report = json.loads((output_dir / "report.json").read_text(encoding="utf-8"))
assert [row["input_path"] for row in report] == [str(first), str(second)]
def test_benchmark_sample_timeout_records_error_and_continues(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
slow = tmp_path / "slow.txt"
fast = tmp_path / "fast.txt"
slow.write_text("slow\n", encoding="utf-8")
fast.write_text("fast\n", encoding="utf-8")
slow.with_suffix(".ref.txt").write_text("slow\n", encoding="utf-8")
fast.with_suffix(".ref.txt").write_text("fast\n", encoding="utf-8")
class FakeEngine:
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
if input_path == slow:
time.sleep(2)
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(
text=input_path.read_text(encoding="utf-8").strip(),
bbox=BBox(0, 0, 10, 10),
confidence=0.99,
page_index=0,
line_id="p0-l0",
)
],
)
]
)
monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine())
results = run_benchmark(
[slow, fast],
tmp_path / "bench",
baselines=["candidate"],
candidate_model_config="model.json",
sample_timeout_seconds=1,
)
assert results[0].status == "error"
assert "TimeoutError" in str(results[0].error)
assert results[1].status == "ok"
def test_benchmark_sample_timeout_uses_process_watchdog_when_available(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
if "fork" not in multiprocessing.get_all_start_methods():
pytest.skip("process watchdog requires fork start method")
source = tmp_path / "slow.txt"
source.write_text("slow\n", encoding="utf-8")
source.with_suffix(".ref.txt").write_text("slow\n", encoding="utf-8")
def fail_if_signal_timer_is_used(*args, **kwargs):
raise AssertionError("signal timer should not be used when fork watchdog is available")
class FakeEngine:
def recognize(self, input_path: Path):
time.sleep(2)
monkeypatch.setattr(signal, "setitimer", fail_if_signal_timer_is_used)
monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine())
results = run_benchmark(
[source],
tmp_path / "bench",
baselines=["candidate"],
candidate_model_config="model.json",
sample_timeout_seconds=1,
)
assert results[0].status == "error"
assert "TimeoutError" in str(results[0].error)
def test_benchmark_timeout_builds_engine_inside_watchdog_worker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
if "fork" not in multiprocessing.get_all_start_methods():
pytest.skip("process watchdog requires fork start method")
source = tmp_path / "sample.txt"
source.write_text("नेपाल\n", encoding="utf-8")
source.with_suffix(".ref.txt").write_text("नेपाल\n", encoding="utf-8")
pid_log = tmp_path / "engine-pids.txt"
parent_pid = os.getpid()
class FakeEngine:
def __init__(self) -> None:
with pid_log.open("a", encoding="utf-8") as handle:
handle.write(f"{os.getpid()}\n")
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(
text=input_path.read_text(encoding="utf-8").strip(),
bbox=BBox(0, 0, 10, 10),
confidence=0.99,
page_index=0,
line_id="p0-l0",
)
],
)
]
)
monkeypatch.setattr("ocrtech.benchmark.create_engine", lambda name, **kwargs: FakeEngine())
results = run_benchmark(
[source],
tmp_path / "bench",
baselines=["candidate"],
candidate_model_config="model.json",
sample_timeout_seconds=5,
)
assert results[0].status == "ok"
engine_pids = [int(line) for line in pid_log.read_text(encoding="utf-8").splitlines()]
assert engine_pids
assert parent_pid not in engine_pids
def test_benchmark_supports_surya_baseline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "sample.txt"
source.write_text("नेपाल\n", encoding="utf-8")
source.with_suffix(".ref.txt").write_text("नेपाल\n", encoding="utf-8")
create_calls: list[str] = []
class FakeEngine:
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=0.88, page_index=0, line_id="p0-l0")],
metadata={"surya_mode": "block"},
)
],
metadata={"surya_mode": "block"},
)
def fake_create_engine(name: str, **kwargs):
create_calls.append(name)
return FakeEngine()
monkeypatch.setattr("ocrtech.benchmark.create_engine", fake_create_engine)
results = run_benchmark([source], tmp_path / "bench", baselines=["surya"])
assert len(results) == 1
assert results[0].status == "ok"
assert results[0].metrics["cer"] == 0
assert create_calls == ["surya"]
def test_summarize_benchmark_report_writes_paired_and_aggregate_metrics(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": "a.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["english"],
"metrics": {"cer": 0.10, "quality_score": 0.90},
"error": None,
},
{
"baseline": "candidate",
"input_path": "b.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s2",
"slices": ["nepali"],
"metrics": {"cer": 0.40, "quality_score": 0.80},
"error": None,
},
{
"baseline": "tesseract",
"input_path": "a.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["english"],
"metrics": {"cer": 0.20, "quality_score": 0.70},
"error": None,
},
{
"baseline": "tesseract",
"input_path": "b.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s2",
"slices": ["nepali"],
"metrics": {"cer": 0.30, "quality_score": 0.75},
"error": None,
},
],
indent=2,
),
encoding="utf-8",
)
summary = summarize_benchmark_report(report, tmp_path / "summary", candidate="candidate", baselines=["tesseract"], metrics=["cer"])
assert (tmp_path / "summary" / "summary.json").exists()
assert (tmp_path / "summary" / "summary.md").exists()
assert summary.aggregate_metrics["candidate"]["all"]["sample_count"] == 2
assert summary.aggregate_metrics["candidate"]["all"]["latency_seconds"] == pytest.approx(1.0)
english = next(item for item in summary.paired_metrics if item.baseline == "tesseract" and item.slice_name == "english")
nepali = next(item for item in summary.paired_metrics if item.baseline == "tesseract" and item.slice_name == "nepali")
assert english.absolute_improvement == pytest.approx(0.10)
assert english.win_count == 1
assert nepali.absolute_improvement == pytest.approx(-0.10)
assert nepali.loss_count == 1
def test_calibrate_quality_router_writes_model_card_with_calibration_provenance(tmp_path: Path) -> None:
eval_manifest = tmp_path / "eval.jsonl"
eval_image = tmp_path / "sample.txt"
eval_image.write_text("नेपाल\n", encoding="utf-8")
write_manifest(
[ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(eval_image), text="नेपाल", metadata={"slices": ["nepali"]})],
eval_manifest,
)
benchmark_report = tmp_path / "report.json"
benchmark_report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": str(eval_image),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.20, "quality_score": 0.95},
"error": None,
},
{
"baseline": "tesseract",
"input_path": str(eval_image),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["nepali"],
"metrics": {"cer": 0.40, "quality_score": 0.70},
"error": None,
},
],
indent=2,
),
encoding="utf-8",
)
experiment_report = tmp_path / "experiment.json"
experiment_report.write_text(
json.dumps(
{
"experiment_id": "calibration",
"output_dir": str(tmp_path / "exp"),
"eval_manifest": str(eval_manifest),
"references_dir": None,
"baselines": ["candidate", "tesseract"],
"input_count": 1,
"datasets": ["fixture"],
"slice_counts": {"nepali": 1},
"benchmark_report": str(benchmark_report),
"benchmark_status_counts": {"ok": 2},
"validation_report": None,
"validation_status": None,
"environment": {},
},
indent=2,
),
encoding="utf-8",
)
calibration, model_path = calibrate_quality_router(
experiment_report,
tmp_path / "calibrated",
model_id="ocrtech-quality-router",
primary_engine="tesseract",
secondary_engine="tesseract",
primary_engine_kwargs={"psm": 6},
secondary_engine_kwargs={"psm": 13},
threshold_start=0.0,
threshold_end=0.1,
threshold_step=0.05,
)
payload = json.loads(model_path.read_text(encoding="utf-8"))
assert (tmp_path / "calibrated" / "calibration.json").exists()
assert calibration.selected_threshold == 0.1
assert payload["backend"] == "quality_select_composite"
assert payload["provenance"]["calibration"]["eval_manifest"]["path"] == str(eval_manifest)
def test_calibrate_tesseract_psm_ensemble_writes_ranked_model_card(tmp_path: Path) -> None:
eval_manifest = tmp_path / "eval.jsonl"
sample = tmp_path / "sample.txt"
sample.write_text("नेपाल\n", encoding="utf-8")
write_manifest(
[ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(sample), text="नेपाल", metadata={"slices": ["nepali"]})],
eval_manifest,
)
def write_experiment(path: Path, benchmark_path: Path, rows: list[dict[str, object]]) -> Path:
benchmark_path.write_text(json.dumps(rows, indent=2), encoding="utf-8")
payload = {
"experiment_id": path.stem,
"output_dir": str(path.parent / path.stem),
"eval_manifest": str(eval_manifest),
"references_dir": None,
"baselines": ["candidate", "tesseract"],
"input_count": 1,
"datasets": ["fixture"],
"slice_counts": {"nepali": 1},
"benchmark_report": str(benchmark_path),
"benchmark_status_counts": {"ok": len(rows)},
"validation_report": None,
"validation_status": None,
"environment": {},
}
path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
return path
primary = write_experiment(
tmp_path / "primary-experiment.json",
tmp_path / "primary-report.json",
[
{"baseline": "tesseract", "input_path": str(sample), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.4, "quality_score": 0.7}, "error": None}
],
)
psm7 = write_experiment(
tmp_path / "psm7-experiment.json",
tmp_path / "psm7-report.json",
[
{"baseline": "candidate", "input_path": str(sample), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.3, "quality_score": 0.6}, "error": None}
],
)
psm13 = write_experiment(
tmp_path / "psm13-experiment.json",
tmp_path / "psm13-report.json",
[
{"baseline": "candidate", "input_path": str(sample), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.2, "quality_score": 0.9}, "error": None}
],
)
calibration, model_path = calibrate_tesseract_psm_ensemble(
primary,
tmp_path / "ensemble",
model_id="ocrtech-psm-ensemble",
variant_experiment_reports={"psm7": psm7, "psm13": psm13},
bias_start=-0.04,
bias_end=0.04,
bias_step=0.04,
)
payload = json.loads(model_path.read_text(encoding="utf-8"))
assert (tmp_path / "ensemble" / "calibration.json").exists()
assert payload["backend"] == "quality_ranked_ensemble"
assert payload["backend_kwargs"]["alternative_engines"][0]["engine"] == "tesseract"
assert calibration.selected_biases["psm13"] >= calibration.selected_biases["psm7"]
def test_calibrate_script_router_writes_model_card(tmp_path: Path) -> None:
eval_manifest = tmp_path / "eval.jsonl"
sample_nep = tmp_path / "nep.png"
sample_eng = tmp_path / "eng.png"
sample_nep.write_bytes(b"nep")
sample_eng.write_bytes(b"eng")
write_manifest(
[
ManifestEntry(sample_id="s1", dataset="fixture", split="eval", image_path=str(sample_nep), text="नेपाल", metadata={"slices": ["nepali"]}),
ManifestEntry(sample_id="s2", dataset="fixture", split="eval", image_path=str(sample_eng), text="hello", metadata={"slices": ["english"]}),
],
eval_manifest,
)
output_dir = tmp_path / "experiment"
(output_dir / "benchmark" / "tesseract" / sample_nep.stem).mkdir(parents=True)
(output_dir / "benchmark" / "tesseract" / sample_eng.stem).mkdir(parents=True)
(output_dir / "benchmark" / "tesseract" / sample_nep.stem / "document.md").write_text("नेपाल सरकार\n", encoding="utf-8")
(output_dir / "benchmark" / "tesseract" / sample_eng.stem / "document.md").write_text("hello world\n", encoding="utf-8")
benchmark_report = tmp_path / "report.json"
benchmark_report.write_text(
json.dumps(
[
{"baseline": "tesseract", "input_path": str(sample_nep), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.40}, "error": None},
{"baseline": "surya", "input_path": str(sample_nep), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["nepali"], "metrics": {"cer": 0.20}, "error": None},
{"baseline": "tesseract", "input_path": str(sample_eng), "status": "ok", "latency_seconds": 1.0, "sample_id": "s2", "slices": ["english"], "metrics": {"cer": 0.10}, "error": None},
{"baseline": "surya", "input_path": str(sample_eng), "status": "ok", "latency_seconds": 1.0, "sample_id": "s2", "slices": ["english"], "metrics": {"cer": 0.50}, "error": None},
],
indent=2,
),
encoding="utf-8",
)
experiment_report = tmp_path / "experiment.json"
experiment_report.write_text(
json.dumps(
{
"experiment_id": "script-calibration",
"output_dir": str(output_dir),
"eval_manifest": str(eval_manifest),
"references_dir": None,
"baselines": ["surya", "tesseract"],
"input_count": 2,
"datasets": ["fixture"],
"slice_counts": {"nepali": 1, "english": 1},
"benchmark_report": str(benchmark_report),
"benchmark_status_counts": {"ok": 4},
"validation_report": None,
"validation_status": None,
"environment": {},
},
indent=2,
),
encoding="utf-8",
)
calibration, model_path = calibrate_script_router(
experiment_report,
tmp_path / "script-router",
model_id="ocrtech-script-router",
primary_engine="tesseract",
secondary_engine="surya",
threshold_start=0.0,
threshold_end=0.5,
threshold_step=0.25,
routing_granularity="page",
secondary_structure_backfill=True,
)
payload = json.loads(model_path.read_text(encoding="utf-8"))
assert (tmp_path / "script-router" / "calibration.json").exists()
assert calibration.selected_threshold in {0.0, 0.25}
assert payload["backend"] == "script_select_composite"
assert payload["backend_kwargs"]["primary_script_threshold"] == calibration.selected_threshold
assert payload["backend_kwargs"]["routing_granularity"] == "page"
assert payload["backend_kwargs"]["secondary_structure_backfill"] is True
def test_calibrate_script_router_supports_latin_script(tmp_path: Path) -> None:
output_dir = tmp_path / "outputs"
(output_dir / "benchmark" / "surya" / "doc-nepali").mkdir(parents=True)
(output_dir / "benchmark" / "surya" / "doc-english").mkdir(parents=True)
(output_dir / "benchmark" / "surya" / "doc-nepali" / "document.md").write_text("नेपाल सरकार\n", encoding="utf-8")
(output_dir / "benchmark" / "surya" / "doc-english" / "document.md").write_text("Invoice Total\n", encoding="utf-8")
benchmark_report = output_dir / "benchmark" / "report.json"
benchmark_report.write_text(
json.dumps(
[
{"baseline": "surya", "input_path": "doc-nepali.png", "sample_id": "doc-nepali", "status": "ok", "metrics": {"cer": 0.1}},
{"baseline": "tesseract", "input_path": "doc-nepali.png", "sample_id": "doc-nepali", "status": "ok", "metrics": {"cer": 0.4}},
{"baseline": "surya", "input_path": "doc-english.png", "sample_id": "doc-english", "status": "ok", "metrics": {"cer": 0.4}},
{"baseline": "tesseract", "input_path": "doc-english.png", "sample_id": "doc-english", "status": "ok", "metrics": {"cer": 0.1}},
],
indent=2,
),
encoding="utf-8",
)
eval_manifest = tmp_path / "eval.jsonl"
eval_manifest.write_text(
"\n".join(
[
json.dumps({"sample_id": "doc-nepali", "dataset": "fixture", "split": "eval", "image_path": "doc-nepali.png", "text": "नेपाल सरकार"}),
json.dumps({"sample_id": "doc-english", "dataset": "fixture", "split": "eval", "image_path": "doc-english.png", "text": "Invoice Total"}),
]
)
+ "\n",
encoding="utf-8",
)
experiment_report = tmp_path / "experiment.json"
experiment_report.write_text(
json.dumps(
{
"experiment_id": "script-calibration-latin",
"output_dir": str(output_dir),
"eval_manifest": str(eval_manifest),
"references_dir": None,
"baselines": ["surya", "tesseract"],
"input_count": 2,
"datasets": ["fixture"],
"slice_counts": {"nepali": 1, "english": 1},
"benchmark_report": str(benchmark_report),
"benchmark_status_counts": {"ok": 4},
"validation_report": None,
"validation_status": None,
"environment": {},
},
indent=2,
),
encoding="utf-8",
)
calibration, model_path = calibrate_script_router(
experiment_report,
tmp_path / "script-router-latin",
model_id="ocrtech-script-router-latin",
primary_engine="surya",
secondary_engine="tesseract",
primary_baseline="surya",
secondary_baseline="tesseract",
script="latin",
threshold_start=0.0,
threshold_end=0.5,
threshold_step=0.25,
)
payload = json.loads(model_path.read_text(encoding="utf-8"))
assert calibration.selected_threshold in {0.0, 0.25}
assert payload["backend_kwargs"]["script"] == "latin"
assert payload["backend_kwargs"]["primary_engine"] == "surya"
assert payload["backend_kwargs"]["secondary_engine"] == "tesseract"
def _stopped_ppocrv3_failure_review(tmp_path: Path) -> Path:
review = tmp_path / "recognizer-failure-review.json"
review.write_text(
json.dumps(
{
"families": [
{
"backend": "paddleocr",
"base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec",
"family_key": "paddleocr::PaddlePaddle/devanagari_PP-OCRv3_mobile_rec",
"recommendation": "stop_family",
"rejected_count": 4,
"total_wins": 0,
"total_losses": 545,
"total_ties": 7,
}
]
}
)
+ "\n",
encoding="utf-8",
)
return review
def _passing_validation_gate() -> dict[str, object]:
return {
"passed": True,
"pairs": 2,
"candidate_mean": 0.1,
"baseline_mean": 0.2,
"absolute_improvement": 0.1,
"relative_improvement": 0.5,
"improvement_ci_low": 0.01,
"win_count": 2,
"loss_count": 0,
"tie_count": 0,
"pair_win_rate": 1.0,
"sign_test_p_value": 0.0,
"gate": {
"baseline": "tesseract",
"metric": "cer",
"min_pairs": 2,
"min_pair_win_rate": 0.5,
"require_ci_above_zero": True,
},
}
from __future__ import annotations
import json
import multiprocessing
import os
import signal
import sys
import time
from pathlib import Path
from types import SimpleNamespace
import pytest
from ocrtech.benchmark import derive_table_postprocess_benchmark, rescore_benchmark_report, run_benchmark, summarize_benchmark_report
from ocrtech.calibration import calibrate_quality_router, calibrate_script_router, calibrate_tesseract_psm_ensemble
from ocrtech.errors import DataValidationError, ValidationError
from ocrtech.manifest import ManifestEntry, sha256_file, write_manifest
from ocrtech.models import ModelCard, audit_model_card, model_admission_status
from ocrtech.schemas import BBox, Document, Page, Table, TableCell, TextLine
from ocrtech.training import (
_hf_text_corrector_metrics_factory,
bundle_recognizer_training,
finalize_hf_recognizer_run,
generate_correction_pairs,
package_hf_text_corrector_model,
package_hf_recognizer_model,
package_paddleocr_model,
run_hf_recognizer_training,
write_corrector_recipe,
write_hf_text_corrector_recipe,
write_hf_recognizer_recipe,
write_recognizer_export_recipe,
write_recognizer_recipe,
)
def test_train_recognizer_writes_paddle_recipe(tmp_path: Path) -> None:
train_image = tmp_path / "train.png"
eval_image = tmp_path / "eval.png"
train_image.write_bytes(b"train")
eval_image.write_bytes(b"eval")
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="train-1",
dataset="local",
split="train",
image_path=str(train_image),
text="नेपाल English",
metadata={"slices": ["nepali", "english"]},
)
],
train_manifest,
)
write_manifest(
[ManifestEntry(sample_id="eval-1", dataset="local", split="eval", image_path=str(eval_image), text="काठमाडौं")],
eval_manifest,
)
config = write_recognizer_recipe(
train_manifest,
eval_manifest,
tmp_path / "run",
train_batch_size=4,
eval_batch_size=6,
learning_rate=0.0003,
epochs=2.5,
)
assert config.exists()
assert (tmp_path / "run" / "labels" / "train.txt").exists()
config_text = config.read_text(encoding="utf-8")
assert "Architecture:" in config_text
assert "algorithm: SVTR_LCNet" in config_text
assert "Loss:" in config_text
assert "PostProcess:" in config_text
assert "Metric:" in config_text
assert "character_dict_path" in config_text
assert "batch_size_per_card: 4" in config_text
assert "batch_size_per_card: 6" in config_text
assert "name: Cosine" in config_text
assert "learning_rate: 0.0003" in config_text
assert "epoch_num: 2.5" in config_text
dictionary = (tmp_path / "run" / "nep_eng_dict.txt").read_text(encoding="utf-8")
assert "क" in dictionary
assert "E" in dictionary
summary = json.loads((tmp_path / "run" / "recognizer-training-summary.json").read_text(encoding="utf-8"))
assert summary["backend"] == "paddleocr"
assert summary["manifests"]["train"]["sample_count"] == 1
assert summary["manifests"]["train"]["slice_counts"] == {"english": 1, "nepali": 1}
assert summary["manifests"]["train"]["contains_ascii"] is True
assert summary["manifests"]["train"]["contains_devanagari"] is True
assert summary["artifacts"]["dictionary"]["contains_ascii"] is True
assert summary["artifacts"]["dictionary"]["contains_devanagari"] is True
assert summary["artifacts"]["config"]["sha256"]
def test_train_recognizer_writes_integral_epochs_as_integer(tmp_path: Path) -> None: