from __future__ import annotations
import json
import importlib.util
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from ocrtech.benchmark import run_benchmark
from ocrtech.experiments import run_experiment
from ocrtech.manifest import sha256_file
from ocrtech.models import ModelCard, audit_model_card, component_baseline_names, inspect_routed_calibration_collapse, model_admission_status, trained_recognizer_provenance_status, write_model_card
from ocrtech.pipeline import parse_document
def test_model_card_audit_passes_for_existing_artifact(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-test",
backend="sidecar",
base_model="fixture",
artifact_paths=[artifact],
notes="test model card",
)
audit = audit_model_card(card, tmp_path / "audit")
assert audit.passed
assert audit.model_id == "ocrtech-test"
assert (tmp_path / "audit" / "model-audit.json").exists()
def test_model_card_records_and_checks_admission_validation(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
validation = tmp_path / "validation.json"
model_path = tmp_path / "model.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(model_path)}},
"gates": [_passing_validation_gate()],
}
)
+ "\n",
encoding="utf-8",
)
card_path = write_model_card(
model_path,
model_id="ocrtech-admitted",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
admission_validation_report=validation,
)
card = ModelCard.from_path(card_path)
status = model_admission_status(card, card_path)
assert status.passed
assert card.metrics["admission_validation"]["sha256"]
def test_model_admission_status_requires_quantitative_gate_evidence(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
validation = tmp_path / "validation.json"
model_path = tmp_path / "model.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(model_path)}},
"gates": [],
}
)
+ "\n",
encoding="utf-8",
)
card_path = write_model_card(
model_path,
model_id="ocrtech-admission-without-gates",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
admission_validation_report=validation,
)
status = model_admission_status(ModelCard.from_path(card_path), card_path)
assert not status.passed
assert "admission validation missing gates" in status.details
def test_model_card_records_fallback_model_card_provenance(tmp_path: Path) -> None:
fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
candidate = write_model_card(
tmp_path / "candidate.json",
model_id="ocrtech-candidate",
backend="sidecar",
base_model="fixture",
fallback_model_card=fallback,
)
payload = json.loads(candidate.read_text(encoding="utf-8"))
assert payload["provenance"]["fallback_model_card"] == {
"path": str(fallback),
"sha256": sha256_file(fallback),
}
def test_trained_recognizer_provenance_requires_training_summary(tmp_path: Path) -> None:
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
image = tmp_path / "line.png"
image.write_bytes(b"line")
row = {
"sample_id": "s1",
"dataset": "fixture",
"split": "train",
"image_path": str(image),
"text": "नेपाल",
}
train_manifest.write_text(json.dumps(row, ensure_ascii=False) + "\n", encoding="utf-8")
eval_manifest.write_text(json.dumps({**row, "split": "eval"}, ensure_ascii=False) + "\n", encoding="utf-8")
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
card_path = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-trained-missing-summary",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
)
status = trained_recognizer_provenance_status(ModelCard.from_path(card_path), card_path)
assert not status.passed
assert "trained recognizer missing training_summary path/sha256 provenance" in status.details
def test_trained_recognizer_provenance_rejects_training_summary_hash_mismatch(tmp_path: Path) -> None:
train_manifest = tmp_path / "train.jsonl"
eval_manifest = tmp_path / "eval.jsonl"
training_summary = tmp_path / "recognizer-training-summary.json"
image = tmp_path / "line.png"
image.write_bytes(b"line")
row = {
"sample_id": "s1",
"dataset": "fixture",
"split": "train",
"image_path": str(image),
"text": "नेपाल",
}
train_manifest.write_text(json.dumps(row, ensure_ascii=False) + "\n", encoding="utf-8")
eval_manifest.write_text(json.dumps({**row, "split": "eval"}, ensure_ascii=False) + "\n", encoding="utf-8")
training_summary.write_text('{"backend": "paddleocr", "epochs": 1}\n', encoding="utf-8")
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
card_path = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-trained-stale-summary",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
train_manifest=train_manifest,
eval_manifest=eval_manifest,
training_summary=training_summary,
)
training_summary.write_text('{"backend": "paddleocr", "epochs": 2}\n', encoding="utf-8")
status = trained_recognizer_provenance_status(ModelCard.from_path(card_path), card_path)
assert not status.passed
assert any("trained recognizer training_summary sha256 mismatch" in detail for detail in status.details)
def test_model_admission_status_fails_on_not_validated_report(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
validation = tmp_path / "validation.json"
model_path = tmp_path / "model.json"
validation.write_text(
json.dumps(
{
"claim_status": "not_validated",
"passed": False,
"provenance": {"candidate_model_config": {"path": str(model_path)}},
"missing_requirements": ["gate failed: cer vs tesseract"],
"gates": [
{
"gate": {
"baseline": "tesseract",
"metric": "cer",
"direction": "lower",
"slice": "nepali",
},
"passed": False,
"pairs": 12,
"candidate_mean": 0.908108,
"baseline_mean": 0.235433,
"absolute_improvement": -0.672676,
"relative_improvement": -2.857191,
"win_count": 0,
"loss_count": 12,
"tie_count": 0,
"reason": "absolute improvement -0.672676 < required 0.0",
}
],
}
)
+ "\n",
encoding="utf-8",
)
card_path = write_model_card(
model_path,
model_id="ocrtech-rejected",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
admission_validation_report=validation,
)
status = model_admission_status(ModelCard.from_path(card_path), card_path)
assert not status.passed
assert any("claim_status=not_validated" in detail for detail in status.details)
assert any("missing_requirement=gate failed: cer vs tesseract" in detail for detail in status.details)
assert any(
"failed_gate=cer baseline=tesseract slice=nepali" in detail
and "candidate_mean=0.908108" in detail
and "loss_count=12" in detail
for detail in status.details
)
def test_model_admission_status_rejects_report_for_different_model(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
other_artifact = tmp_path / "other.pdiparams"
other_artifact.write_bytes(b"other weights")
other_model = write_model_card(
tmp_path / "other-model.json",
model_id="ocrtech-other",
backend="paddleocr",
base_model="fixture",
artifact_paths=[other_artifact],
backend_kwargs={"lang": "ne"},
)
validation = tmp_path / "validation.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(other_model)}},
"gates": [_passing_validation_gate()],
}
)
+ "\n",
encoding="utf-8",
)
card_path = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-current",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
admission_validation_report=validation,
)
status = model_admission_status(ModelCard.from_path(card_path), card_path)
assert not status.passed
assert any("model_id mismatch" in detail for detail in status.details)
def test_model_card_audit_fails_for_missing_artifact(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-missing",
backend="sidecar",
base_model="fixture",
artifact_paths=[tmp_path / "missing.pdmodel"],
)
audit = audit_model_card(card)
assert not audit.passed
assert "required artifact missing" in audit.issues[0]
def test_nested_model_kwargs_resolve_relative_paths_from_model_card(tmp_path: Path) -> None:
from ocrtech.engines import _resolve_model_kwargs
config_dir = tmp_path / "candidate"
config_dir.mkdir()
card_path = config_dir / "model.json"
artifacts = config_dir / "artifacts"
artifacts.mkdir()
card = write_model_card(
card_path,
model_id="ocrtech-nested-paths",
backend="script_select_composite",
base_model="fixture",
backend_kwargs={
"primary_engine": "tesseract",
"secondary_engine": "paddleocr",
"secondary_engine_kwargs": {
"text_recognition_model_dir": "artifacts",
"metadata": {"dict_path": "artifacts/nep_eng_dict.txt"},
},
},
)
resolved = _resolve_model_kwargs(ModelCard.from_path(card), card)
secondary = resolved["secondary_engine_kwargs"]
assert isinstance(secondary, dict)
assert secondary["text_recognition_model_dir"] == str(artifacts)
metadata = secondary["metadata"]
assert isinstance(metadata, dict)
assert metadata["dict_path"] == str(artifacts / "nep_eng_dict.txt")
def test_paddle_candidate_requires_recorded_artifact(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-no-artifacts",
backend="paddleocr",
base_model="fixture",
paddleocr_kwargs={"lang": "devanagari"},
)
audit = audit_model_card(card)
assert not audit.passed
assert "requires at least one recorded artifact" in audit.issues[0]
def test_hf_candidate_requires_recorded_artifact(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-hf-no-artifacts",
backend="hf_vision_encoder_decoder",
base_model="microsoft/trocr-small-printed",
)
audit = audit_model_card(card)
assert not audit.passed
assert "requires at least one recorded artifact" in audit.issues[0]
def test_tesseract_candidate_audit_passes_without_artifacts(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-tess",
backend="tesseract",
base_model="tesseract-ocr",
backend_kwargs={"language": "nep+eng", "psm": 7},
)
audit = audit_model_card(card)
assert audit.passed
assert audit.backend == "tesseract"
def test_surya_candidate_audit_passes_without_artifacts(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-surya",
backend="surya",
base_model="datalab-to/surya-ocr-2",
backend_kwargs={"mode": "block"},
)
audit = audit_model_card(card)
assert audit.passed
assert audit.backend == "surya"
def test_quality_select_candidate_audit_requires_named_engines(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-quality-router",
backend="quality_select_composite",
base_model="tesseract-ocr",
backend_kwargs={"primary_engine": "tesseract"},
)
audit = audit_model_card(card)
assert not audit.passed
assert any("secondary_engine" in issue for issue in audit.issues)
def test_quality_ranked_ensemble_audit_requires_alternatives(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-quality-ensemble",
backend="quality_ranked_ensemble",
base_model="tesseract-ocr",
backend_kwargs={"primary_engine": "tesseract"},
)
audit = audit_model_card(card)
assert not audit.passed
assert any("alternative_engines" in issue for issue in audit.issues)
def test_script_select_candidate_audit_requires_named_engines(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={"primary_engine": "tesseract"},
)
audit = audit_model_card(card)
assert not audit.passed
assert any("secondary_engine" in issue for issue in audit.issues)
def test_line_align_candidate_audit_requires_named_engines(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-line-align",
backend="line_align_composite",
base_model="surya+tesseract",
backend_kwargs={"primary_engine": "surya"},
)
audit = audit_model_card(card)
assert not audit.passed
assert any("secondary_engine" in issue for issue in audit.issues)
def test_table_cell_refine_candidate_audit_passes_with_defaults(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-table-refine",
backend="table_cell_refine_composite",
base_model="surya+tesseract",
backend_kwargs={"structure_engine": "surya", "crop_engine": "tesseract"},
)
audit = audit_model_card(card)
assert audit.passed
assert audit.backend == "table_cell_refine_composite"
def test_table_cell_refine_audit_requires_candidate_crop_model_config(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-table-refine-candidate-missing-config",
backend="table_cell_refine_composite",
base_model="surya+trained-candidate",
backend_kwargs={"structure_engine": "surya", "crop_engine": "candidate"},
)
audit = audit_model_card(card)
assert not audit.passed
assert "table_cell_refine_composite crop_engine=candidate requires backend_kwargs.crop_engine_kwargs.model_config" in audit.issues
def test_figure_caption_refine_candidate_audit_passes_with_defaults(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={"primary_engine": "surya", "caption_engine": "tesseract"},
)
audit = audit_model_card(card)
assert audit.passed
assert audit.backend == "figure_caption_refine_composite"
def test_figure_caption_refine_audit_requires_candidate_caption_model_config(tmp_path: Path) -> None:
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-candidate-missing-config",
backend="figure_caption_refine_composite",
base_model="surya+trained-candidate",
backend_kwargs={"primary_engine": "surya", "caption_engine": "candidate"},
)
audit = audit_model_card(card)
assert not audit.passed
assert "figure_caption_refine_composite caption_engine=candidate requires backend_kwargs.caption_engine_kwargs.model_config" in audit.issues
def test_component_baseline_names_normalizes_component_engines(tmp_path: Path) -> None:
card = ModelCard.from_path(
write_model_card(
tmp_path / "model.json",
model_id="ocrtech-quality-ensemble",
backend="quality_ranked_ensemble",
base_model="tesseract+paddle",
backend_kwargs={
"primary_engine": "tesseract",
"alternative_engines": [
{"engine": "paddleocr", "bias": 0.1},
{"engine": "surya", "bias": 0.2},
],
},
)
)
names = component_baseline_names(card)
assert names == ["stock-paddle", "surya", "tesseract"]
def test_inspect_routed_calibration_collapse_accepts_repo_relative_metrics_report(tmp_path: Path, monkeypatch) -> None:
monkeypatch.chdir(tmp_path)
report = tmp_path / "outputs" / "script-router" / "calibration.json"
report.parent.mkdir(parents=True)
report.write_text(
'{"selected_threshold":0.0,"metric":"cer","selected_trial":{"pairs":5,"secondary_picks":5}}' + "\n",
encoding="utf-8",
)
model_dir = tmp_path / "models"
model_dir.mkdir()
card = write_model_card(
model_dir / "model.json",
model_id="ocrtech-script-router",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.0},
metrics_report=Path("outputs/script-router/calibration.json"),
)
reason, details = inspect_routed_calibration_collapse(ModelCard.from_path(card), card)
assert reason is not None
assert "collapses to the secondary engine" in reason
assert any("report=" in item for item in details)
def test_candidate_engine_uses_explicit_model_config(tmp_path: Path) -> None:
source = tmp_path / "page.txt"
source.write_text("नेपाल सरकार\nhello world\n", encoding="utf-8")
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["model_id"] == "ocrtech-sidecar"
assert document.metadata["quality"]["passed"]
def test_candidate_engine_supports_hf_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
model_dir = tmp_path / "model"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-hf",
backend="hf_vision_encoder_decoder",
base_model="microsoft/trocr-small-printed",
artifact_paths=[model_dir],
backend_kwargs={"model_dir": str(model_dir), "processor_dir": str(model_dir)},
)
class FakeEngine:
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=None, page_index=0, line_id="p0-l0")])])
monkeypatch.setattr("ocrtech.engines.HuggingFaceVisionTextEngine", FakeEngine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "hf_vision_encoder_decoder"
assert document.text == "नेपाल"
def test_candidate_engine_supports_tesseract_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-tess",
backend="tesseract",
base_model="tesseract-ocr",
backend_kwargs={"language": "nep+eng", "psm": 7},
)
class FakeEngine:
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=0.95, page_index=0, line_id="p0-l0")])])
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeEngine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "tesseract"
assert document.text == "नेपाल"
def test_candidate_engine_supports_surya_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-surya",
backend="surya",
base_model="datalab-to/surya-ocr-2",
backend_kwargs={"mode": "block"},
)
class FakeEngine:
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=0.91, page_index=0, line_id="p0-l0")],
metadata={"surya_mode": "block"},
)
],
metadata={"surya_mode": "block"},
)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeEngine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "surya"
assert document.text == "नेपाल"
def test_surya_engine_converts_table_html_to_structured_table() -> None:
from ocrtech.engines import SuryaEngine
engine = object.__new__(SuryaEngine)
page_result = SimpleNamespace(
blocks=[
SimpleNamespace(
polygon=[[0.0, 0.0], [120.0, 0.0], [120.0, 60.0], [0.0, 60.0]],
confidence=0.88,
label="Table",
raw_label="Table",
reading_order=0,
html="",
skipped=False,
error=False,
),
SimpleNamespace(
polygon=[[0.0, 70.0], [120.0, 70.0], [120.0, 90.0], [0.0, 90.0]],
confidence=0.93,
label="Text",
raw_label="Text",
reading_order=1,
html="
नेपाल सरकार
",
skipped=False,
error=False,
),
]
)
table_result = SimpleNamespace(
html="",
mode="full",
error=False,
rows=[],
cols=[],
)
output = engine._convert_page_result(page_result, 120, 90, "full_page", page_index=0, table_results={0: table_result})
assert len(output.tables) == 1
assert output.tables[0].html.startswith("")
assert [cell.text for cell in output.tables[0].cells] == ["नाम", "उमेर", "राम", "३२"]
assert output.tables[0].metadata["surya_table_mode"] == "full"
assert output.pages[0].text_lines[0].text == "नेपाल सरकार"
def test_candidate_engine_supports_text_correction_composite_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
model_dir = tmp_path / "model"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-corrected",
backend="text_correction_composite",
base_model="google/byt5-small",
artifact_paths=[model_dir],
backend_kwargs={"model_dir": str(model_dir), "tokenizer_dir": str(model_dir), "base_engine": "tesseract"},
)
class FakeEngine:
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="सही पाठ", bbox=BBox(0, 0, 10, 10), confidence=0.9, page_index=0, line_id="p0-l0")])])
monkeypatch.setattr("ocrtech.engines.TextCorrectionCompositeEngine", FakeEngine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "text_correction_composite"
assert document.text == "सही पाठ"
def test_candidate_engine_supports_quality_select_composite_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-quality-router",
backend="quality_select_composite",
base_model="tesseract-ocr",
backend_kwargs={
"primary_engine": "tesseract",
"secondary_engine": "tesseract",
"primary_engine_kwargs": {"psm": 6},
"secondary_engine_kwargs": {"psm": 13},
"secondary_quality_margin": 0.02,
},
)
class FakeEngine:
def __init__(self, *, psm: int, **kwargs):
self.name = f"tesseract-psm-{psm}"
self.psm = psm
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
confidence = 0.80 if self.psm == 6 else 1.0
text = "default" if self.psm == 6 else "routed"
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text=text, bbox=BBox(0, 0, 10, 10), confidence=confidence, page_index=0, line_id="p0-l0")])],
metadata={"psm": self.psm},
)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeEngine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "quality_select_composite"
assert document.text == "routed"
routing = document.metadata["quality_select_composite"]
assert routing["chosen_engine"] == "tesseract-psm-13"
assert routing["secondary_quality_score"] > routing["primary_quality_score"]
def test_candidate_engine_supports_quality_ranked_ensemble_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-quality-ensemble",
backend="quality_ranked_ensemble",
base_model="tesseract-ocr",
backend_kwargs={
"primary_engine": "tesseract",
"primary_engine_kwargs": {"psm": 6},
"alternative_engines": [
{"label": "psm7", "engine": "tesseract", "engine_kwargs": {"psm": 7}, "quality_bias": -0.05},
{"label": "psm13", "engine": "tesseract", "engine_kwargs": {"psm": 13}, "quality_bias": 0.05},
],
},
)
class FakeEngine:
def __init__(self, *, psm: int, **kwargs):
self.name = f"tesseract-psm-{psm}"
self.psm = psm
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
confidence = {6: 0.8, 7: 0.7, 13: 0.85}[self.psm]
text = {6: "base", 7: "alt7", 13: "alt13"}[self.psm]
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text=text, bbox=BBox(0, 0, 10, 10), confidence=confidence, page_index=0, line_id="p0-l0")])],
metadata={"psm": self.psm},
)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeEngine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "quality_ranked_ensemble"
assert document.text == "alt13"
routing = document.metadata["quality_ranked_ensemble"]
assert routing["chosen_engine"] == "psm13"
def test_candidate_engine_supports_script_select_composite_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={
"primary_engine": "tesseract",
"secondary_engine": "surya",
"primary_script_threshold": 0.2,
},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="नेपाल सरकार", bbox=BBox(0, 0, 10, 10), confidence=0.9, page_index=0, line_id="p0-l0")])],
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="राउट गरिएको", bbox=BBox(0, 0, 10, 10), confidence=0.92, page_index=0, line_id="p0-l0")])],
)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "script_select_composite"
assert document.text == "राउट गरिएको"
routing = document.metadata["script_select_composite"]
assert routing["chosen_engine"] == "surya"
def test_candidate_engine_supports_script_select_composite_latin_routing(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router-latin",
backend="script_select_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"secondary_engine": "tesseract",
"script": "latin",
"primary_script_threshold": 0.2,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="Invoice Total", bbox=BBox(0, 0, 10, 10), confidence=0.7, page_index=0, line_id="p0-l0")])],
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="Better English OCR", bbox=BBox(0, 0, 10, 10), confidence=0.95, page_index=0, line_id="p0-l0")])],
)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "script_select_composite"
assert document.text == "Better English OCR"
routing = document.metadata["script_select_composite"]
assert routing["chosen_engine"] == "tesseract"
def test_candidate_engine_supports_script_select_composite_page_routing(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router-page",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={
"primary_engine": "tesseract",
"secondary_engine": "surya",
"primary_script_threshold": 0.2,
"routing_granularity": "page",
},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(page_index=0, text_lines=[TextLine(text="Invoice Total", bbox=BBox(0, 0, 10, 10), confidence=0.97, page_index=0, line_id="p0-l0")]),
Page(page_index=1, text_lines=[TextLine(text="नेपाल सरकार", bbox=BBox(0, 20, 10, 10), confidence=0.91, page_index=1, line_id="p1-l0")]),
],
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(page_index=0, text_lines=[TextLine(text="Wrong Page", bbox=BBox(0, 0, 10, 10), confidence=0.3, page_index=0, line_id="p0-l0")]),
Page(page_index=1, text_lines=[TextLine(text="राउट गरिएको", bbox=BBox(0, 20, 10, 10), confidence=0.95, page_index=1, line_id="p1-l0")]),
],
)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "script_select_composite"
assert document.pages[0].blocks[0].text == "Invoice Total"
assert document.pages[1].blocks[0].text == "राउट गरिएको"
routing = document.metadata["script_select_composite"]
assert routing["chosen_engine"] == "mixed"
assert routing["routing_granularity"] == "page"
assert routing["selected_page_indexes"] == [0, 1]
assert routing["page_routing"][0]["chosen_engine"] == "tesseract"
assert routing["page_routing"][1]["chosen_engine"] == "surya"
def test_candidate_engine_supports_script_select_composite_line_routing(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router-line",
backend="script_select_composite",
base_model="tesseract+nepali-recognizer",
backend_kwargs={
"primary_engine": "tesseract",
"secondary_engine": "surya",
"primary_script_threshold": 0.2,
"routing_granularity": "line",
},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(text="Invoice Total", bbox=BBox(0, 0, 100, 20), confidence=0.97, page_index=0, line_id="p0-l0"),
TextLine(text="नेपाल सरकार", bbox=BBox(0, 30, 100, 20), confidence=0.45, page_index=0, line_id="p0-l1"),
],
)
],
)
class FakeSecondary:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(text="Wrong English", bbox=BBox(0, 0, 100, 20), confidence=0.30, page_index=0, line_id="s0"),
TextLine(text="नेपाल सरकार", bbox=BBox(0, 30, 100, 20), confidence=0.95, page_index=0, line_id="s1"),
],
)
],
)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSecondary)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "script_select_composite"
assert document.text.splitlines() == ["Invoice Total", "नेपाल सरकार"]
routing = document.metadata["script_select_composite"]
assert routing["chosen_engine"] == "mixed"
assert routing["routing_granularity"] == "line"
assert routing["secondary_line_count"] == 1
assert routing["line_routing"][0]["chosen_engine"] == "tesseract"
assert routing["line_routing"][1]["chosen_engine"] == "surya"
def test_script_select_composite_can_backfill_secondary_structure(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-script-router-backfill",
backend="script_select_composite",
base_model="tesseract+surya",
backend_kwargs={
"primary_engine": "tesseract",
"secondary_engine": "surya",
"primary_script_threshold": 0.9,
"secondary_structure_backfill": True,
},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="Invoice Total", bbox=BBox(0, 0, 10, 10), confidence=0.95, page_index=0, line_id="p0-l0")])],
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="ignored", bbox=BBox(0, 0, 10, 10), confidence=0.8, page_index=0, line_id="p0-l0")])],
tables=[
Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(0, 0, 100, 50),
cells=[TableCell(row=0, col=0, text="Item"), TableCell(row=1, col=0, text="Masks")],
)
],
)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.text == "Invoice Total"
assert len(document.tables) == 1
routing = document.metadata["script_select_composite"]
assert routing["chosen_engine"] == "tesseract"
assert routing["secondary_structure_backfill"] is True
assert document.metadata["structure_backfill"]["secondary_tables_pages"] == [0]
def test_candidate_engine_supports_line_align_composite_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake-png")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-line-align",
backend="line_align_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"secondary_engine": "tesseract",
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(text="दोस्रो", bbox=BBox(0, 20, 40, 10), confidence=0.95, page_index=0, line_id="p0-l-surya-1"),
TextLine(text="पहिलो", bbox=BBox(0, 0, 40, 10), confidence=0.96, page_index=0, line_id="p0-l-surya-0"),
],
)
],
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(text="wrong one", bbox=BBox(0, 0, 40, 10), confidence=0.5, page_index=0, line_id="p0-l0"),
TextLine(text="wrong two", bbox=BBox(0, 20, 40, 10), confidence=0.5, page_index=0, line_id="p0-l1"),
],
)
],
)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "line_align_composite"
assert document.pages[0].text_lines[0].text == "पहिलो"
assert document.pages[0].text_lines[0].line_id == "p0-l0"
assert document.pages[0].text_lines[1].text == "दोस्रो"
assert document.text == "पहिलो\nदोस्रो"
alignment = document.metadata["line_align_composite"]
assert alignment["primary_engine"] == "surya"
assert alignment["secondary_engine"] == "tesseract"
assert alignment["matched_line_count"] == 2
def test_candidate_engine_supports_table_cell_refine_composite_backend(tmp_path: Path, monkeypatch) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
from PIL import Image
source = tmp_path / "page.png"
Image.new("RGB", (120, 80), "white").save(source)
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-table-refine",
backend="table_cell_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"structure_engine": "surya",
"crop_engine": "tesseract",
"crop_padding": 0,
"replacement_policy": "always",
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 10, 80, 40),
cells=[
TableCell(row=0, col=0, text="old-a"),
TableCell(row=0, col=1, text="old-b"),
],
)
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="old-a old-b", bbox=BBox(10, 10, 80, 20), page_index=0)])],
tables=[table],
)
class FakeTesseract:
name = "tesseract"
calls: list[Path] = []
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
self.calls.append(input_path)
text = "new-a" if len(self.calls) == 1 else "new-b"
return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text=text, bbox=BBox(0, 0, 10, 10), page_index=0)])])
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "table_cell_refine_composite"
assert [cell.text for cell in document.tables[0].cells] == ["new-a", "new-b"]
refine = document.metadata["table_cell_refine_composite"]
assert refine["attempted_cells"] == 2
assert refine["replaced_cells"] == 2
assert document.tables[0].metadata["table_cell_refine"]["attempted_cells"] == 2
def test_table_cell_refine_supports_candidate_crop_engine(tmp_path: Path, monkeypatch) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
from PIL import Image
source = tmp_path / "page.png"
Image.new("RGB", (120, 80), "white").save(source)
leaf_card = write_model_card(
tmp_path / "leaf-crop.json",
model_id="ocrtech-leaf-crop",
backend="sidecar",
base_model="fixture",
)
card = write_model_card(
tmp_path / "composite.json",
model_id="ocrtech-table-refine-candidate-crop",
backend="table_cell_refine_composite",
base_model="surya+trained-candidate",
backend_kwargs={
"structure_engine": "surya",
"crop_engine": "candidate",
"crop_engine_kwargs": {"model_config": leaf_card.name},
"crop_padding": 0,
"replacement_policy": "always",
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 10, 80, 40),
cells=[TableCell(row=0, col=0, text="old")],
)
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="old", bbox=BBox(10, 10, 80, 20), page_index=0)])],
tables=[table],
)
class FakeCandidate:
name = "candidate"
seen_model_config: Path | None = None
def __init__(self, *, model_config: str | Path):
self.seen_model_config = Path(model_config)
FakeCandidate.seen_model_config = self.seen_model_config
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="trained", bbox=BBox(0, 0, 10, 10), page_index=0)])])
def fake_create_engine(name: str, **kwargs: object):
if name == "surya":
return FakeSurya(**kwargs)
if name == "candidate":
return FakeCandidate(**kwargs)
raise AssertionError(f"unexpected engine request: {name}, kwargs={kwargs}")
monkeypatch.setattr("ocrtech.engines.create_engine", fake_create_engine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert FakeCandidate.seen_model_config == leaf_card
assert document.tables[0].cells[0].text == "trained"
assert document.metadata["table_cell_refine_composite"]["crop_engine"] == "candidate"
def test_table_cell_refine_can_deduplicate_overlapping_tables(tmp_path: Path, monkeypatch) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
from PIL import Image
source = tmp_path / "page.png"
Image.new("RGB", (120, 80), "white").save(source)
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-table-refine-dedup",
backend="table_cell_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"structure_engine": "surya",
"crop_engine": "tesseract",
"replacement_policy": "empty",
"deduplicate_tables": True,
"duplicate_iou_threshold": 0.9,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell
good = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 10, 80, 40),
cells=[
TableCell(row=0, col=0, text="नाम"),
TableCell(row=0, col=1, text="रकम"),
TableCell(row=1, col=0, text="किरण"),
TableCell(row=1, col=1, text="२१००"),
],
)
collapsed = Table(
table_id="p0-t1",
page_index=0,
bbox=BBox(10, 10, 80, 40),
cells=[TableCell(row=0, col=0, text="नामरकमकिरण२१००")],
)
return EngineOutput(pages=[Page(page_index=0)], tables=[collapsed, good])
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
raise AssertionError("non-empty cells should not be reread with replacement_policy=empty")
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert len(document.tables) == 1
assert [cell.text for cell in document.tables[0].cells] == ["नाम", "रकम", "किरण", "२१००"]
assert document.metadata["table_cell_refine_composite"]["dropped_duplicate_tables"] == 1
def test_candidate_engine_supports_figure_caption_refine_composite_backend(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
lexicon_path = tmp_path / "names.txt"
lexicon_path.write_text("किरण\nमाया\nसीता\nराम\n", encoding="utf-8")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "tesseract",
"min_caption_iou": 0.5,
"min_score_delta": 1,
"correct_structural_labels": True,
"table_name_lexicon_path": str(lexicon_path),
"table_name_max_distance": 2,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine
lines = [
TextLine(text="वार्षिक प्रतिदिन", bbox=BBox(10, 0, 220, 8), confidence=0.9, page_index=0),
TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.9, page_index=0),
TextLine(text="चतुर्थ १: Revenue growth from 2021 to 2025.", bbox=BBox(10, 50, 500, 30), confidence=0.9, page_index=0),
TextLine(text="नष्कर्षः अन्तिम दुई वर्षमा वृद्धि भएको छ।", bbox=BBox(10, 90, 500, 30), confidence=0.9, page_index=0),
]
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 140, 300, 60),
cells=[
TableCell(row=0, col=0, text="नाम"),
TableCell(row=0, col=1, text="रकम"),
TableCell(row=1, col=0, text="करिण"),
TableCell(row=1, col=1, text="२१००"),
],
)
return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], tables=[table], metadata={"engine": "surya"})
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
lines = [
TextLine(text="वार्षिक प्रतविदन", bbox=BBox(10, 0, 220, 8), confidence=0.8, page_index=0),
TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.8, page_index=0),
TextLine(text="चत्रि १: Revenue growth from 2021 to 2025.", bbox=BBox(10, 38, 500, 42), confidence=0.8, page_index=0),
TextLine(text="नष्कर्षः अन्तिम दुई वर्षमा वृद्धि भएको छ।", bbox=BBox(10, 90, 500, 30), confidence=0.8, page_index=0),
]
return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], metadata={"engine": "tesseract"})
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.metadata["candidate_model"]["backend"] == "figure_caption_refine_composite"
assert document.metadata["figure_caption_refine_composite"]["replaced_captions"] == 1
assert document.figures[0].caption == "चित्र १: Revenue growth from 2021 to 2025."
assert document.figures[0].metadata["line_ids"] == ["p0-l1", "p0-l2"]
assert document.pages[0].text_lines[0].text == "वार्षिक प्रतिवेदन"
assert document.pages[0].text_lines[2].metadata["figure_caption_refine"]["original_text"].startswith("चतुर्थ")
assert document.pages[0].text_lines[2].metadata["structural_label_correction"]["corrected_text"].startswith("चित्र")
assert document.pages[0].text_lines[3].text.startswith("निष्कर्ष:")
assert [cell.text for cell in document.tables[0].cells] == ["नाम", "रकम", "किरण", "२१००"]
assert document.metadata["figure_caption_refine_composite"]["table_name_corrections"] == 1
def test_figure_caption_refine_supports_candidate_caption_engine(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
leaf_card = write_model_card(
tmp_path / "leaf-caption.json",
model_id="ocrtech-leaf-caption",
backend="sidecar",
base_model="fixture",
)
card = write_model_card(
tmp_path / "composite.json",
model_id="ocrtech-figure-caption-candidate",
backend="figure_caption_refine_composite",
base_model="surya+trained-candidate",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "candidate",
"caption_engine_kwargs": {"model_config": leaf_card.name},
"min_caption_iou": 0.5,
"min_score_delta": 1,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
lines = [
TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.9, page_index=0),
TextLine(text="चतुर्थ १: Revenue growth.", bbox=BBox(10, 50, 500, 30), confidence=0.9, page_index=0),
]
return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], metadata={"engine": "surya"})
class FakeCandidate:
name = "candidate"
seen_model_config: Path | None = None
def __init__(self, *, model_config: str | Path):
self.seen_model_config = Path(model_config)
FakeCandidate.seen_model_config = self.seen_model_config
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
lines = [
TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.8, page_index=0),
TextLine(text="चित्र १: Revenue growth.", bbox=BBox(10, 50, 500, 30), confidence=0.8, page_index=0),
]
return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], metadata={"engine": "candidate"})
def fake_create_engine(name: str, **kwargs: object):
if name == "surya":
return FakeSurya(**kwargs)
if name == "candidate":
return FakeCandidate(**kwargs)
raise AssertionError(f"unexpected engine request: {name}, kwargs={kwargs}")
monkeypatch.setattr("ocrtech.engines.create_engine", fake_create_engine)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert FakeCandidate.seen_model_config == leaf_card
assert document.metadata["figure_caption_refine_composite"]["caption_engine"] == "candidate"
assert document.metadata["figure_caption_refine_composite"]["replaced_captions"] == 1
assert document.figures[0].caption == "चित्र १: Revenue growth."
def test_figure_caption_refine_corrects_daily_ledger_structural_title(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine-daily-ledger",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "tesseract",
"correct_structural_labels": True,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(text="दैनकि हसिाब", bbox=BBox(10, 0, 220, 20), confidence=0.9, page_index=0),
],
)
],
metadata={"engine": "surya"},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, TextLine
return EngineOutput(
pages=[
Page(
page_index=0,
text_lines=[
TextLine(text="दैनकि हसिाब", bbox=BBox(10, 0, 220, 20), confidence=0.8, page_index=0),
],
)
],
metadata={"engine": "tesseract"},
)
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.pages[0].text_lines[0].text == "दैनिक हिसाब"
assert document.pages[0].text_lines[0].metadata["structural_label_correction"] == {
"original_text": "दैनकि हसिाब",
"corrected_text": "दैनिक हिसाब",
}
assert document.text == "दैनिक हिसाब"
def test_figure_caption_refine_preserves_table_name_suffix_when_using_lexicon(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
lexicon_path = tmp_path / "names.txt"
lexicon_path.write_text("किरण\nमाया\n", encoding="utf-8")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine-suffixed-name",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "tesseract",
"table_name_lexicon_path": str(lexicon_path),
"table_name_max_distance": 2,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 40, 300, 60),
cells=[
TableCell(row=0, col=0, text="नाम"),
TableCell(row=0, col=1, text="रकम"),
TableCell(row=1, col=0, text="करिण-302"),
TableCell(row=1, col=1, text="२१००"),
TableCell(row=2, col=0, text="माया-302"),
TableCell(row=2, col=1, text="२९००"),
],
)
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="दैनिक हिसाब", bbox=BBox(10, 0, 200, 20), page_index=0)])],
tables=[table],
metadata={"engine": "surya"},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
return EngineOutput(pages=[], metadata={"engine": "tesseract"})
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert [cell.text for cell in document.tables[0].cells] == ["नाम", "रकम", "किरण-302", "२१००", "माया-302", "२९००"]
assert document.metadata["figure_caption_refine_composite"]["table_name_corrections"] == 1
def test_figure_caption_refine_corrects_name_cells_below_grouped_header(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
lexicon_path = tmp_path / "names.txt"
lexicon_path.write_text("किरण\nमाया\n", encoding="utf-8")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine-grouped-header-name",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "tesseract",
"table_name_lexicon_path": str(lexicon_path),
"table_name_max_distance": 2,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 40, 300, 120),
cells=[
TableCell(row=0, col=0, text="Ledger Summary", colspan=2),
TableCell(row=0, col=2, text="Status"),
TableCell(row=1, col=0, text="नाम"),
TableCell(row=1, col=1, text="रकम"),
TableCell(row=1, col=2, text="Remarks"),
TableCell(row=2, col=0, text="करिण-302"),
TableCell(row=2, col=1, text="२१००"),
TableCell(row=2, col=2, text="Paid"),
],
)
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="समेकित हिसाब", bbox=BBox(10, 0, 200, 20), page_index=0)])],
tables=[table],
metadata={"engine": "surya"},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
return EngineOutput(pages=[], metadata={"engine": "tesseract"})
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert [cell.text for cell in document.tables[0].cells if cell.col == 0] == ["Ledger Summary", "नाम", "किरण-302"]
assert document.metadata["figure_caption_refine_composite"]["table_name_corrections"] == 1
def test_figure_caption_refine_infers_flat_repeated_header_colspan(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine-infer-colspan",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "tesseract",
"infer_table_header_colspans": True,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 40, 300, 120),
cells=[
TableCell(row=0, col=0, text="Ledger Summary"),
TableCell(row=0, col=1, text="Ledger Summary"),
TableCell(row=0, col=2, text="Status"),
TableCell(row=1, col=0, text="नाम"),
TableCell(row=1, col=1, text="रकम"),
TableCell(row=1, col=2, text="Remarks"),
],
)
return EngineOutput(
pages=[Page(page_index=0, text_lines=[TextLine(text="समेकित हिसाब", bbox=BBox(10, 0, 200, 20), page_index=0)])],
tables=[table],
metadata={"engine": "surya"},
)
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
return EngineOutput(pages=[], metadata={"engine": "tesseract"})
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert document.tables[0].cells[0].text == "Ledger Summary"
assert document.tables[0].cells[0].colspan == 2
assert len(document.tables[0].cells) == 5
assert 'colspan="2"' in document.tables[0].html
assert document.metadata["figure_caption_refine_composite"]["inferred_table_header_colspans"] == 1
def test_figure_caption_refine_normalizes_separator_and_duplicate_table_columns(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine-normalize-grid",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "tesseract",
"normalize_table_grids": True,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 40, 300, 120),
cells=[
TableCell(row=0, col=0, text="—"),
TableCell(row=0, col=1, text="Skull cap"),
TableCell(row=0, col=2, text="1D"),
TableCell(row=0, col=3, text="1D"),
TableCell(row=1, col=0, text="—"),
TableCell(row=1, col=1, text="Helmet"),
TableCell(row=1, col=2, text="2D"),
TableCell(row=1, col=3, text="2D"),
],
)
return EngineOutput(pages=[Page(page_index=0)], tables=[table], metadata={"engine": "surya"})
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
return EngineOutput(pages=[], metadata={"engine": "tesseract"})
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert [cell.text for cell in document.tables[0].cells] == ["Skull cap", "1D", "Helmet", "2D"]
assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_dropped_columns"] == 2
assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_merged_rows"] == 0
assert document.tables[0].metadata["normalized_table_grid_dropped_columns"] == 2
def test_figure_caption_refine_merges_sparse_bullet_table_rows(tmp_path: Path, monkeypatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-figure-caption-refine-normalize-row-fragments",
backend="figure_caption_refine_composite",
base_model="surya+tesseract",
backend_kwargs={
"primary_engine": "surya",
"caption_engine": "tesseract",
"normalize_table_grids": True,
},
)
class FakeSurya:
name = "surya"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell
table = Table(
table_id="p0-t0",
page_index=0,
bbox=BBox(10, 40, 300, 120),
cells=[
TableCell(row=0, col=0, text="Type of Anomaly"),
TableCell(row=0, col=1, text="Variant"),
TableCell(row=0, col=2, text="Type"),
TableCell(row=1, col=0, text=""),
TableCell(row=1, col=1, text="Coronaries originating"),
TableCell(row=1, col=2, text="- left main"),
TableCell(row=2, col=0, text="- right coronary"),
TableCell(row=2, col=1, text=""),
TableCell(row=2, col=2, text=""),
],
)
return EngineOutput(pages=[Page(page_index=0)], tables=[table], metadata={"engine": "surya"})
class FakeTesseract:
name = "tesseract"
def __init__(self, **kwargs):
self.kwargs = kwargs
def recognize(self, input_path: Path):
from ocrtech.engines import EngineOutput
return EngineOutput(pages=[], metadata={"engine": "tesseract"})
monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya)
monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract)
document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card)
assert [cell.text for cell in document.tables[0].cells] == [
"Type of Anomaly",
"Variant",
"Type",
"",
"Coronaries originating",
"- left main\n- right coronary",
]
assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_dropped_columns"] == 0
assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_merged_rows"] == 1
assert document.tables[0].metadata["normalized_table_grid_merged_rows"] == 1
def test_text_correction_composite_clears_generation_max_length(tmp_path: Path, monkeypatch) -> None:
model_dir = tmp_path / "model"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
class FakeTorch:
class cuda:
@staticmethod
def is_available() -> bool:
return False
class backends:
class mps:
@staticmethod
def is_available() -> bool:
return False
@staticmethod
def no_grad():
class _NoGrad:
def __enter__(self):
return None
def __exit__(self, exc_type, exc, tb):
return False
return _NoGrad()
class FakeTokenizer:
@classmethod
def from_pretrained(cls, source: str, use_fast: bool = False):
return cls()
def __call__(self, texts, padding: bool, truncation: bool, return_tensors: str):
return {"input_ids": SimpleNamespace(to=lambda device: [[1, 2, 3]])}
def batch_decode(self, generated_ids, skip_special_tokens: bool):
return ["सही पाठ"]
class FakeGenerationConfig:
def __init__(self) -> None:
self.max_length = 256
fake_model = SimpleNamespace(generation_config=FakeGenerationConfig())
def fake_from_pretrained(source: str):
fake_model.to = lambda device: fake_model
fake_model.eval = lambda: None
fake_model.generate = lambda **kwargs: [[1, 2, 3]]
return fake_model
monkeypatch.setitem(sys.modules, "torch", FakeTorch)
monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(AutoModelForSeq2SeqLM=SimpleNamespace(from_pretrained=fake_from_pretrained), AutoTokenizer=FakeTokenizer))
monkeypatch.setattr("ocrtech.engines.create_engine", lambda name, **kwargs: SimpleNamespace(name=name, recognize=lambda input_path: SimpleNamespace(pages=[], tables=[], figures=[], metadata={})))
from ocrtech.engines import TextCorrectionCompositeEngine
TextCorrectionCompositeEngine(model_dir=str(model_dir), tokenizer_dir=str(model_dir), base_engine="tesseract")
assert fake_model.generation_config.max_length is None
def test_hf_engine_loads_processor_without_fast_tokenizer(tmp_path: Path, monkeypatch) -> None:
model_dir = tmp_path / "model"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}", encoding="utf-8")
calls: list[tuple[str, object]] = []
class FakeTorch:
class cuda:
@staticmethod
def is_available() -> bool:
return False
class backends:
class mps:
@staticmethod
def is_available() -> bool:
return False
@staticmethod
def no_grad():
class _NoGrad:
def __enter__(self):
return None
def __exit__(self, exc_type, exc, tb):
return False
return _NoGrad()
class FakeImage:
@staticmethod
def open(path: Path):
class _Image:
def convert(self, mode: str):
return self
return _Image()
class FakeProcessor:
@classmethod
def from_pretrained(cls, source: str, **kwargs):
calls.append(("processor", kwargs))
return cls()
def __call__(self, image, return_tensors: str):
return SimpleNamespace(pixel_values=SimpleNamespace(to=lambda device: "pixels"))
def batch_decode(self, generated_ids, skip_special_tokens: bool):
return ["नेपाल"]
class FakeModel:
@classmethod
def from_pretrained(cls, source: str):
calls.append(("model", source))
return cls()
def to(self, device: str):
return self
def eval(self):
return None
def generate(self, pixel_values, max_new_tokens: int):
return [[1, 2, 3]]
monkeypatch.setitem(sys.modules, "torch", FakeTorch)
monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(TrOCRProcessor=FakeProcessor, VisionEncoderDecoderModel=FakeModel))
monkeypatch.setitem(sys.modules, "PIL", SimpleNamespace(Image=FakeImage))
from ocrtech.engines import HuggingFaceVisionTextEngine
engine = HuggingFaceVisionTextEngine(model_dir=str(model_dir))
output = engine.recognize(tmp_path / "page.png")
assert calls[0] == ("processor", {"backend": "pil"})
assert output.pages[0].text_lines[0].text == "नेपाल"
def test_huggingface_vision_text_engine_sanitizes_list_extra_special_tokens(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
model_dir = tmp_path / "model"
model_dir.mkdir()
(model_dir / "tokenizer_config.json").write_text(
json.dumps({"extra_special_tokens": ["", ""]}),
encoding="utf-8",
)
calls: list[tuple[str, object]] = []
class FakeTorch:
class cuda:
@staticmethod
def is_available() -> bool:
return False
class backends:
class mps:
@staticmethod
def is_available() -> bool:
return False
@staticmethod
def no_grad():
class _NoGrad:
def __enter__(self):
return None
def __exit__(self, exc_type, exc, tb):
return False
return _NoGrad()
class FakeImage:
@staticmethod
def open(path: Path):
class _Image:
def convert(self, mode: str):
return self
return _Image()
class FakeProcessor:
@classmethod
def from_pretrained(cls, source: str, **kwargs):
config = json.loads((Path(source) / "tokenizer_config.json").read_text(encoding="utf-8"))
calls.append(("processor", config.get("extra_special_tokens")))
if isinstance(config.get("extra_special_tokens"), list):
raise AttributeError("'list' object has no attribute 'keys'")
return cls()
def __call__(self, image, return_tensors: str):
return SimpleNamespace(pixel_values=SimpleNamespace(to=lambda device: "pixels"))
def batch_decode(self, generated_ids, skip_special_tokens: bool):
return ["नेपाल"]
class FakeModel:
generation_config = SimpleNamespace(max_length=128)
@classmethod
def from_pretrained(cls, source: str):
return cls()
def to(self, device: str):
return self
def eval(self):
return None
def generate(self, pixel_values, max_new_tokens: int):
return [[1, 2, 3]]
monkeypatch.setitem(sys.modules, "torch", FakeTorch)
monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(TrOCRProcessor=FakeProcessor, VisionEncoderDecoderModel=FakeModel))
monkeypatch.setitem(sys.modules, "PIL", SimpleNamespace(Image=FakeImage))
from ocrtech.engines import HuggingFaceVisionTextEngine
engine = HuggingFaceVisionTextEngine(model_dir=str(model_dir))
output = engine.recognize(tmp_path / "page.png")
assert calls == [("processor", ["", ""]), ("processor", {})]
assert output.pages[0].text_lines[0].text == "नेपाल"
def test_benchmark_candidate_model_config(tmp_path: Path) -> None:
source = tmp_path / "page.txt"
source.write_text("hello world\n", encoding="utf-8")
source.with_suffix(".ref.txt").write_text("hello world\n", encoding="utf-8")
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
results = run_benchmark([source], tmp_path / "bench", baselines=["candidate"], candidate_model_config=card)
assert results[0].status == "ok"
assert results[0].metrics["cer"] == 0
assert results[0].metrics["quality_score"] > 0
def test_experiment_records_candidate_model_config(tmp_path: Path) -> None:
pack_dir = tmp_path / "pack"
inputs_dir = pack_dir / "inputs"
refs_dir = pack_dir / "references"
manifests_dir = pack_dir / "manifests"
inputs_dir.mkdir(parents=True)
refs_dir.mkdir()
manifests_dir.mkdir()
source = inputs_dir / "sample.txt"
source.write_text("hello world\n", encoding="utf-8")
refs_dir.joinpath("sample.txt.ref.txt").write_text("hello world\n", encoding="utf-8")
manifest = manifests_dir / "eval.jsonl"
manifest.write_text(
'{"sample_id":"s1","dataset":"fixture","split":"eval","image_path":"'
+ str(source)
+ '","text":"hello world","sha256":null,"metadata":{"slices":["english"]}}\n',
encoding="utf-8",
)
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
fallback_card = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture")
report = run_experiment(
manifest,
tmp_path / "experiment",
baselines=["candidate"],
references_dir=refs_dir,
candidate_model_config=card,
fallback_model_config=fallback_card,
low_confidence_threshold=0.73,
run_validation=False,
)
assert report.environment["candidate_model_config"] == str(card)
assert report.environment["fallback_model_config"] == str(fallback_card)
assert report.environment["fallback_model_config_sha256"]
assert report.environment["low_confidence_threshold"] == 0.73
def _passing_validation_gate() -> dict[str, object]:
return {
"passed": True,
"pairs": 2,
"candidate_mean": 0.1,
"baseline_mean": 0.2,
"absolute_improvement": 0.1,
"relative_improvement": 0.5,
"improvement_ci_low": 0.01,
"win_count": 2,
"loss_count": 0,
"tie_count": 0,
"pair_win_rate": 1.0,
"sign_test_p_value": 0.0,
"gate": {
"baseline": "tesseract",
"metric": "cer",
"direction": "lower",
"min_pairs": 2,
"min_absolute_margin": 0.0,
"min_relative_improvement": 0.0,
"min_pair_win_rate": 0.5,
"max_p_value": 1.0,
"require_ci_above_zero": True,
},
}