ocr-tech-k2 / tests /test_models_candidate.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
87.5 kB
from __future__ import annotations import json import importlib.util import sys from pathlib import Path from types import SimpleNamespace import pytest from ocrtech.benchmark import run_benchmark from ocrtech.experiments import run_experiment from ocrtech.manifest import sha256_file from ocrtech.models import ModelCard, audit_model_card, component_baseline_names, inspect_routed_calibration_collapse, model_admission_status, trained_recognizer_provenance_status, write_model_card from ocrtech.pipeline import parse_document def test_model_card_audit_passes_for_existing_artifact(tmp_path: Path) -> None: artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-test", backend="sidecar", base_model="fixture", artifact_paths=[artifact], notes="test model card", ) audit = audit_model_card(card, tmp_path / "audit") assert audit.passed assert audit.model_id == "ocrtech-test" assert (tmp_path / "audit" / "model-audit.json").exists() def test_model_card_records_and_checks_admission_validation(tmp_path: Path) -> None: artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") validation = tmp_path / "validation.json" model_path = tmp_path / "model.json" validation.write_text( json.dumps( { "claim_status": "validated", "passed": True, "provenance": {"candidate_model_config": {"path": str(model_path)}}, "gates": [_passing_validation_gate()], } ) + "\n", encoding="utf-8", ) card_path = write_model_card( model_path, model_id="ocrtech-admitted", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne"}, admission_validation_report=validation, ) card = ModelCard.from_path(card_path) status = model_admission_status(card, card_path) assert status.passed assert card.metrics["admission_validation"]["sha256"] def test_model_admission_status_requires_quantitative_gate_evidence(tmp_path: Path) -> None: artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") validation = tmp_path / "validation.json" model_path = tmp_path / "model.json" validation.write_text( json.dumps( { "claim_status": "validated", "passed": True, "provenance": {"candidate_model_config": {"path": str(model_path)}}, "gates": [], } ) + "\n", encoding="utf-8", ) card_path = write_model_card( model_path, model_id="ocrtech-admission-without-gates", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne"}, admission_validation_report=validation, ) status = model_admission_status(ModelCard.from_path(card_path), card_path) assert not status.passed assert "admission validation missing gates" in status.details def test_model_card_records_fallback_model_card_provenance(tmp_path: Path) -> None: fallback = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") candidate = write_model_card( tmp_path / "candidate.json", model_id="ocrtech-candidate", backend="sidecar", base_model="fixture", fallback_model_card=fallback, ) payload = json.loads(candidate.read_text(encoding="utf-8")) assert payload["provenance"]["fallback_model_card"] == { "path": str(fallback), "sha256": sha256_file(fallback), } def test_trained_recognizer_provenance_requires_training_summary(tmp_path: Path) -> None: train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" image = tmp_path / "line.png" image.write_bytes(b"line") row = { "sample_id": "s1", "dataset": "fixture", "split": "train", "image_path": str(image), "text": "नेपाल", } train_manifest.write_text(json.dumps(row, ensure_ascii=False) + "\n", encoding="utf-8") eval_manifest.write_text(json.dumps({**row, "split": "eval"}, ensure_ascii=False) + "\n", encoding="utf-8") artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") card_path = write_model_card( tmp_path / "model.json", model_id="ocrtech-trained-missing-summary", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne"}, train_manifest=train_manifest, eval_manifest=eval_manifest, ) status = trained_recognizer_provenance_status(ModelCard.from_path(card_path), card_path) assert not status.passed assert "trained recognizer missing training_summary path/sha256 provenance" in status.details def test_trained_recognizer_provenance_rejects_training_summary_hash_mismatch(tmp_path: Path) -> None: train_manifest = tmp_path / "train.jsonl" eval_manifest = tmp_path / "eval.jsonl" training_summary = tmp_path / "recognizer-training-summary.json" image = tmp_path / "line.png" image.write_bytes(b"line") row = { "sample_id": "s1", "dataset": "fixture", "split": "train", "image_path": str(image), "text": "नेपाल", } train_manifest.write_text(json.dumps(row, ensure_ascii=False) + "\n", encoding="utf-8") eval_manifest.write_text(json.dumps({**row, "split": "eval"}, ensure_ascii=False) + "\n", encoding="utf-8") training_summary.write_text('{"backend": "paddleocr", "epochs": 1}\n', encoding="utf-8") artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") card_path = write_model_card( tmp_path / "model.json", model_id="ocrtech-trained-stale-summary", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne"}, train_manifest=train_manifest, eval_manifest=eval_manifest, training_summary=training_summary, ) training_summary.write_text('{"backend": "paddleocr", "epochs": 2}\n', encoding="utf-8") status = trained_recognizer_provenance_status(ModelCard.from_path(card_path), card_path) assert not status.passed assert any("trained recognizer training_summary sha256 mismatch" in detail for detail in status.details) def test_model_admission_status_fails_on_not_validated_report(tmp_path: Path) -> None: artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") validation = tmp_path / "validation.json" model_path = tmp_path / "model.json" validation.write_text( json.dumps( { "claim_status": "not_validated", "passed": False, "provenance": {"candidate_model_config": {"path": str(model_path)}}, "missing_requirements": ["gate failed: cer vs tesseract"], "gates": [ { "gate": { "baseline": "tesseract", "metric": "cer", "direction": "lower", "slice": "nepali", }, "passed": False, "pairs": 12, "candidate_mean": 0.908108, "baseline_mean": 0.235433, "absolute_improvement": -0.672676, "relative_improvement": -2.857191, "win_count": 0, "loss_count": 12, "tie_count": 0, "reason": "absolute improvement -0.672676 < required 0.0", } ], } ) + "\n", encoding="utf-8", ) card_path = write_model_card( model_path, model_id="ocrtech-rejected", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne"}, admission_validation_report=validation, ) status = model_admission_status(ModelCard.from_path(card_path), card_path) assert not status.passed assert any("claim_status=not_validated" in detail for detail in status.details) assert any("missing_requirement=gate failed: cer vs tesseract" in detail for detail in status.details) assert any( "failed_gate=cer baseline=tesseract slice=nepali" in detail and "candidate_mean=0.908108" in detail and "loss_count=12" in detail for detail in status.details ) def test_model_admission_status_rejects_report_for_different_model(tmp_path: Path) -> None: artifact = tmp_path / "inference.pdiparams" artifact.write_bytes(b"weights") other_artifact = tmp_path / "other.pdiparams" other_artifact.write_bytes(b"other weights") other_model = write_model_card( tmp_path / "other-model.json", model_id="ocrtech-other", backend="paddleocr", base_model="fixture", artifact_paths=[other_artifact], backend_kwargs={"lang": "ne"}, ) validation = tmp_path / "validation.json" validation.write_text( json.dumps( { "claim_status": "validated", "passed": True, "provenance": {"candidate_model_config": {"path": str(other_model)}}, "gates": [_passing_validation_gate()], } ) + "\n", encoding="utf-8", ) card_path = write_model_card( tmp_path / "model.json", model_id="ocrtech-current", backend="paddleocr", base_model="fixture", artifact_paths=[artifact], backend_kwargs={"lang": "ne"}, admission_validation_report=validation, ) status = model_admission_status(ModelCard.from_path(card_path), card_path) assert not status.passed assert any("model_id mismatch" in detail for detail in status.details) def test_model_card_audit_fails_for_missing_artifact(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-missing", backend="sidecar", base_model="fixture", artifact_paths=[tmp_path / "missing.pdmodel"], ) audit = audit_model_card(card) assert not audit.passed assert "required artifact missing" in audit.issues[0] def test_nested_model_kwargs_resolve_relative_paths_from_model_card(tmp_path: Path) -> None: from ocrtech.engines import _resolve_model_kwargs config_dir = tmp_path / "candidate" config_dir.mkdir() card_path = config_dir / "model.json" artifacts = config_dir / "artifacts" artifacts.mkdir() card = write_model_card( card_path, model_id="ocrtech-nested-paths", backend="script_select_composite", base_model="fixture", backend_kwargs={ "primary_engine": "tesseract", "secondary_engine": "paddleocr", "secondary_engine_kwargs": { "text_recognition_model_dir": "artifacts", "metadata": {"dict_path": "artifacts/nep_eng_dict.txt"}, }, }, ) resolved = _resolve_model_kwargs(ModelCard.from_path(card), card) secondary = resolved["secondary_engine_kwargs"] assert isinstance(secondary, dict) assert secondary["text_recognition_model_dir"] == str(artifacts) metadata = secondary["metadata"] assert isinstance(metadata, dict) assert metadata["dict_path"] == str(artifacts / "nep_eng_dict.txt") def test_paddle_candidate_requires_recorded_artifact(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-no-artifacts", backend="paddleocr", base_model="fixture", paddleocr_kwargs={"lang": "devanagari"}, ) audit = audit_model_card(card) assert not audit.passed assert "requires at least one recorded artifact" in audit.issues[0] def test_hf_candidate_requires_recorded_artifact(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-hf-no-artifacts", backend="hf_vision_encoder_decoder", base_model="microsoft/trocr-small-printed", ) audit = audit_model_card(card) assert not audit.passed assert "requires at least one recorded artifact" in audit.issues[0] def test_tesseract_candidate_audit_passes_without_artifacts(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-tess", backend="tesseract", base_model="tesseract-ocr", backend_kwargs={"language": "nep+eng", "psm": 7}, ) audit = audit_model_card(card) assert audit.passed assert audit.backend == "tesseract" def test_surya_candidate_audit_passes_without_artifacts(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-surya", backend="surya", base_model="datalab-to/surya-ocr-2", backend_kwargs={"mode": "block"}, ) audit = audit_model_card(card) assert audit.passed assert audit.backend == "surya" def test_quality_select_candidate_audit_requires_named_engines(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-quality-router", backend="quality_select_composite", base_model="tesseract-ocr", backend_kwargs={"primary_engine": "tesseract"}, ) audit = audit_model_card(card) assert not audit.passed assert any("secondary_engine" in issue for issue in audit.issues) def test_quality_ranked_ensemble_audit_requires_alternatives(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-quality-ensemble", backend="quality_ranked_ensemble", base_model="tesseract-ocr", backend_kwargs={"primary_engine": "tesseract"}, ) audit = audit_model_card(card) assert not audit.passed assert any("alternative_engines" in issue for issue in audit.issues) def test_script_select_candidate_audit_requires_named_engines(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={"primary_engine": "tesseract"}, ) audit = audit_model_card(card) assert not audit.passed assert any("secondary_engine" in issue for issue in audit.issues) def test_line_align_candidate_audit_requires_named_engines(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-line-align", backend="line_align_composite", base_model="surya+tesseract", backend_kwargs={"primary_engine": "surya"}, ) audit = audit_model_card(card) assert not audit.passed assert any("secondary_engine" in issue for issue in audit.issues) def test_table_cell_refine_candidate_audit_passes_with_defaults(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-table-refine", backend="table_cell_refine_composite", base_model="surya+tesseract", backend_kwargs={"structure_engine": "surya", "crop_engine": "tesseract"}, ) audit = audit_model_card(card) assert audit.passed assert audit.backend == "table_cell_refine_composite" def test_table_cell_refine_audit_requires_candidate_crop_model_config(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-table-refine-candidate-missing-config", backend="table_cell_refine_composite", base_model="surya+trained-candidate", backend_kwargs={"structure_engine": "surya", "crop_engine": "candidate"}, ) audit = audit_model_card(card) assert not audit.passed assert "table_cell_refine_composite crop_engine=candidate requires backend_kwargs.crop_engine_kwargs.model_config" in audit.issues def test_figure_caption_refine_candidate_audit_passes_with_defaults(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={"primary_engine": "surya", "caption_engine": "tesseract"}, ) audit = audit_model_card(card) assert audit.passed assert audit.backend == "figure_caption_refine_composite" def test_figure_caption_refine_audit_requires_candidate_caption_model_config(tmp_path: Path) -> None: card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-candidate-missing-config", backend="figure_caption_refine_composite", base_model="surya+trained-candidate", backend_kwargs={"primary_engine": "surya", "caption_engine": "candidate"}, ) audit = audit_model_card(card) assert not audit.passed assert "figure_caption_refine_composite caption_engine=candidate requires backend_kwargs.caption_engine_kwargs.model_config" in audit.issues def test_component_baseline_names_normalizes_component_engines(tmp_path: Path) -> None: card = ModelCard.from_path( write_model_card( tmp_path / "model.json", model_id="ocrtech-quality-ensemble", backend="quality_ranked_ensemble", base_model="tesseract+paddle", backend_kwargs={ "primary_engine": "tesseract", "alternative_engines": [ {"engine": "paddleocr", "bias": 0.1}, {"engine": "surya", "bias": 0.2}, ], }, ) ) names = component_baseline_names(card) assert names == ["stock-paddle", "surya", "tesseract"] def test_inspect_routed_calibration_collapse_accepts_repo_relative_metrics_report(tmp_path: Path, monkeypatch) -> None: monkeypatch.chdir(tmp_path) report = tmp_path / "outputs" / "script-router" / "calibration.json" report.parent.mkdir(parents=True) report.write_text( '{"selected_threshold":0.0,"metric":"cer","selected_trial":{"pairs":5,"secondary_picks":5}}' + "\n", encoding="utf-8", ) model_dir = tmp_path / "models" model_dir.mkdir() card = write_model_card( model_dir / "model.json", model_id="ocrtech-script-router", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={"primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.0}, metrics_report=Path("outputs/script-router/calibration.json"), ) reason, details = inspect_routed_calibration_collapse(ModelCard.from_path(card), card) assert reason is not None assert "collapses to the secondary engine" in reason assert any("report=" in item for item in details) def test_candidate_engine_uses_explicit_model_config(tmp_path: Path) -> None: source = tmp_path / "page.txt" source.write_text("नेपाल सरकार\nhello world\n", encoding="utf-8") card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["model_id"] == "ocrtech-sidecar" assert document.metadata["quality"]["passed"] def test_candidate_engine_supports_hf_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") model_dir = tmp_path / "model" model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-hf", backend="hf_vision_encoder_decoder", base_model="microsoft/trocr-small-printed", artifact_paths=[model_dir], backend_kwargs={"model_dir": str(model_dir), "processor_dir": str(model_dir)}, ) class FakeEngine: def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=None, page_index=0, line_id="p0-l0")])]) monkeypatch.setattr("ocrtech.engines.HuggingFaceVisionTextEngine", FakeEngine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "hf_vision_encoder_decoder" assert document.text == "नेपाल" def test_candidate_engine_supports_tesseract_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-tess", backend="tesseract", base_model="tesseract-ocr", backend_kwargs={"language": "nep+eng", "psm": 7}, ) class FakeEngine: def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=0.95, page_index=0, line_id="p0-l0")])]) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeEngine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "tesseract" assert document.text == "नेपाल" def test_candidate_engine_supports_surya_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-surya", backend="surya", base_model="datalab-to/surya-ocr-2", backend_kwargs={"mode": "block"}, ) class FakeEngine: def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[TextLine(text="नेपाल", bbox=BBox(0, 0, 10, 10), confidence=0.91, page_index=0, line_id="p0-l0")], metadata={"surya_mode": "block"}, ) ], metadata={"surya_mode": "block"}, ) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeEngine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "surya" assert document.text == "नेपाल" def test_surya_engine_converts_table_html_to_structured_table() -> None: from ocrtech.engines import SuryaEngine engine = object.__new__(SuryaEngine) page_result = SimpleNamespace( blocks=[ SimpleNamespace( polygon=[[0.0, 0.0], [120.0, 0.0], [120.0, 60.0], [0.0, 60.0]], confidence=0.88, label="Table", raw_label="Table", reading_order=0, html="", skipped=False, error=False, ), SimpleNamespace( polygon=[[0.0, 70.0], [120.0, 70.0], [120.0, 90.0], [0.0, 90.0]], confidence=0.93, label="Text", raw_label="Text", reading_order=1, html="<p>नेपाल सरकार</p>", skipped=False, error=False, ), ] ) table_result = SimpleNamespace( html="<table><tr><th>नाम</th><th>उमेर</th></tr><tr><td>राम</td><td>३२</td></tr></table>", mode="full", error=False, rows=[], cols=[], ) output = engine._convert_page_result(page_result, 120, 90, "full_page", page_index=0, table_results={0: table_result}) assert len(output.tables) == 1 assert output.tables[0].html.startswith("<table>") assert [cell.text for cell in output.tables[0].cells] == ["नाम", "उमेर", "राम", "३२"] assert output.tables[0].metadata["surya_table_mode"] == "full" assert output.pages[0].text_lines[0].text == "नेपाल सरकार" def test_candidate_engine_supports_text_correction_composite_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") model_dir = tmp_path / "model" model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-corrected", backend="text_correction_composite", base_model="google/byt5-small", artifact_paths=[model_dir], backend_kwargs={"model_dir": str(model_dir), "tokenizer_dir": str(model_dir), "base_engine": "tesseract"}, ) class FakeEngine: def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="सही पाठ", bbox=BBox(0, 0, 10, 10), confidence=0.9, page_index=0, line_id="p0-l0")])]) monkeypatch.setattr("ocrtech.engines.TextCorrectionCompositeEngine", FakeEngine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "text_correction_composite" assert document.text == "सही पाठ" def test_candidate_engine_supports_quality_select_composite_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-quality-router", backend="quality_select_composite", base_model="tesseract-ocr", backend_kwargs={ "primary_engine": "tesseract", "secondary_engine": "tesseract", "primary_engine_kwargs": {"psm": 6}, "secondary_engine_kwargs": {"psm": 13}, "secondary_quality_margin": 0.02, }, ) class FakeEngine: def __init__(self, *, psm: int, **kwargs): self.name = f"tesseract-psm-{psm}" self.psm = psm def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine confidence = 0.80 if self.psm == 6 else 1.0 text = "default" if self.psm == 6 else "routed" return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text=text, bbox=BBox(0, 0, 10, 10), confidence=confidence, page_index=0, line_id="p0-l0")])], metadata={"psm": self.psm}, ) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeEngine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "quality_select_composite" assert document.text == "routed" routing = document.metadata["quality_select_composite"] assert routing["chosen_engine"] == "tesseract-psm-13" assert routing["secondary_quality_score"] > routing["primary_quality_score"] def test_candidate_engine_supports_quality_ranked_ensemble_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-quality-ensemble", backend="quality_ranked_ensemble", base_model="tesseract-ocr", backend_kwargs={ "primary_engine": "tesseract", "primary_engine_kwargs": {"psm": 6}, "alternative_engines": [ {"label": "psm7", "engine": "tesseract", "engine_kwargs": {"psm": 7}, "quality_bias": -0.05}, {"label": "psm13", "engine": "tesseract", "engine_kwargs": {"psm": 13}, "quality_bias": 0.05}, ], }, ) class FakeEngine: def __init__(self, *, psm: int, **kwargs): self.name = f"tesseract-psm-{psm}" self.psm = psm def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine confidence = {6: 0.8, 7: 0.7, 13: 0.85}[self.psm] text = {6: "base", 7: "alt7", 13: "alt13"}[self.psm] return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text=text, bbox=BBox(0, 0, 10, 10), confidence=confidence, page_index=0, line_id="p0-l0")])], metadata={"psm": self.psm}, ) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeEngine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "quality_ranked_ensemble" assert document.text == "alt13" routing = document.metadata["quality_ranked_ensemble"] assert routing["chosen_engine"] == "psm13" def test_candidate_engine_supports_script_select_composite_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={ "primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.2, }, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="नेपाल सरकार", bbox=BBox(0, 0, 10, 10), confidence=0.9, page_index=0, line_id="p0-l0")])], ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="राउट गरिएको", bbox=BBox(0, 0, 10, 10), confidence=0.92, page_index=0, line_id="p0-l0")])], ) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "script_select_composite" assert document.text == "राउट गरिएको" routing = document.metadata["script_select_composite"] assert routing["chosen_engine"] == "surya" def test_candidate_engine_supports_script_select_composite_latin_routing(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router-latin", backend="script_select_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "secondary_engine": "tesseract", "script": "latin", "primary_script_threshold": 0.2, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="Invoice Total", bbox=BBox(0, 0, 10, 10), confidence=0.7, page_index=0, line_id="p0-l0")])], ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="Better English OCR", bbox=BBox(0, 0, 10, 10), confidence=0.95, page_index=0, line_id="p0-l0")])], ) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "script_select_composite" assert document.text == "Better English OCR" routing = document.metadata["script_select_composite"] assert routing["chosen_engine"] == "tesseract" def test_candidate_engine_supports_script_select_composite_page_routing(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router-page", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={ "primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.2, "routing_granularity": "page", }, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page(page_index=0, text_lines=[TextLine(text="Invoice Total", bbox=BBox(0, 0, 10, 10), confidence=0.97, page_index=0, line_id="p0-l0")]), Page(page_index=1, text_lines=[TextLine(text="नेपाल सरकार", bbox=BBox(0, 20, 10, 10), confidence=0.91, page_index=1, line_id="p1-l0")]), ], ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page(page_index=0, text_lines=[TextLine(text="Wrong Page", bbox=BBox(0, 0, 10, 10), confidence=0.3, page_index=0, line_id="p0-l0")]), Page(page_index=1, text_lines=[TextLine(text="राउट गरिएको", bbox=BBox(0, 20, 10, 10), confidence=0.95, page_index=1, line_id="p1-l0")]), ], ) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "script_select_composite" assert document.pages[0].blocks[0].text == "Invoice Total" assert document.pages[1].blocks[0].text == "राउट गरिएको" routing = document.metadata["script_select_composite"] assert routing["chosen_engine"] == "mixed" assert routing["routing_granularity"] == "page" assert routing["selected_page_indexes"] == [0, 1] assert routing["page_routing"][0]["chosen_engine"] == "tesseract" assert routing["page_routing"][1]["chosen_engine"] == "surya" def test_candidate_engine_supports_script_select_composite_line_routing(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router-line", backend="script_select_composite", base_model="tesseract+nepali-recognizer", backend_kwargs={ "primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.2, "routing_granularity": "line", }, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine(text="Invoice Total", bbox=BBox(0, 0, 100, 20), confidence=0.97, page_index=0, line_id="p0-l0"), TextLine(text="नेपाल सरकार", bbox=BBox(0, 30, 100, 20), confidence=0.45, page_index=0, line_id="p0-l1"), ], ) ], ) class FakeSecondary: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine(text="Wrong English", bbox=BBox(0, 0, 100, 20), confidence=0.30, page_index=0, line_id="s0"), TextLine(text="नेपाल सरकार", bbox=BBox(0, 30, 100, 20), confidence=0.95, page_index=0, line_id="s1"), ], ) ], ) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSecondary) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "script_select_composite" assert document.text.splitlines() == ["Invoice Total", "नेपाल सरकार"] routing = document.metadata["script_select_composite"] assert routing["chosen_engine"] == "mixed" assert routing["routing_granularity"] == "line" assert routing["secondary_line_count"] == 1 assert routing["line_routing"][0]["chosen_engine"] == "tesseract" assert routing["line_routing"][1]["chosen_engine"] == "surya" def test_script_select_composite_can_backfill_secondary_structure(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-script-router-backfill", backend="script_select_composite", base_model="tesseract+surya", backend_kwargs={ "primary_engine": "tesseract", "secondary_engine": "surya", "primary_script_threshold": 0.9, "secondary_structure_backfill": True, }, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="Invoice Total", bbox=BBox(0, 0, 10, 10), confidence=0.95, page_index=0, line_id="p0-l0")])], ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="ignored", bbox=BBox(0, 0, 10, 10), confidence=0.8, page_index=0, line_id="p0-l0")])], tables=[ Table( table_id="p0-t0", page_index=0, bbox=BBox(0, 0, 100, 50), cells=[TableCell(row=0, col=0, text="Item"), TableCell(row=1, col=0, text="Masks")], ) ], ) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.text == "Invoice Total" assert len(document.tables) == 1 routing = document.metadata["script_select_composite"] assert routing["chosen_engine"] == "tesseract" assert routing["secondary_structure_backfill"] is True assert document.metadata["structure_backfill"]["secondary_tables_pages"] == [0] def test_candidate_engine_supports_line_align_composite_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake-png") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-line-align", backend="line_align_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "secondary_engine": "tesseract", }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine(text="दोस्रो", bbox=BBox(0, 20, 40, 10), confidence=0.95, page_index=0, line_id="p0-l-surya-1"), TextLine(text="पहिलो", bbox=BBox(0, 0, 40, 10), confidence=0.96, page_index=0, line_id="p0-l-surya-0"), ], ) ], ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine(text="wrong one", bbox=BBox(0, 0, 40, 10), confidence=0.5, page_index=0, line_id="p0-l0"), TextLine(text="wrong two", bbox=BBox(0, 20, 40, 10), confidence=0.5, page_index=0, line_id="p0-l1"), ], ) ], ) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "line_align_composite" assert document.pages[0].text_lines[0].text == "पहिलो" assert document.pages[0].text_lines[0].line_id == "p0-l0" assert document.pages[0].text_lines[1].text == "दोस्रो" assert document.text == "पहिलो\nदोस्रो" alignment = document.metadata["line_align_composite"] assert alignment["primary_engine"] == "surya" assert alignment["secondary_engine"] == "tesseract" assert alignment["matched_line_count"] == 2 def test_candidate_engine_supports_table_cell_refine_composite_backend(tmp_path: Path, monkeypatch) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") from PIL import Image source = tmp_path / "page.png" Image.new("RGB", (120, 80), "white").save(source) card = write_model_card( tmp_path / "model.json", model_id="ocrtech-table-refine", backend="table_cell_refine_composite", base_model="surya+tesseract", backend_kwargs={ "structure_engine": "surya", "crop_engine": "tesseract", "crop_padding": 0, "replacement_policy": "always", }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 10, 80, 40), cells=[ TableCell(row=0, col=0, text="old-a"), TableCell(row=0, col=1, text="old-b"), ], ) return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="old-a old-b", bbox=BBox(10, 10, 80, 20), page_index=0)])], tables=[table], ) class FakeTesseract: name = "tesseract" calls: list[Path] = [] def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine self.calls.append(input_path) text = "new-a" if len(self.calls) == 1 else "new-b" return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text=text, bbox=BBox(0, 0, 10, 10), page_index=0)])]) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "table_cell_refine_composite" assert [cell.text for cell in document.tables[0].cells] == ["new-a", "new-b"] refine = document.metadata["table_cell_refine_composite"] assert refine["attempted_cells"] == 2 assert refine["replaced_cells"] == 2 assert document.tables[0].metadata["table_cell_refine"]["attempted_cells"] == 2 def test_table_cell_refine_supports_candidate_crop_engine(tmp_path: Path, monkeypatch) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") from PIL import Image source = tmp_path / "page.png" Image.new("RGB", (120, 80), "white").save(source) leaf_card = write_model_card( tmp_path / "leaf-crop.json", model_id="ocrtech-leaf-crop", backend="sidecar", base_model="fixture", ) card = write_model_card( tmp_path / "composite.json", model_id="ocrtech-table-refine-candidate-crop", backend="table_cell_refine_composite", base_model="surya+trained-candidate", backend_kwargs={ "structure_engine": "surya", "crop_engine": "candidate", "crop_engine_kwargs": {"model_config": leaf_card.name}, "crop_padding": 0, "replacement_policy": "always", }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 10, 80, 40), cells=[TableCell(row=0, col=0, text="old")], ) return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="old", bbox=BBox(10, 10, 80, 20), page_index=0)])], tables=[table], ) class FakeCandidate: name = "candidate" seen_model_config: Path | None = None def __init__(self, *, model_config: str | Path): self.seen_model_config = Path(model_config) FakeCandidate.seen_model_config = self.seen_model_config def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput(pages=[Page(page_index=0, text_lines=[TextLine(text="trained", bbox=BBox(0, 0, 10, 10), page_index=0)])]) def fake_create_engine(name: str, **kwargs: object): if name == "surya": return FakeSurya(**kwargs) if name == "candidate": return FakeCandidate(**kwargs) raise AssertionError(f"unexpected engine request: {name}, kwargs={kwargs}") monkeypatch.setattr("ocrtech.engines.create_engine", fake_create_engine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert FakeCandidate.seen_model_config == leaf_card assert document.tables[0].cells[0].text == "trained" assert document.metadata["table_cell_refine_composite"]["crop_engine"] == "candidate" def test_table_cell_refine_can_deduplicate_overlapping_tables(tmp_path: Path, monkeypatch) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") from PIL import Image source = tmp_path / "page.png" Image.new("RGB", (120, 80), "white").save(source) card = write_model_card( tmp_path / "model.json", model_id="ocrtech-table-refine-dedup", backend="table_cell_refine_composite", base_model="surya+tesseract", backend_kwargs={ "structure_engine": "surya", "crop_engine": "tesseract", "replacement_policy": "empty", "deduplicate_tables": True, "duplicate_iou_threshold": 0.9, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell good = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 10, 80, 40), cells=[ TableCell(row=0, col=0, text="नाम"), TableCell(row=0, col=1, text="रकम"), TableCell(row=1, col=0, text="किरण"), TableCell(row=1, col=1, text="२१००"), ], ) collapsed = Table( table_id="p0-t1", page_index=0, bbox=BBox(10, 10, 80, 40), cells=[TableCell(row=0, col=0, text="नामरकमकिरण२१००")], ) return EngineOutput(pages=[Page(page_index=0)], tables=[collapsed, good]) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): raise AssertionError("non-empty cells should not be reread with replacement_policy=empty") monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert len(document.tables) == 1 assert [cell.text for cell in document.tables[0].cells] == ["नाम", "रकम", "किरण", "२१००"] assert document.metadata["table_cell_refine_composite"]["dropped_duplicate_tables"] == 1 def test_candidate_engine_supports_figure_caption_refine_composite_backend(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") lexicon_path = tmp_path / "names.txt" lexicon_path.write_text("किरण\nमाया\nसीता\nराम\n", encoding="utf-8") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "caption_engine": "tesseract", "min_caption_iou": 0.5, "min_score_delta": 1, "correct_structural_labels": True, "table_name_lexicon_path": str(lexicon_path), "table_name_max_distance": 2, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine lines = [ TextLine(text="वार्षिक प्रतिदिन", bbox=BBox(10, 0, 220, 8), confidence=0.9, page_index=0), TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.9, page_index=0), TextLine(text="चतुर्थ १: Revenue growth from 2021 to 2025.", bbox=BBox(10, 50, 500, 30), confidence=0.9, page_index=0), TextLine(text="नष्कर्षः अन्तिम दुई वर्षमा वृद्धि भएको छ।", bbox=BBox(10, 90, 500, 30), confidence=0.9, page_index=0), ] table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 140, 300, 60), cells=[ TableCell(row=0, col=0, text="नाम"), TableCell(row=0, col=1, text="रकम"), TableCell(row=1, col=0, text="करिण"), TableCell(row=1, col=1, text="२१००"), ], ) return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], tables=[table], metadata={"engine": "surya"}) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine lines = [ TextLine(text="वार्षिक प्रतविदन", bbox=BBox(10, 0, 220, 8), confidence=0.8, page_index=0), TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.8, page_index=0), TextLine(text="चत्रि १: Revenue growth from 2021 to 2025.", bbox=BBox(10, 38, 500, 42), confidence=0.8, page_index=0), TextLine(text="नष्कर्षः अन्तिम दुई वर्षमा वृद्धि भएको छ।", bbox=BBox(10, 90, 500, 30), confidence=0.8, page_index=0), ] return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], metadata={"engine": "tesseract"}) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.metadata["candidate_model"]["backend"] == "figure_caption_refine_composite" assert document.metadata["figure_caption_refine_composite"]["replaced_captions"] == 1 assert document.figures[0].caption == "चित्र १: Revenue growth from 2021 to 2025." assert document.figures[0].metadata["line_ids"] == ["p0-l1", "p0-l2"] assert document.pages[0].text_lines[0].text == "वार्षिक प्रतिवेदन" assert document.pages[0].text_lines[2].metadata["figure_caption_refine"]["original_text"].startswith("चतुर्थ") assert document.pages[0].text_lines[2].metadata["structural_label_correction"]["corrected_text"].startswith("चित्र") assert document.pages[0].text_lines[3].text.startswith("निष्कर्ष:") assert [cell.text for cell in document.tables[0].cells] == ["नाम", "रकम", "किरण", "२१००"] assert document.metadata["figure_caption_refine_composite"]["table_name_corrections"] == 1 def test_figure_caption_refine_supports_candidate_caption_engine(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") leaf_card = write_model_card( tmp_path / "leaf-caption.json", model_id="ocrtech-leaf-caption", backend="sidecar", base_model="fixture", ) card = write_model_card( tmp_path / "composite.json", model_id="ocrtech-figure-caption-candidate", backend="figure_caption_refine_composite", base_model="surya+trained-candidate", backend_kwargs={ "primary_engine": "surya", "caption_engine": "candidate", "caption_engine_kwargs": {"model_config": leaf_card.name}, "min_caption_iou": 0.5, "min_score_delta": 1, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine lines = [ TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.9, page_index=0), TextLine(text="चतुर्थ १: Revenue growth.", bbox=BBox(10, 50, 500, 30), confidence=0.9, page_index=0), ] return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], metadata={"engine": "surya"}) class FakeCandidate: name = "candidate" seen_model_config: Path | None = None def __init__(self, *, model_config: str | Path): self.seen_model_config = Path(model_config) FakeCandidate.seen_model_config = self.seen_model_config def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine lines = [ TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 10, 250, 30), confidence=0.8, page_index=0), TextLine(text="चित्र १: Revenue growth.", bbox=BBox(10, 50, 500, 30), confidence=0.8, page_index=0), ] return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], metadata={"engine": "candidate"}) def fake_create_engine(name: str, **kwargs: object): if name == "surya": return FakeSurya(**kwargs) if name == "candidate": return FakeCandidate(**kwargs) raise AssertionError(f"unexpected engine request: {name}, kwargs={kwargs}") monkeypatch.setattr("ocrtech.engines.create_engine", fake_create_engine) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert FakeCandidate.seen_model_config == leaf_card assert document.metadata["figure_caption_refine_composite"]["caption_engine"] == "candidate" assert document.metadata["figure_caption_refine_composite"]["replaced_captions"] == 1 assert document.figures[0].caption == "चित्र १: Revenue growth." def test_figure_caption_refine_corrects_daily_ledger_structural_title(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine-daily-ledger", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "caption_engine": "tesseract", "correct_structural_labels": True, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine(text="दैनकि हसिाब", bbox=BBox(10, 0, 220, 20), confidence=0.9, page_index=0), ], ) ], metadata={"engine": "surya"}, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, TextLine return EngineOutput( pages=[ Page( page_index=0, text_lines=[ TextLine(text="दैनकि हसिाब", bbox=BBox(10, 0, 220, 20), confidence=0.8, page_index=0), ], ) ], metadata={"engine": "tesseract"}, ) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.pages[0].text_lines[0].text == "दैनिक हिसाब" assert document.pages[0].text_lines[0].metadata["structural_label_correction"] == { "original_text": "दैनकि हसिाब", "corrected_text": "दैनिक हिसाब", } assert document.text == "दैनिक हिसाब" def test_figure_caption_refine_preserves_table_name_suffix_when_using_lexicon(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") lexicon_path = tmp_path / "names.txt" lexicon_path.write_text("किरण\nमाया\n", encoding="utf-8") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine-suffixed-name", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "caption_engine": "tesseract", "table_name_lexicon_path": str(lexicon_path), "table_name_max_distance": 2, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 40, 300, 60), cells=[ TableCell(row=0, col=0, text="नाम"), TableCell(row=0, col=1, text="रकम"), TableCell(row=1, col=0, text="करिण-302"), TableCell(row=1, col=1, text="२१००"), TableCell(row=2, col=0, text="माया-302"), TableCell(row=2, col=1, text="२९००"), ], ) return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="दैनिक हिसाब", bbox=BBox(10, 0, 200, 20), page_index=0)])], tables=[table], metadata={"engine": "surya"}, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput return EngineOutput(pages=[], metadata={"engine": "tesseract"}) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert [cell.text for cell in document.tables[0].cells] == ["नाम", "रकम", "किरण-302", "२१००", "माया-302", "२९००"] assert document.metadata["figure_caption_refine_composite"]["table_name_corrections"] == 1 def test_figure_caption_refine_corrects_name_cells_below_grouped_header(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") lexicon_path = tmp_path / "names.txt" lexicon_path.write_text("किरण\nमाया\n", encoding="utf-8") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine-grouped-header-name", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "caption_engine": "tesseract", "table_name_lexicon_path": str(lexicon_path), "table_name_max_distance": 2, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 40, 300, 120), cells=[ TableCell(row=0, col=0, text="Ledger Summary", colspan=2), TableCell(row=0, col=2, text="Status"), TableCell(row=1, col=0, text="नाम"), TableCell(row=1, col=1, text="रकम"), TableCell(row=1, col=2, text="Remarks"), TableCell(row=2, col=0, text="करिण-302"), TableCell(row=2, col=1, text="२१००"), TableCell(row=2, col=2, text="Paid"), ], ) return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="समेकित हिसाब", bbox=BBox(10, 0, 200, 20), page_index=0)])], tables=[table], metadata={"engine": "surya"}, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput return EngineOutput(pages=[], metadata={"engine": "tesseract"}) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert [cell.text for cell in document.tables[0].cells if cell.col == 0] == ["Ledger Summary", "नाम", "किरण-302"] assert document.metadata["figure_caption_refine_composite"]["table_name_corrections"] == 1 def test_figure_caption_refine_infers_flat_repeated_header_colspan(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine-infer-colspan", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "caption_engine": "tesseract", "infer_table_header_colspans": True, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell, TextLine table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 40, 300, 120), cells=[ TableCell(row=0, col=0, text="Ledger Summary"), TableCell(row=0, col=1, text="Ledger Summary"), TableCell(row=0, col=2, text="Status"), TableCell(row=1, col=0, text="नाम"), TableCell(row=1, col=1, text="रकम"), TableCell(row=1, col=2, text="Remarks"), ], ) return EngineOutput( pages=[Page(page_index=0, text_lines=[TextLine(text="समेकित हिसाब", bbox=BBox(10, 0, 200, 20), page_index=0)])], tables=[table], metadata={"engine": "surya"}, ) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput return EngineOutput(pages=[], metadata={"engine": "tesseract"}) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert document.tables[0].cells[0].text == "Ledger Summary" assert document.tables[0].cells[0].colspan == 2 assert len(document.tables[0].cells) == 5 assert 'colspan="2"' in document.tables[0].html assert document.metadata["figure_caption_refine_composite"]["inferred_table_header_colspans"] == 1 def test_figure_caption_refine_normalizes_separator_and_duplicate_table_columns(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine-normalize-grid", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "caption_engine": "tesseract", "normalize_table_grids": True, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 40, 300, 120), cells=[ TableCell(row=0, col=0, text="—"), TableCell(row=0, col=1, text="Skull cap"), TableCell(row=0, col=2, text="1D"), TableCell(row=0, col=3, text="1D"), TableCell(row=1, col=0, text="—"), TableCell(row=1, col=1, text="Helmet"), TableCell(row=1, col=2, text="2D"), TableCell(row=1, col=3, text="2D"), ], ) return EngineOutput(pages=[Page(page_index=0)], tables=[table], metadata={"engine": "surya"}) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput return EngineOutput(pages=[], metadata={"engine": "tesseract"}) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert [cell.text for cell in document.tables[0].cells] == ["Skull cap", "1D", "Helmet", "2D"] assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_dropped_columns"] == 2 assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_merged_rows"] == 0 assert document.tables[0].metadata["normalized_table_grid_dropped_columns"] == 2 def test_figure_caption_refine_merges_sparse_bullet_table_rows(tmp_path: Path, monkeypatch) -> None: source = tmp_path / "page.png" source.write_bytes(b"fake image bytes") card = write_model_card( tmp_path / "model.json", model_id="ocrtech-figure-caption-refine-normalize-row-fragments", backend="figure_caption_refine_composite", base_model="surya+tesseract", backend_kwargs={ "primary_engine": "surya", "caption_engine": "tesseract", "normalize_table_grids": True, }, ) class FakeSurya: name = "surya" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import BBox, EngineOutput, Page, Table, TableCell table = Table( table_id="p0-t0", page_index=0, bbox=BBox(10, 40, 300, 120), cells=[ TableCell(row=0, col=0, text="Type of Anomaly"), TableCell(row=0, col=1, text="Variant"), TableCell(row=0, col=2, text="Type"), TableCell(row=1, col=0, text=""), TableCell(row=1, col=1, text="Coronaries originating"), TableCell(row=1, col=2, text="- left main"), TableCell(row=2, col=0, text="- right coronary"), TableCell(row=2, col=1, text=""), TableCell(row=2, col=2, text=""), ], ) return EngineOutput(pages=[Page(page_index=0)], tables=[table], metadata={"engine": "surya"}) class FakeTesseract: name = "tesseract" def __init__(self, **kwargs): self.kwargs = kwargs def recognize(self, input_path: Path): from ocrtech.engines import EngineOutput return EngineOutput(pages=[], metadata={"engine": "tesseract"}) monkeypatch.setattr("ocrtech.engines.SuryaEngine", FakeSurya) monkeypatch.setattr("ocrtech.engines.TesseractEngine", FakeTesseract) document = parse_document(source, tmp_path / "out", engine_name="candidate", model_config=card) assert [cell.text for cell in document.tables[0].cells] == [ "Type of Anomaly", "Variant", "Type", "", "Coronaries originating", "- left main\n- right coronary", ] assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_dropped_columns"] == 0 assert document.metadata["figure_caption_refine_composite"]["normalized_table_grid_merged_rows"] == 1 assert document.tables[0].metadata["normalized_table_grid_merged_rows"] == 1 def test_text_correction_composite_clears_generation_max_length(tmp_path: Path, monkeypatch) -> None: model_dir = tmp_path / "model" model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") class FakeTorch: class cuda: @staticmethod def is_available() -> bool: return False class backends: class mps: @staticmethod def is_available() -> bool: return False @staticmethod def no_grad(): class _NoGrad: def __enter__(self): return None def __exit__(self, exc_type, exc, tb): return False return _NoGrad() class FakeTokenizer: @classmethod def from_pretrained(cls, source: str, use_fast: bool = False): return cls() def __call__(self, texts, padding: bool, truncation: bool, return_tensors: str): return {"input_ids": SimpleNamespace(to=lambda device: [[1, 2, 3]])} def batch_decode(self, generated_ids, skip_special_tokens: bool): return ["सही पाठ"] class FakeGenerationConfig: def __init__(self) -> None: self.max_length = 256 fake_model = SimpleNamespace(generation_config=FakeGenerationConfig()) def fake_from_pretrained(source: str): fake_model.to = lambda device: fake_model fake_model.eval = lambda: None fake_model.generate = lambda **kwargs: [[1, 2, 3]] return fake_model monkeypatch.setitem(sys.modules, "torch", FakeTorch) monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(AutoModelForSeq2SeqLM=SimpleNamespace(from_pretrained=fake_from_pretrained), AutoTokenizer=FakeTokenizer)) monkeypatch.setattr("ocrtech.engines.create_engine", lambda name, **kwargs: SimpleNamespace(name=name, recognize=lambda input_path: SimpleNamespace(pages=[], tables=[], figures=[], metadata={}))) from ocrtech.engines import TextCorrectionCompositeEngine TextCorrectionCompositeEngine(model_dir=str(model_dir), tokenizer_dir=str(model_dir), base_engine="tesseract") assert fake_model.generation_config.max_length is None def test_hf_engine_loads_processor_without_fast_tokenizer(tmp_path: Path, monkeypatch) -> None: model_dir = tmp_path / "model" model_dir.mkdir() (model_dir / "config.json").write_text("{}", encoding="utf-8") calls: list[tuple[str, object]] = [] class FakeTorch: class cuda: @staticmethod def is_available() -> bool: return False class backends: class mps: @staticmethod def is_available() -> bool: return False @staticmethod def no_grad(): class _NoGrad: def __enter__(self): return None def __exit__(self, exc_type, exc, tb): return False return _NoGrad() class FakeImage: @staticmethod def open(path: Path): class _Image: def convert(self, mode: str): return self return _Image() class FakeProcessor: @classmethod def from_pretrained(cls, source: str, **kwargs): calls.append(("processor", kwargs)) return cls() def __call__(self, image, return_tensors: str): return SimpleNamespace(pixel_values=SimpleNamespace(to=lambda device: "pixels")) def batch_decode(self, generated_ids, skip_special_tokens: bool): return ["नेपाल"] class FakeModel: @classmethod def from_pretrained(cls, source: str): calls.append(("model", source)) return cls() def to(self, device: str): return self def eval(self): return None def generate(self, pixel_values, max_new_tokens: int): return [[1, 2, 3]] monkeypatch.setitem(sys.modules, "torch", FakeTorch) monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(TrOCRProcessor=FakeProcessor, VisionEncoderDecoderModel=FakeModel)) monkeypatch.setitem(sys.modules, "PIL", SimpleNamespace(Image=FakeImage)) from ocrtech.engines import HuggingFaceVisionTextEngine engine = HuggingFaceVisionTextEngine(model_dir=str(model_dir)) output = engine.recognize(tmp_path / "page.png") assert calls[0] == ("processor", {"backend": "pil"}) assert output.pages[0].text_lines[0].text == "नेपाल" def test_huggingface_vision_text_engine_sanitizes_list_extra_special_tokens(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: model_dir = tmp_path / "model" model_dir.mkdir() (model_dir / "tokenizer_config.json").write_text( json.dumps({"extra_special_tokens": ["<s>", "</s>"]}), encoding="utf-8", ) calls: list[tuple[str, object]] = [] class FakeTorch: class cuda: @staticmethod def is_available() -> bool: return False class backends: class mps: @staticmethod def is_available() -> bool: return False @staticmethod def no_grad(): class _NoGrad: def __enter__(self): return None def __exit__(self, exc_type, exc, tb): return False return _NoGrad() class FakeImage: @staticmethod def open(path: Path): class _Image: def convert(self, mode: str): return self return _Image() class FakeProcessor: @classmethod def from_pretrained(cls, source: str, **kwargs): config = json.loads((Path(source) / "tokenizer_config.json").read_text(encoding="utf-8")) calls.append(("processor", config.get("extra_special_tokens"))) if isinstance(config.get("extra_special_tokens"), list): raise AttributeError("'list' object has no attribute 'keys'") return cls() def __call__(self, image, return_tensors: str): return SimpleNamespace(pixel_values=SimpleNamespace(to=lambda device: "pixels")) def batch_decode(self, generated_ids, skip_special_tokens: bool): return ["नेपाल"] class FakeModel: generation_config = SimpleNamespace(max_length=128) @classmethod def from_pretrained(cls, source: str): return cls() def to(self, device: str): return self def eval(self): return None def generate(self, pixel_values, max_new_tokens: int): return [[1, 2, 3]] monkeypatch.setitem(sys.modules, "torch", FakeTorch) monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(TrOCRProcessor=FakeProcessor, VisionEncoderDecoderModel=FakeModel)) monkeypatch.setitem(sys.modules, "PIL", SimpleNamespace(Image=FakeImage)) from ocrtech.engines import HuggingFaceVisionTextEngine engine = HuggingFaceVisionTextEngine(model_dir=str(model_dir)) output = engine.recognize(tmp_path / "page.png") assert calls == [("processor", ["<s>", "</s>"]), ("processor", {})] assert output.pages[0].text_lines[0].text == "नेपाल" def test_benchmark_candidate_model_config(tmp_path: Path) -> None: source = tmp_path / "page.txt" source.write_text("hello world\n", encoding="utf-8") source.with_suffix(".ref.txt").write_text("hello world\n", encoding="utf-8") card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") results = run_benchmark([source], tmp_path / "bench", baselines=["candidate"], candidate_model_config=card) assert results[0].status == "ok" assert results[0].metrics["cer"] == 0 assert results[0].metrics["quality_score"] > 0 def test_experiment_records_candidate_model_config(tmp_path: Path) -> None: pack_dir = tmp_path / "pack" inputs_dir = pack_dir / "inputs" refs_dir = pack_dir / "references" manifests_dir = pack_dir / "manifests" inputs_dir.mkdir(parents=True) refs_dir.mkdir() manifests_dir.mkdir() source = inputs_dir / "sample.txt" source.write_text("hello world\n", encoding="utf-8") refs_dir.joinpath("sample.txt.ref.txt").write_text("hello world\n", encoding="utf-8") manifest = manifests_dir / "eval.jsonl" manifest.write_text( '{"sample_id":"s1","dataset":"fixture","split":"eval","image_path":"' + str(source) + '","text":"hello world","sha256":null,"metadata":{"slices":["english"]}}\n', encoding="utf-8", ) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") fallback_card = write_model_card(tmp_path / "fallback.json", model_id="ocrtech-fallback", backend="sidecar", base_model="fixture") report = run_experiment( manifest, tmp_path / "experiment", baselines=["candidate"], references_dir=refs_dir, candidate_model_config=card, fallback_model_config=fallback_card, low_confidence_threshold=0.73, run_validation=False, ) assert report.environment["candidate_model_config"] == str(card) assert report.environment["fallback_model_config"] == str(fallback_card) assert report.environment["fallback_model_config_sha256"] assert report.environment["low_confidence_threshold"] == 0.73 def _passing_validation_gate() -> dict[str, object]: return { "passed": True, "pairs": 2, "candidate_mean": 0.1, "baseline_mean": 0.2, "absolute_improvement": 0.1, "relative_improvement": 0.5, "improvement_ci_low": 0.01, "win_count": 2, "loss_count": 0, "tie_count": 0, "pair_win_rate": 1.0, "sign_test_p_value": 0.0, "gate": { "baseline": "tesseract", "metric": "cer", "direction": "lower", "min_pairs": 2, "min_absolute_margin": 0.0, "min_relative_improvement": 0.0, "min_pair_win_rate": 0.5, "max_p_value": 1.0, "require_ci_above_zero": True, }, }
from __future__ import annotations
import json
import importlib.util
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from ocrtech.benchmark import run_benchmark
from ocrtech.experiments import run_experiment
from ocrtech.manifest import sha256_file
from ocrtech.models import ModelCard, audit_model_card, component_baseline_names, inspect_routed_calibration_collapse, model_admission_status, trained_recognizer_provenance_status, write_model_card
from ocrtech.pipeline import parse_document
def test_model_card_audit_passes_for_existing_artifact(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
card = write_model_card(
tmp_path / "model.json",
model_id="ocrtech-test",
backend="sidecar",
base_model="fixture",
artifact_paths=[artifact],
notes="test model card",
)
audit = audit_model_card(card, tmp_path / "audit")
assert audit.passed
assert audit.model_id == "ocrtech-test"
assert (tmp_path / "audit" / "model-audit.json").exists()
def test_model_card_records_and_checks_admission_validation(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
validation = tmp_path / "validation.json"
model_path = tmp_path / "model.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(model_path)}},
"gates": [_passing_validation_gate()],
}
)
+ "\n",
encoding="utf-8",
)
card_path = write_model_card(
model_path,
model_id="ocrtech-admitted",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
admission_validation_report=validation,
)
card = ModelCard.from_path(card_path)
status = model_admission_status(card, card_path)
assert status.passed
assert card.metrics["admission_validation"]["sha256"]
def test_model_admission_status_requires_quantitative_gate_evidence(tmp_path: Path) -> None:
artifact = tmp_path / "inference.pdiparams"
artifact.write_bytes(b"weights")
validation = tmp_path / "validation.json"
model_path = tmp_path / "model.json"
validation.write_text(
json.dumps(
{
"claim_status": "validated",
"passed": True,
"provenance": {"candidate_model_config": {"path": str(model_path)}},
"gates": [],
}
)
+ "\n",
encoding="utf-8",
)
card_path = write_model_card(
model_path,
model_id="ocrtech-admission-without-gates",
backend="paddleocr",
base_model="fixture",
artifact_paths=[artifact],
backend_kwargs={"lang": "ne"},
admission_validation_report=validation,
)
status = model_admission_status(ModelCard.from_path(card_path), card_path)
assert not status.passed