from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech import pipeline
from ocrtech.benchmark import _quality_metrics
from ocrtech.engines import EngineOutput
from ocrtech.pipeline import parse_document
from ocrtech.schemas import BBox, Figure, Page, Table, TableCell, TextLine
def test_parse_doc_exports_json_markdown_tables_and_figures(tmp_path: Path) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
sidecar = tmp_path / "page.ocr.json"
sidecar.write_text(
json.dumps(
{
"pages": [
{
"width": 800,
"height": 1000,
"lines": [
{"text": "नेपाल सरकार", "bbox": [10, 10, 200, 30], "confidence": 0.99},
{"text": "नाम उमेर जिल्ला", "bbox": [10, 80, 400, 20], "confidence": 0.95},
{"text": "राम ३२ काठमाडौं", "bbox": [10, 110, 420, 20], "confidence": 0.94},
{"text": "Sita 29 Lalitpur", "bbox": [10, 140, 420, 20], "confidence": 0.93},
{"text": "This is English text.", "bbox": [10, 220, 300, 20], "confidence": 0.98},
],
}
],
"figures": [
{
"figure_id": "fig-0",
"page_index": 0,
"bbox": [100, 300, 200, 120],
"caption": "चित्र १",
"summary": "A figure placeholder.",
"confidence": 0.8,
}
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
assert len(document.pages) == 1
assert len(document.tables) == 1
assert len(document.figures) == 1
assert (tmp_path / "out" / "document.json").exists()
assert (tmp_path / "out" / "document.md").exists()
assert (tmp_path / "out" / "tables" / "p0-t0.csv").exists()
assert (tmp_path / "out" / "tables" / "p0-t0.html").exists()
assert (tmp_path / "out" / "figures" / "metadata.json").exists()
assert (tmp_path / "out" / "quality.json").exists()
exported = json.loads((tmp_path / "out" / "document.json").read_text(encoding="utf-8"))
assert exported["tables"][0]["csv_path"] == str(tmp_path / "out" / "tables" / "p0-t0.csv")
assert exported["tables"][0]["html_path"] == str(tmp_path / "out" / "tables" / "p0-t0.html")
assert exported["tables"][0]["html"].startswith("
")
markdown = (tmp_path / "out" / "document.md").read_text(encoding="utf-8")
assert "# नेपाल सरकार" in markdown
assert "| नाम | उमेर | जिल्ला |" in markdown
assert "This is English text." in markdown
assert document.metadata["quality"]["passed"]
def test_export_document_crops_figure_from_source_image_when_image_path_missing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
try:
from PIL import Image
except ImportError:
pytest.skip("Pillow is not installed")
Image.new("RGB", (120, 100), "white").save(source)
class FigureEngine:
def recognize(self, input_path: Path) -> EngineOutput:
line = TextLine(text="[Figure: chart]", bbox=BBox(10, 10, 60, 20), confidence=0.9, page_index=0)
figure = Figure(figure_id="fig-0", page_index=0, bbox=BBox(20, 30, 40, 35), summary="chart")
return EngineOutput(pages=[Page(page_index=0, text_lines=[line])], figures=[figure], metadata={"engine": "figure-engine"})
monkeypatch.setattr(pipeline, "create_engine", lambda name, **kwargs: FigureEngine())
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
cropped = tmp_path / "out" / "figures" / "fig-0.png"
assert cropped.exists()
assert document.figures[0].image_path == str(cropped)
assert document.figures[0].metadata["export_source"] == "source_crop"
with Image.open(cropped) as image:
assert image.size == (40, 35)
metadata = json.loads((tmp_path / "out" / "figures" / "metadata.json").read_text(encoding="utf-8"))
assert metadata[0]["image_path"] == str(cropped)
def test_export_document_records_warning_when_figure_image_cannot_be_created(tmp_path: Path) -> None:
source = tmp_path / "page.pdf"
source.write_bytes(b"%PDF-1.4\n")
sidecar = tmp_path / "page.ocr.json"
sidecar.write_text(
json.dumps(
{
"pages": [{"lines": [{"text": "[Figure: chart]", "bbox": [10, 10, 80, 20], "confidence": 0.9}]}],
"figures": [{"figure_id": "fig-0", "page_index": 0, "bbox": [20, 30, 40, 35], "summary": "chart"}],
}
),
encoding="utf-8",
)
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
assert not (tmp_path / "out" / "figures" / "fig-0.png").exists()
warnings = document.figures[0].metadata["export_warnings"]
assert warnings == ["source image format cannot be cropped: .pdf"]
metadata = json.loads((tmp_path / "out" / "figures" / "metadata.json").read_text(encoding="utf-8"))
assert metadata[0]["metadata"]["export_warnings"] == warnings
def test_export_document_copies_relative_figure_image_from_page_bundle(tmp_path: Path) -> None:
source = tmp_path / "bundle"
source.mkdir()
figure_image = source / "embedded-figure.png"
figure_image.write_bytes(b"figure bytes")
(source / "document.ocr.json").write_text(
json.dumps(
{
"pages": [{"lines": [{"text": "[Figure: chart]", "bbox": [10, 10, 80, 20], "confidence": 0.9}]}],
"figures": [
{
"figure_id": "fig-0",
"page_index": 0,
"bbox": [20, 30, 40, 35],
"image_path": "embedded-figure.png",
}
],
}
),
encoding="utf-8",
)
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
copied = tmp_path / "out" / "figures" / "embedded-figure.png"
assert copied.read_bytes() == b"figure bytes"
assert document.figures[0].image_path == str(copied)
def test_export_document_crops_figure_from_page_bundle(tmp_path: Path) -> None:
source = tmp_path / "bundle"
source.mkdir()
try:
from PIL import Image
except ImportError:
pytest.skip("Pillow is not installed")
Image.new("RGB", (120, 100), "white").save(source / "page-0000.png")
Image.new("RGB", (160, 140), "white").save(source / "page-0001.png")
(source / "document.ocr.json").write_text(
json.dumps(
{
"pages": [
{"lines": [{"text": "page one", "bbox": [0, 0, 80, 20], "confidence": 0.9}]},
{"lines": [{"text": "[Figure: chart]", "bbox": [10, 10, 80, 20], "confidence": 0.9}]},
],
"figures": [{"figure_id": "fig-1", "page_index": 1, "bbox": [30, 40, 50, 45], "summary": "chart"}],
}
),
encoding="utf-8",
)
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
cropped = tmp_path / "out" / "figures" / "fig-1.png"
assert cropped.exists()
assert document.figures[0].image_path == str(cropped)
assert document.figures[0].metadata["export_source"] == "source_crop"
with Image.open(cropped) as image:
assert image.size == (50, 45)
def test_export_document_records_warning_when_page_bundle_has_no_images(tmp_path: Path) -> None:
source = tmp_path / "bundle"
source.mkdir()
(source / "document.ocr.json").write_text(
json.dumps(
{
"pages": [{"lines": [{"text": "[Figure: chart]", "bbox": [10, 10, 80, 20], "confidence": 0.9}]}],
"figures": [{"figure_id": "fig-0", "page_index": 0, "bbox": [20, 30, 40, 35], "summary": "chart"}],
}
),
encoding="utf-8",
)
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
warnings = document.figures[0].metadata["export_warnings"]
assert warnings == [f"page bundle has no croppable image files: {source}"]
def test_fallback_triggers_on_low_confidence(tmp_path: Path) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
sidecar = tmp_path / "page.ocr.json"
sidecar.write_text(
json.dumps(
{
"pages": [
{
"lines": [
{"text": "low confidence", "bbox": [0, 0, 100, 20], "confidence": 0.2},
]
}
]
}
),
encoding="utf-8",
)
document = parse_document(source, tmp_path / "out", engine_name="sidecar", fallback_engine="paddleocr")
assert "fallback_error" in document.metadata
assert document.metadata["fallback"]["triggered"] is True
assert document.metadata["fallback"]["outcome"] == "error"
assert document.metadata["fallback"]["reason"] == "average_confidence_below_threshold"
assert document.pages[0].blocks[0].text == "low confidence"
def test_fallback_success_records_decision_and_benchmark_metrics(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
class PrimaryEngine:
def recognize(self, input_path: Path) -> EngineOutput:
line = TextLine(text="primary text", bbox=BBox(0, 0, 100, 20), confidence=0.2, page_index=0, line_id="p0-l0")
return EngineOutput(pages=[Page(page_index=0, text_lines=[line])], metadata={"engine": "primary"})
class FallbackEngine:
def recognize(self, input_path: Path) -> EngineOutput:
line = TextLine(text="fallback text", bbox=BBox(0, 0, 120, 20), confidence=0.95, page_index=0, line_id="p0-l0")
return EngineOutput(pages=[Page(page_index=0, text_lines=[line])], metadata={"engine": "fallback"})
def fake_create_engine(name: str, **kwargs: object) -> object:
if name == "sidecar":
return PrimaryEngine()
if name == "paddleocr":
return FallbackEngine()
raise AssertionError(f"unexpected engine request: {name}, kwargs={kwargs}")
monkeypatch.setattr(pipeline, "create_engine", fake_create_engine)
document = parse_document(source, tmp_path / "out", engine_name="sidecar", fallback_engine="paddleocr")
metrics = _quality_metrics(document)
assert document.pages[0].blocks[0].text == "fallback text"
assert document.metadata["fallback"] == {
"configured": True,
"triggered": True,
"outcome": "success",
"primary_engine": "sidecar",
"fallback_engine": "paddleocr",
"fallback_model_config": None,
"primary_average_confidence": 0.2,
"threshold": 0.8,
"reason": "average_confidence_below_threshold",
}
assert metrics["fallback_triggered"] == 1.0
assert metrics["fallback_succeeded"] == 1.0
assert metrics["fallback_failed"] == 0.0
assert metrics["fallback_primary_average_confidence"] == 0.2
def test_fallback_not_triggered_records_reason(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
class PrimaryEngine:
def recognize(self, input_path: Path) -> EngineOutput:
line = TextLine(text="primary text", bbox=BBox(0, 0, 100, 20), confidence=0.99, page_index=0, line_id="p0-l0")
return EngineOutput(pages=[Page(page_index=0, text_lines=[line])], metadata={"engine": "primary"})
def fake_create_engine(name: str, **kwargs: object) -> object:
if name == "sidecar":
return PrimaryEngine()
raise AssertionError(f"fallback should not be constructed: {name}, kwargs={kwargs}")
monkeypatch.setattr(pipeline, "create_engine", fake_create_engine)
document = parse_document(source, tmp_path / "out", engine_name="sidecar", fallback_engine="paddleocr")
metrics = _quality_metrics(document)
assert document.metadata["fallback"]["triggered"] is False
assert document.metadata["fallback"]["outcome"] == "not_triggered"
assert document.metadata["fallback"]["reason"] == "confidence_above_threshold"
assert metrics["fallback_triggered"] == 0.0
def test_fallback_model_config_uses_candidate_engine(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
fallback_model = tmp_path / "fallback-model.json"
fallback_model.write_text("{}", encoding="utf-8")
calls: list[tuple[str, dict[str, object]]] = []
class PrimaryEngine:
def recognize(self, input_path: Path) -> EngineOutput:
line = TextLine(text="primary text", bbox=BBox(0, 0, 100, 20), confidence=0.2, page_index=0, line_id="p0-l0")
return EngineOutput(pages=[Page(page_index=0, text_lines=[line])], metadata={"engine": "primary"})
class FallbackEngine:
def recognize(self, input_path: Path) -> EngineOutput:
line = TextLine(text="model fallback", bbox=BBox(0, 0, 140, 20), confidence=0.96, page_index=0, line_id="p0-l0")
return EngineOutput(pages=[Page(page_index=0, text_lines=[line])], metadata={"engine": "fallback"})
def fake_create_engine(name: str, **kwargs: object) -> object:
calls.append((name, kwargs))
if name == "sidecar":
return PrimaryEngine()
if name == "candidate":
return FallbackEngine()
raise AssertionError(f"unexpected engine request: {name}, kwargs={kwargs}")
monkeypatch.setattr(pipeline, "create_engine", fake_create_engine)
document = parse_document(source, tmp_path / "out", engine_name="sidecar", fallback_model_config=fallback_model)
assert calls == [("sidecar", {}), ("candidate", {"model_config": fallback_model})]
assert document.pages[0].blocks[0].text == "model fallback"
assert document.metadata["fallback"]["fallback_engine"] == "candidate"
assert document.metadata["fallback"]["fallback_model_config"] == str(fallback_model)
assert document.metadata["fallback"]["outcome"] == "success"
def test_build_document_does_not_duplicate_engine_supplied_table(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
class TableEngine:
def recognize(self, input_path: Path) -> EngineOutput:
table_bbox = BBox(10, 20, 300, 50)
table = Table(
table_id="p0-t0",
page_index=0,
bbox=table_bbox,
cells=[
TableCell(row=0, col=0, text="नाम"),
TableCell(row=0, col=1, text="रकम"),
TableCell(row=0, col=2, text="Remarks"),
TableCell(row=1, col=0, text="किरण"),
TableCell(row=1, col=1, text="२१००"),
TableCell(row=1, col=2, text="Paid"),
],
)
lines = [
TextLine(text="नाम रकम Remarks", bbox=BBox(10, 20, 300, 20), confidence=0.9, page_index=0),
TextLine(text="किरण २१०० Paid", bbox=BBox(10, 50, 300, 20), confidence=0.9, page_index=0),
]
return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], tables=[table], metadata={"engine": "table-engine"})
monkeypatch.setattr(pipeline, "create_engine", lambda name, **kwargs: TableEngine())
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
assert len(document.tables) == 1
assert document.pages[0].blocks[0].table_id == "p0-t0"
assert (tmp_path / "out" / "tables" / "p0-t0.csv").exists()
def test_figure_marker_uses_nearby_ocr_damaged_caption(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
class FigureEngine:
def recognize(self, input_path: Path) -> EngineOutput:
lines = [
TextLine(text="वार्षिक प्रतिवेदन", bbox=BBox(10, 10, 200, 30), confidence=0.9, page_index=0),
TextLine(text="[Figure: revenue chart]", bbox=BBox(10, 60, 260, 30), confidence=0.9, page_index=0),
TextLine(text="चतुर्थ १: Revenue growth from 2021 to 2025.", bbox=BBox(10, 100, 500, 30), confidence=0.9, page_index=0),
TextLine(text="निष्कर्ष: अन्तिम दुई वर्षमा वृद्धि भएको छ।", bbox=BBox(10, 150, 500, 30), confidence=0.9, page_index=0),
]
return EngineOutput(pages=[Page(page_index=0, text_lines=lines)], metadata={"engine": "figure-engine"})
monkeypatch.setattr(pipeline, "create_engine", lambda name, **kwargs: FigureEngine())
document = parse_document(source, tmp_path / "out", engine_name="sidecar")
assert len(document.figures) == 1
assert document.figures[0].summary == "[Figure: revenue chart]"
assert document.figures[0].caption == "चतुर्थ १: Revenue growth from 2021 to 2025."
assert document.pages[0].blocks[1].text == "चतुर्थ १: Revenue growth from 2021 to 2025."
assert "निष्कर्ष" in document.pages[0].blocks[2].text
def test_fallback_records_unexpected_runtime_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
source = tmp_path / "page.png"
source.write_bytes(b"fake image bytes")
class PrimaryEngine:
def recognize(self, input_path: Path) -> EngineOutput:
line = TextLine(text="primary text", bbox=BBox(0, 0, 100, 20), confidence=0.2, page_index=0, line_id="p0-l0")
return EngineOutput(pages=[Page(page_index=0, text_lines=[line])], metadata={"engine": "primary"})
def fake_create_engine(name: str, **kwargs: object) -> object:
if name == "sidecar":
return PrimaryEngine()
if name == "paddleocr":
raise RuntimeError("paddle runtime missing")
raise AssertionError(f"unexpected engine request: {name}, kwargs={kwargs}")
monkeypatch.setattr(pipeline, "create_engine", fake_create_engine)
document = parse_document(source, tmp_path / "out", engine_name="sidecar", fallback_engine="paddleocr")
assert document.metadata["fallback_error"] == "RuntimeError: paddle runtime missing"
assert document.pages[0].blocks[0].text == "primary text"