from __future__ import annotations import importlib.util import json from pathlib import Path import pytest from ocrtech.benchmark import run_benchmark from ocrtech.crops import extract_crop_manifest from ocrtech.errors import DataValidationError from ocrtech.evalpack import audit_eval_pack, create_eval_pack from ocrtech.manifest import load_manifest def test_create_text_eval_pack_and_audit(tmp_path: Path) -> None: summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) manifest = Path(summary.manifest_path) assert manifest.exists() assert summary.sample_count == 6 assert not summary.claim_ready assert "table" in summary.slices assert "complex_table" in summary.slices assert "multi_page" in summary.slices entries = load_manifest(manifest) assert all(Path(entry.image_path).exists() for entry in entries) assert all(Path(str(entry.metadata["reference_path"])).exists() for entry in entries) audit = audit_eval_pack(manifest, tmp_path / "audit") assert audit.passed assert not audit.claim_ready assert (tmp_path / "audit" / "eval-pack-audit.json").exists() def test_create_full_image_claim_pack_is_claim_ready_when_degradations_cover_required_slices(tmp_path: Path) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") summary = create_eval_pack( tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean", "scan", "blur", "low_contrast", "skew"], ) audit = audit_eval_pack(summary.manifest_path) assert summary.claim_ready assert audit.claim_ready assert audit.passed def test_create_image_eval_pack_writes_multi_page_bundle(tmp_path: Path) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") summary = create_eval_pack(tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean"]) entries = load_manifest(Path(summary.manifest_path)) multi_page_entry = next(entry for entry in entries if "multi-page-brief" in entry.sample_id) bundle = Path(multi_page_entry.image_path) assert summary.sample_count == 6 assert bundle.is_dir() assert (bundle / "page-0001.png").exists() assert (bundle / "page-0002.png").exists() def test_image_eval_pack_references_include_render_layout(tmp_path: Path) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") summary = create_eval_pack( tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean"], templates=["table-ledger"], ) entry = load_manifest(Path(summary.manifest_path))[0] reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8")) layout = reference["metadata"]["layout"] assert layout["schema_version"] == 1 assert layout["pages"][0]["lines"][0]["text"] == "दैनिक हिसाब" assert layout["pages"][0]["table_cells"][0]["text"] == "नाम" assert len(layout["pages"][0]["table_cells"][0]["bbox"]) == 4 def test_complex_table_eval_pack_writes_span_reference(tmp_path: Path) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") summary = create_eval_pack( tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean"], templates=["complex-table-ledger"], ) entry = load_manifest(Path(summary.manifest_path))[0] reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8")) cells = reference["tables"][0]["cells"] assert "complex_table" in entry.metadata["slices"] assert cells[0] == {"row": 0, "col": 0, "text": "Ledger Summary", "rowspan": 1, "colspan": 2} assert any(cell["text"] == "नाम" for cell in cells) assert reference["metadata"]["layout"]["pages"][0]["table_cells"][0]["text"] == "Ledger Summary" def test_extract_crop_manifest_from_rendered_eval_pack(tmp_path: Path) -> None: if importlib.util.find_spec("PIL") is None: pytest.skip("Pillow not installed in local test environment") summary = create_eval_pack( tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean"], templates=["table-ledger", "figure-caption"], ) crop_summary = extract_crop_manifest( summary.manifest_path, tmp_path / "crops", crop_types=["line", "table_cell", "figure_caption"], ) entries = load_manifest(crop_summary.manifest_path) assert crop_summary.crop_type_counts["line"] >= 3 assert crop_summary.crop_type_counts["table_cell"] == 12 assert crop_summary.crop_type_counts["figure_caption"] == 1 assert all(Path(entry.image_path).exists() for entry in entries) assert all(entry.metadata["source_sample_id"] for entry in entries) assert any(entry.text == "नाम" and entry.metadata["crop_type"] == "table_cell" for entry in entries) assert any(entry.metadata["crop_type"] == "figure_caption" for entry in entries) assert (tmp_path / "crops" / "crop-manifest-summary.md").exists() def test_extract_crop_manifest_requires_layout_metadata(tmp_path: Path) -> None: summary = create_eval_pack( tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"], templates=["table-ledger"], ) with pytest.raises(DataValidationError, match="renderer layout metadata"): extract_crop_manifest(summary.manifest_path, tmp_path / "crops") def test_count_per_template_materializes_distinct_text_variants(tmp_path: Path) -> None: summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=2, degradations=["clean"]) entries = load_manifest(Path(summary.manifest_path)) mixed_entries = sorted((entry for entry in entries if "mixed-paragraph" in entry.sample_id), key=lambda entry: entry.sample_id) assert len(mixed_entries) == 2 assert mixed_entries[0].text != mixed_entries[1].text multi_page_entries = sorted((entry for entry in entries if "multi-page-brief" in entry.sample_id), key=lambda entry: entry.sample_id) assert len(multi_page_entries) == 2 assert multi_page_entries[0].text != multi_page_entries[1].text def test_create_eval_pack_can_select_templates_and_variant_offset(tmp_path: Path) -> None: summary = create_eval_pack( tmp_path / "pack", input_format="text", count_per_template=2, degradations=["clean"], templates=["table-ledger", "figure-caption"], variant_offset=10, ) entries = load_manifest(Path(summary.manifest_path)) assert summary.sample_count == 4 assert all(("table-ledger" in entry.sample_id or "figure-caption" in entry.sample_id) for entry in entries) assert any("010" in entry.sample_id for entry in entries) assert any("011" in entry.sample_id for entry in entries) table_entries = [entry for entry in entries if "table-ledger" in entry.sample_id] assert len({entry.text for entry in table_entries}) == 2 def test_eval_pack_feeds_benchmark_with_structured_references(tmp_path: Path) -> None: summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) manifest = Path(summary.manifest_path) entries = load_manifest(manifest) inputs = [entry.image_path for entry in entries] results = run_benchmark( inputs, tmp_path / "bench", baselines=["ours"], references_dir=tmp_path / "pack" / "references", eval_manifest=manifest, ) table_result = next(item for item in results if item.sample_id and "table-ledger" in item.sample_id) figure_result = next(item for item in results if item.sample_id and "figure-caption" in item.sample_id) multi_page_result = next(item for item in results if item.sample_id and "multi-page-brief" in item.sample_id) assert table_result.metrics["table_cell_f1"] == 1.0 assert figure_result.metrics["figure_detection_f1"] == 1.0 assert multi_page_result.metrics["table_cell_f1"] == 1.0 assert multi_page_result.metrics["figure_detection_f1"] == 1.0 assert "table" in table_result.slices assert "figure" in figure_result.slices assert "multi_page" in multi_page_result.slices assert (tmp_path / "bench" / "report.json").exists()