File size: 8,732 Bytes
1ef1de7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from ocrtech.benchmark import run_benchmark
from ocrtech.crops import extract_crop_manifest
from ocrtech.errors import DataValidationError
from ocrtech.evalpack import audit_eval_pack, create_eval_pack
from ocrtech.manifest import load_manifest
def test_create_text_eval_pack_and_audit(tmp_path: Path) -> None:
summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
manifest = Path(summary.manifest_path)
assert manifest.exists()
assert summary.sample_count == 6
assert not summary.claim_ready
assert "table" in summary.slices
assert "complex_table" in summary.slices
assert "multi_page" in summary.slices
entries = load_manifest(manifest)
assert all(Path(entry.image_path).exists() for entry in entries)
assert all(Path(str(entry.metadata["reference_path"])).exists() for entry in entries)
audit = audit_eval_pack(manifest, tmp_path / "audit")
assert audit.passed
assert not audit.claim_ready
assert (tmp_path / "audit" / "eval-pack-audit.json").exists()
def test_create_full_image_claim_pack_is_claim_ready_when_degradations_cover_required_slices(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean", "scan", "blur", "low_contrast", "skew"],
)
audit = audit_eval_pack(summary.manifest_path)
assert summary.claim_ready
assert audit.claim_ready
assert audit.passed
def test_create_image_eval_pack_writes_multi_page_bundle(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean"])
entries = load_manifest(Path(summary.manifest_path))
multi_page_entry = next(entry for entry in entries if "multi-page-brief" in entry.sample_id)
bundle = Path(multi_page_entry.image_path)
assert summary.sample_count == 6
assert bundle.is_dir()
assert (bundle / "page-0001.png").exists()
assert (bundle / "page-0002.png").exists()
def test_image_eval_pack_references_include_render_layout(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean"],
templates=["table-ledger"],
)
entry = load_manifest(Path(summary.manifest_path))[0]
reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8"))
layout = reference["metadata"]["layout"]
assert layout["schema_version"] == 1
assert layout["pages"][0]["lines"][0]["text"] == "दैनिक हिसाब"
assert layout["pages"][0]["table_cells"][0]["text"] == "नाम"
assert len(layout["pages"][0]["table_cells"][0]["bbox"]) == 4
def test_complex_table_eval_pack_writes_span_reference(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean"],
templates=["complex-table-ledger"],
)
entry = load_manifest(Path(summary.manifest_path))[0]
reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8"))
cells = reference["tables"][0]["cells"]
assert "complex_table" in entry.metadata["slices"]
assert cells[0] == {"row": 0, "col": 0, "text": "Ledger Summary", "rowspan": 1, "colspan": 2}
assert any(cell["text"] == "नाम" for cell in cells)
assert reference["metadata"]["layout"]["pages"][0]["table_cells"][0]["text"] == "Ledger Summary"
def test_extract_crop_manifest_from_rendered_eval_pack(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean"],
templates=["table-ledger", "figure-caption"],
)
crop_summary = extract_crop_manifest(
summary.manifest_path,
tmp_path / "crops",
crop_types=["line", "table_cell", "figure_caption"],
)
entries = load_manifest(crop_summary.manifest_path)
assert crop_summary.crop_type_counts["line"] >= 3
assert crop_summary.crop_type_counts["table_cell"] == 12
assert crop_summary.crop_type_counts["figure_caption"] == 1
assert all(Path(entry.image_path).exists() for entry in entries)
assert all(entry.metadata["source_sample_id"] for entry in entries)
assert any(entry.text == "नाम" and entry.metadata["crop_type"] == "table_cell" for entry in entries)
assert any(entry.metadata["crop_type"] == "figure_caption" for entry in entries)
assert (tmp_path / "crops" / "crop-manifest-summary.md").exists()
def test_extract_crop_manifest_requires_layout_metadata(tmp_path: Path) -> None:
summary = create_eval_pack(
tmp_path / "pack",
input_format="text",
count_per_template=1,
degradations=["clean"],
templates=["table-ledger"],
)
with pytest.raises(DataValidationError, match="renderer layout metadata"):
extract_crop_manifest(summary.manifest_path, tmp_path / "crops")
def test_count_per_template_materializes_distinct_text_variants(tmp_path: Path) -> None:
summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=2, degradations=["clean"])
entries = load_manifest(Path(summary.manifest_path))
mixed_entries = sorted((entry for entry in entries if "mixed-paragraph" in entry.sample_id), key=lambda entry: entry.sample_id)
assert len(mixed_entries) == 2
assert mixed_entries[0].text != mixed_entries[1].text
multi_page_entries = sorted((entry for entry in entries if "multi-page-brief" in entry.sample_id), key=lambda entry: entry.sample_id)
assert len(multi_page_entries) == 2
assert multi_page_entries[0].text != multi_page_entries[1].text
def test_create_eval_pack_can_select_templates_and_variant_offset(tmp_path: Path) -> None:
summary = create_eval_pack(
tmp_path / "pack",
input_format="text",
count_per_template=2,
degradations=["clean"],
templates=["table-ledger", "figure-caption"],
variant_offset=10,
)
entries = load_manifest(Path(summary.manifest_path))
assert summary.sample_count == 4
assert all(("table-ledger" in entry.sample_id or "figure-caption" in entry.sample_id) for entry in entries)
assert any("010" in entry.sample_id for entry in entries)
assert any("011" in entry.sample_id for entry in entries)
table_entries = [entry for entry in entries if "table-ledger" in entry.sample_id]
assert len({entry.text for entry in table_entries}) == 2
def test_eval_pack_feeds_benchmark_with_structured_references(tmp_path: Path) -> None:
summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
manifest = Path(summary.manifest_path)
entries = load_manifest(manifest)
inputs = [entry.image_path for entry in entries]
results = run_benchmark(
inputs,
tmp_path / "bench",
baselines=["ours"],
references_dir=tmp_path / "pack" / "references",
eval_manifest=manifest,
)
table_result = next(item for item in results if item.sample_id and "table-ledger" in item.sample_id)
figure_result = next(item for item in results if item.sample_id and "figure-caption" in item.sample_id)
multi_page_result = next(item for item in results if item.sample_id and "multi-page-brief" in item.sample_id)
assert table_result.metrics["table_cell_f1"] == 1.0
assert figure_result.metrics["figure_detection_f1"] == 1.0
assert multi_page_result.metrics["table_cell_f1"] == 1.0
assert multi_page_result.metrics["figure_detection_f1"] == 1.0
assert "table" in table_result.slices
assert "figure" in figure_result.slices
assert "multi_page" in multi_page_result.slices
assert (tmp_path / "bench" / "report.json").exists()
|