ocr-tech-k2 / tests /test_evalpack.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
8.73 kB
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import pytest
from ocrtech.benchmark import run_benchmark
from ocrtech.crops import extract_crop_manifest
from ocrtech.errors import DataValidationError
from ocrtech.evalpack import audit_eval_pack, create_eval_pack
from ocrtech.manifest import load_manifest
def test_create_text_eval_pack_and_audit(tmp_path: Path) -> None:
summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
manifest = Path(summary.manifest_path)
assert manifest.exists()
assert summary.sample_count == 6
assert not summary.claim_ready
assert "table" in summary.slices
assert "complex_table" in summary.slices
assert "multi_page" in summary.slices
entries = load_manifest(manifest)
assert all(Path(entry.image_path).exists() for entry in entries)
assert all(Path(str(entry.metadata["reference_path"])).exists() for entry in entries)
audit = audit_eval_pack(manifest, tmp_path / "audit")
assert audit.passed
assert not audit.claim_ready
assert (tmp_path / "audit" / "eval-pack-audit.json").exists()
def test_create_full_image_claim_pack_is_claim_ready_when_degradations_cover_required_slices(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean", "scan", "blur", "low_contrast", "skew"],
)
audit = audit_eval_pack(summary.manifest_path)
assert summary.claim_ready
assert audit.claim_ready
assert audit.passed
def test_create_image_eval_pack_writes_multi_page_bundle(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean"])
entries = load_manifest(Path(summary.manifest_path))
multi_page_entry = next(entry for entry in entries if "multi-page-brief" in entry.sample_id)
bundle = Path(multi_page_entry.image_path)
assert summary.sample_count == 6
assert bundle.is_dir()
assert (bundle / "page-0001.png").exists()
assert (bundle / "page-0002.png").exists()
def test_image_eval_pack_references_include_render_layout(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean"],
templates=["table-ledger"],
)
entry = load_manifest(Path(summary.manifest_path))[0]
reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8"))
layout = reference["metadata"]["layout"]
assert layout["schema_version"] == 1
assert layout["pages"][0]["lines"][0]["text"] == "दैनिक हिसाब"
assert layout["pages"][0]["table_cells"][0]["text"] == "नाम"
assert len(layout["pages"][0]["table_cells"][0]["bbox"]) == 4
def test_complex_table_eval_pack_writes_span_reference(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean"],
templates=["complex-table-ledger"],
)
entry = load_manifest(Path(summary.manifest_path))[0]
reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8"))
cells = reference["tables"][0]["cells"]
assert "complex_table" in entry.metadata["slices"]
assert cells[0] == {"row": 0, "col": 0, "text": "Ledger Summary", "rowspan": 1, "colspan": 2}
assert any(cell["text"] == "नाम" for cell in cells)
assert reference["metadata"]["layout"]["pages"][0]["table_cells"][0]["text"] == "Ledger Summary"
def test_extract_crop_manifest_from_rendered_eval_pack(tmp_path: Path) -> None:
if importlib.util.find_spec("PIL") is None:
pytest.skip("Pillow not installed in local test environment")
summary = create_eval_pack(
tmp_path / "pack",
input_format="image",
count_per_template=1,
degradations=["clean"],
templates=["table-ledger", "figure-caption"],
)
crop_summary = extract_crop_manifest(
summary.manifest_path,
tmp_path / "crops",
crop_types=["line", "table_cell", "figure_caption"],
)
entries = load_manifest(crop_summary.manifest_path)
assert crop_summary.crop_type_counts["line"] >= 3
assert crop_summary.crop_type_counts["table_cell"] == 12
assert crop_summary.crop_type_counts["figure_caption"] == 1
assert all(Path(entry.image_path).exists() for entry in entries)
assert all(entry.metadata["source_sample_id"] for entry in entries)
assert any(entry.text == "नाम" and entry.metadata["crop_type"] == "table_cell" for entry in entries)
assert any(entry.metadata["crop_type"] == "figure_caption" for entry in entries)
assert (tmp_path / "crops" / "crop-manifest-summary.md").exists()
def test_extract_crop_manifest_requires_layout_metadata(tmp_path: Path) -> None:
summary = create_eval_pack(
tmp_path / "pack",
input_format="text",
count_per_template=1,
degradations=["clean"],
templates=["table-ledger"],
)
with pytest.raises(DataValidationError, match="renderer layout metadata"):
extract_crop_manifest(summary.manifest_path, tmp_path / "crops")
def test_count_per_template_materializes_distinct_text_variants(tmp_path: Path) -> None:
summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=2, degradations=["clean"])
entries = load_manifest(Path(summary.manifest_path))
mixed_entries = sorted((entry for entry in entries if "mixed-paragraph" in entry.sample_id), key=lambda entry: entry.sample_id)
assert len(mixed_entries) == 2
assert mixed_entries[0].text != mixed_entries[1].text
multi_page_entries = sorted((entry for entry in entries if "multi-page-brief" in entry.sample_id), key=lambda entry: entry.sample_id)
assert len(multi_page_entries) == 2
assert multi_page_entries[0].text != multi_page_entries[1].text
def test_create_eval_pack_can_select_templates_and_variant_offset(tmp_path: Path) -> None:
summary = create_eval_pack(
tmp_path / "pack",
input_format="text",
count_per_template=2,
degradations=["clean"],
templates=["table-ledger", "figure-caption"],
variant_offset=10,
)
entries = load_manifest(Path(summary.manifest_path))
assert summary.sample_count == 4
assert all(("table-ledger" in entry.sample_id or "figure-caption" in entry.sample_id) for entry in entries)
assert any("010" in entry.sample_id for entry in entries)
assert any("011" in entry.sample_id for entry in entries)
table_entries = [entry for entry in entries if "table-ledger" in entry.sample_id]
assert len({entry.text for entry in table_entries}) == 2
def test_eval_pack_feeds_benchmark_with_structured_references(tmp_path: Path) -> None:
summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
manifest = Path(summary.manifest_path)
entries = load_manifest(manifest)
inputs = [entry.image_path for entry in entries]
results = run_benchmark(
inputs,
tmp_path / "bench",
baselines=["ours"],
references_dir=tmp_path / "pack" / "references",
eval_manifest=manifest,
)
table_result = next(item for item in results if item.sample_id and "table-ledger" in item.sample_id)
figure_result = next(item for item in results if item.sample_id and "figure-caption" in item.sample_id)
multi_page_result = next(item for item in results if item.sample_id and "multi-page-brief" in item.sample_id)
assert table_result.metrics["table_cell_f1"] == 1.0
assert figure_result.metrics["figure_detection_f1"] == 1.0
assert multi_page_result.metrics["table_cell_f1"] == 1.0
assert multi_page_result.metrics["figure_detection_f1"] == 1.0
assert "table" in table_result.slices
assert "figure" in figure_result.slices
assert "multi_page" in multi_page_result.slices
assert (tmp_path / "bench" / "report.json").exists()