Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_evalpack.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 8.73 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_evalpack.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_evalpack.py
-
curl -L -o test_evalpack.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_evalpack.py
8.73 kB
| from __future__ import annotations | |
| import importlib.util | |
| import json | |
| from pathlib import Path | |
| import pytest | |
| from ocrtech.benchmark import run_benchmark | |
| from ocrtech.crops import extract_crop_manifest | |
| from ocrtech.errors import DataValidationError | |
| from ocrtech.evalpack import audit_eval_pack, create_eval_pack | |
| from ocrtech.manifest import load_manifest | |
| def test_create_text_eval_pack_and_audit(tmp_path: Path) -> None: | |
| summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| manifest = Path(summary.manifest_path) | |
| assert manifest.exists() | |
| assert summary.sample_count == 6 | |
| assert not summary.claim_ready | |
| assert "table" in summary.slices | |
| assert "complex_table" in summary.slices | |
| assert "multi_page" in summary.slices | |
| entries = load_manifest(manifest) | |
| assert all(Path(entry.image_path).exists() for entry in entries) | |
| assert all(Path(str(entry.metadata["reference_path"])).exists() for entry in entries) | |
| audit = audit_eval_pack(manifest, tmp_path / "audit") | |
| assert audit.passed | |
| assert not audit.claim_ready | |
| assert (tmp_path / "audit" / "eval-pack-audit.json").exists() | |
| def test_create_full_image_claim_pack_is_claim_ready_when_degradations_cover_required_slices(tmp_path: Path) -> None: | |
| if importlib.util.find_spec("PIL") is None: | |
| pytest.skip("Pillow not installed in local test environment") | |
| summary = create_eval_pack( | |
| tmp_path / "pack", | |
| input_format="image", | |
| count_per_template=1, | |
| degradations=["clean", "scan", "blur", "low_contrast", "skew"], | |
| ) | |
| audit = audit_eval_pack(summary.manifest_path) | |
| assert summary.claim_ready | |
| assert audit.claim_ready | |
| assert audit.passed | |
| def test_create_image_eval_pack_writes_multi_page_bundle(tmp_path: Path) -> None: | |
| if importlib.util.find_spec("PIL") is None: | |
| pytest.skip("Pillow not installed in local test environment") | |
| summary = create_eval_pack(tmp_path / "pack", input_format="image", count_per_template=1, degradations=["clean"]) | |
| entries = load_manifest(Path(summary.manifest_path)) | |
| multi_page_entry = next(entry for entry in entries if "multi-page-brief" in entry.sample_id) | |
| bundle = Path(multi_page_entry.image_path) | |
| assert summary.sample_count == 6 | |
| assert bundle.is_dir() | |
| assert (bundle / "page-0001.png").exists() | |
| assert (bundle / "page-0002.png").exists() | |
| def test_image_eval_pack_references_include_render_layout(tmp_path: Path) -> None: | |
| if importlib.util.find_spec("PIL") is None: | |
| pytest.skip("Pillow not installed in local test environment") | |
| summary = create_eval_pack( | |
| tmp_path / "pack", | |
| input_format="image", | |
| count_per_template=1, | |
| degradations=["clean"], | |
| templates=["table-ledger"], | |
| ) | |
| entry = load_manifest(Path(summary.manifest_path))[0] | |
| reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8")) | |
| layout = reference["metadata"]["layout"] | |
| assert layout["schema_version"] == 1 | |
| assert layout["pages"][0]["lines"][0]["text"] == "दैनिक हिसाब" | |
| assert layout["pages"][0]["table_cells"][0]["text"] == "नाम" | |
| assert len(layout["pages"][0]["table_cells"][0]["bbox"]) == 4 | |
| def test_complex_table_eval_pack_writes_span_reference(tmp_path: Path) -> None: | |
| if importlib.util.find_spec("PIL") is None: | |
| pytest.skip("Pillow not installed in local test environment") | |
| summary = create_eval_pack( | |
| tmp_path / "pack", | |
| input_format="image", | |
| count_per_template=1, | |
| degradations=["clean"], | |
| templates=["complex-table-ledger"], | |
| ) | |
| entry = load_manifest(Path(summary.manifest_path))[0] | |
| reference = json.loads(Path(str(entry.metadata["reference_path"])).read_text(encoding="utf-8")) | |
| cells = reference["tables"][0]["cells"] | |
| assert "complex_table" in entry.metadata["slices"] | |
| assert cells[0] == {"row": 0, "col": 0, "text": "Ledger Summary", "rowspan": 1, "colspan": 2} | |
| assert any(cell["text"] == "नाम" for cell in cells) | |
| assert reference["metadata"]["layout"]["pages"][0]["table_cells"][0]["text"] == "Ledger Summary" | |
| def test_extract_crop_manifest_from_rendered_eval_pack(tmp_path: Path) -> None: | |
| if importlib.util.find_spec("PIL") is None: | |
| pytest.skip("Pillow not installed in local test environment") | |
| summary = create_eval_pack( | |
| tmp_path / "pack", | |
| input_format="image", | |
| count_per_template=1, | |
| degradations=["clean"], | |
| templates=["table-ledger", "figure-caption"], | |
| ) | |
| crop_summary = extract_crop_manifest( | |
| summary.manifest_path, | |
| tmp_path / "crops", | |
| crop_types=["line", "table_cell", "figure_caption"], | |
| ) | |
| entries = load_manifest(crop_summary.manifest_path) | |
| assert crop_summary.crop_type_counts["line"] >= 3 | |
| assert crop_summary.crop_type_counts["table_cell"] == 12 | |
| assert crop_summary.crop_type_counts["figure_caption"] == 1 | |
| assert all(Path(entry.image_path).exists() for entry in entries) | |
| assert all(entry.metadata["source_sample_id"] for entry in entries) | |
| assert any(entry.text == "नाम" and entry.metadata["crop_type"] == "table_cell" for entry in entries) | |
| assert any(entry.metadata["crop_type"] == "figure_caption" for entry in entries) | |
| assert (tmp_path / "crops" / "crop-manifest-summary.md").exists() | |
| def test_extract_crop_manifest_requires_layout_metadata(tmp_path: Path) -> None: | |
| summary = create_eval_pack( | |
| tmp_path / "pack", | |
| input_format="text", | |
| count_per_template=1, | |
| degradations=["clean"], | |
| templates=["table-ledger"], | |
| ) | |
| with pytest.raises(DataValidationError, match="renderer layout metadata"): | |
| extract_crop_manifest(summary.manifest_path, tmp_path / "crops") | |
| def test_count_per_template_materializes_distinct_text_variants(tmp_path: Path) -> None: | |
| summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=2, degradations=["clean"]) | |
| entries = load_manifest(Path(summary.manifest_path)) | |
| mixed_entries = sorted((entry for entry in entries if "mixed-paragraph" in entry.sample_id), key=lambda entry: entry.sample_id) | |
| assert len(mixed_entries) == 2 | |
| assert mixed_entries[0].text != mixed_entries[1].text | |
| multi_page_entries = sorted((entry for entry in entries if "multi-page-brief" in entry.sample_id), key=lambda entry: entry.sample_id) | |
| assert len(multi_page_entries) == 2 | |
| assert multi_page_entries[0].text != multi_page_entries[1].text | |
| def test_create_eval_pack_can_select_templates_and_variant_offset(tmp_path: Path) -> None: | |
| summary = create_eval_pack( | |
| tmp_path / "pack", | |
| input_format="text", | |
| count_per_template=2, | |
| degradations=["clean"], | |
| templates=["table-ledger", "figure-caption"], | |
| variant_offset=10, | |
| ) | |
| entries = load_manifest(Path(summary.manifest_path)) | |
| assert summary.sample_count == 4 | |
| assert all(("table-ledger" in entry.sample_id or "figure-caption" in entry.sample_id) for entry in entries) | |
| assert any("010" in entry.sample_id for entry in entries) | |
| assert any("011" in entry.sample_id for entry in entries) | |
| table_entries = [entry for entry in entries if "table-ledger" in entry.sample_id] | |
| assert len({entry.text for entry in table_entries}) == 2 | |
| def test_eval_pack_feeds_benchmark_with_structured_references(tmp_path: Path) -> None: | |
| summary = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| manifest = Path(summary.manifest_path) | |
| entries = load_manifest(manifest) | |
| inputs = [entry.image_path for entry in entries] | |
| results = run_benchmark( | |
| inputs, | |
| tmp_path / "bench", | |
| baselines=["ours"], | |
| references_dir=tmp_path / "pack" / "references", | |
| eval_manifest=manifest, | |
| ) | |
| table_result = next(item for item in results if item.sample_id and "table-ledger" in item.sample_id) | |
| figure_result = next(item for item in results if item.sample_id and "figure-caption" in item.sample_id) | |
| multi_page_result = next(item for item in results if item.sample_id and "multi-page-brief" in item.sample_id) | |
| assert table_result.metrics["table_cell_f1"] == 1.0 | |
| assert figure_result.metrics["figure_detection_f1"] == 1.0 | |
| assert multi_page_result.metrics["table_cell_f1"] == 1.0 | |
| assert multi_page_result.metrics["figure_detection_f1"] == 1.0 | |
| assert "table" in table_result.slices | |
| assert "figure" in figure_result.slices | |
| assert "multi_page" in multi_page_result.slices | |
| assert (tmp_path / "bench" / "report.json").exists() | |