Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_public_benchmarks.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 7.54 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_public_benchmarks.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_public_benchmarks.py
-
curl -L -o test_public_benchmarks.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_public_benchmarks.py
7.54 kB
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import pytest | |
| from ocrtech.errors import DataValidationError | |
| from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest | |
| from ocrtech.public_benchmarks import audit_public_benchmark_manifest, list_public_benchmarks, prepare_public_benchmark | |
| def test_list_public_benchmarks_includes_document_parsing_targets() -> None: | |
| datasets = {spec.dataset for spec in list_public_benchmarks()} | |
| assert "opendatalab/OmniDocBench" in datasets | |
| assert "allenai/olmOCR-bench" in datasets | |
| assert "PaddlePaddle/Real5-OmniDocBench" in datasets | |
| def test_audit_public_benchmark_manifest_passes_with_provenance_and_references(tmp_path: Path) -> None: | |
| image = tmp_path / "page.png" | |
| image.write_bytes(b"image") | |
| reference = tmp_path / "page.ref.json" | |
| reference.write_text('{"text":"hello"}\n', encoding="utf-8") | |
| manifest = tmp_path / "public.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="omni-1", | |
| dataset="opendatalab/OmniDocBench", | |
| split="eval", | |
| image_path=str(image), | |
| text="hello", | |
| sha256=sha256_file(image), | |
| metadata={ | |
| "benchmark_name": "opendatalab/OmniDocBench", | |
| "public_benchmark": True, | |
| "source_url": "https://huggingface.co/datasets/opendatalab/OmniDocBench", | |
| "reference_path": str(reference), | |
| "slices": ["public", "document_parsing", "table"], | |
| "text_sha256": sha256_text("hello"), | |
| }, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| audit = audit_public_benchmark_manifest(manifest, tmp_path / "audit", min_samples=1) | |
| assert audit.passed | |
| assert audit.benchmark == "opendatalab/OmniDocBench" | |
| assert audit.slice_counts["public"] == 1 | |
| assert (tmp_path / "audit" / "public-benchmark-audit.json").exists() | |
| def test_audit_public_benchmark_manifest_fails_without_required_metadata(tmp_path: Path) -> None: | |
| image = tmp_path / "page.png" | |
| image.write_bytes(b"image") | |
| manifest = tmp_path / "public.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="omni-1", | |
| dataset="opendatalab/OmniDocBench", | |
| split="eval", | |
| image_path=str(image), | |
| text="hello", | |
| sha256=sha256_file(image), | |
| metadata={"slices": ["document_parsing"]}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| audit = audit_public_benchmark_manifest(manifest, tmp_path / "audit", min_samples=1) | |
| assert not audit.passed | |
| assert any("metadata.public_benchmark" in issue for issue in audit.issues) | |
| assert any("metadata.reference_path" in issue for issue in audit.issues) | |
| assert any("required public benchmark slice missing: public" in issue for issue in audit.issues) | |
| def test_prepare_public_benchmark_converts_verified_omnidocbench_schema(tmp_path: Path) -> None: | |
| root = tmp_path / "source" | |
| images = root / "images" | |
| images.mkdir(parents=True) | |
| image = images / "page-1.png" | |
| image.write_bytes(b"fake-png") | |
| annotation = root / "OmniDocBench.json" | |
| annotation.write_text( | |
| json.dumps( | |
| [ | |
| { | |
| "page_info": { | |
| "image_path": "page-1.png", | |
| "page_no": 0, | |
| "height": 640, | |
| "width": 480, | |
| "page_attribute": { | |
| "language": "english", | |
| "subset": "table_hard", | |
| "data_source": "research_report", | |
| }, | |
| }, | |
| "layout_dets": [ | |
| { | |
| "category_type": "title", | |
| "text": "Quarterly Result", | |
| "ignore": False, | |
| "order": 1, | |
| }, | |
| { | |
| "category_type": "table", | |
| "html": "<table><tr><th colspan=\"2\">Revenue</th></tr><tr><td>Nepal</td><td>42</td></tr></table>", | |
| "ignore": False, | |
| "order": 2, | |
| "anno_id": "tbl-1", | |
| }, | |
| ], | |
| } | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| report = prepare_public_benchmark( | |
| "opendatalab/OmniDocBench", | |
| tmp_path / "prepared", | |
| limit=1, | |
| language="english", | |
| require_tables=True, | |
| annotation_path=annotation, | |
| local_image_root=root, | |
| ) | |
| manifest_path = Path(report.manifest_path) | |
| entries = load_manifest(manifest_path) | |
| assert report.sample_count == 1 | |
| assert len(entries) == 1 | |
| assert entries[0].metadata["public_benchmark"] is True | |
| assert "public" in entries[0].metadata["slices"] | |
| assert "table" in entries[0].metadata["slices"] | |
| reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8")) | |
| assert reference["text"] == "Quarterly Result\nRevenue\nNepal 42" | |
| assert reference["tables"][0]["cells"][0]["text"] == "Revenue" | |
| assert reference["tables"][0]["cells"][0]["colspan"] == 2 | |
| audit = audit_public_benchmark_manifest(manifest_path, tmp_path / "audit", min_samples=1) | |
| assert audit.passed | |
| def test_prepare_public_benchmark_filters_omnidocbench_rows(tmp_path: Path) -> None: | |
| root = tmp_path / "source" | |
| images = root / "images" | |
| images.mkdir(parents=True) | |
| (images / "english.png").write_bytes(b"english") | |
| (images / "chinese.png").write_bytes(b"chinese") | |
| annotation = root / "OmniDocBench.json" | |
| rows = [] | |
| for image_name, language in [("english.png", "english"), ("chinese.png", "simplified_chinese")]: | |
| rows.append( | |
| { | |
| "page_info": { | |
| "image_path": image_name, | |
| "page_no": 0, | |
| "height": 100, | |
| "width": 100, | |
| "page_attribute": { | |
| "language": language, | |
| "subset": "v1.5", | |
| "data_source": "book", | |
| }, | |
| }, | |
| "layout_dets": [{"category_type": "text_block", "text": language, "ignore": False, "order": 1}], | |
| } | |
| ) | |
| annotation.write_text(json.dumps(rows), encoding="utf-8") | |
| report = prepare_public_benchmark( | |
| "opendatalab/OmniDocBench", | |
| tmp_path / "prepared", | |
| language="english", | |
| annotation_path=annotation, | |
| local_image_root=root, | |
| ) | |
| entries = load_manifest(report.manifest_path) | |
| assert [entry.text for entry in entries] == ["english"] | |
| def test_prepare_public_benchmark_fails_on_missing_omnidocbench_schema(tmp_path: Path) -> None: | |
| root = tmp_path / "source" | |
| root.mkdir() | |
| annotation = root / "OmniDocBench.json" | |
| annotation.write_text(json.dumps([{"page_info": {}}]), encoding="utf-8") | |
| with pytest.raises(DataValidationError, match="page_info.page_attribute"): | |
| prepare_public_benchmark( | |
| "opendatalab/OmniDocBench", | |
| tmp_path / "prepared", | |
| annotation_path=annotation, | |
| local_image_root=root, | |
| ) | |