from __future__ import annotations import json from pathlib import Path import pytest from ocrtech.errors import DataValidationError from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest from ocrtech.public_benchmarks import audit_public_benchmark_manifest, list_public_benchmarks, prepare_public_benchmark def test_list_public_benchmarks_includes_document_parsing_targets() -> None: datasets = {spec.dataset for spec in list_public_benchmarks()} assert "opendatalab/OmniDocBench" in datasets assert "allenai/olmOCR-bench" in datasets assert "PaddlePaddle/Real5-OmniDocBench" in datasets def test_audit_public_benchmark_manifest_passes_with_provenance_and_references(tmp_path: Path) -> None: image = tmp_path / "page.png" image.write_bytes(b"image") reference = tmp_path / "page.ref.json" reference.write_text('{"text":"hello"}\n', encoding="utf-8") manifest = tmp_path / "public.jsonl" write_manifest( [ ManifestEntry( sample_id="omni-1", dataset="opendatalab/OmniDocBench", split="eval", image_path=str(image), text="hello", sha256=sha256_file(image), metadata={ "benchmark_name": "opendatalab/OmniDocBench", "public_benchmark": True, "source_url": "https://huggingface.co/datasets/opendatalab/OmniDocBench", "reference_path": str(reference), "slices": ["public", "document_parsing", "table"], "text_sha256": sha256_text("hello"), }, ) ], manifest, ) audit = audit_public_benchmark_manifest(manifest, tmp_path / "audit", min_samples=1) assert audit.passed assert audit.benchmark == "opendatalab/OmniDocBench" assert audit.slice_counts["public"] == 1 assert (tmp_path / "audit" / "public-benchmark-audit.json").exists() def test_audit_public_benchmark_manifest_fails_without_required_metadata(tmp_path: Path) -> None: image = tmp_path / "page.png" image.write_bytes(b"image") manifest = tmp_path / "public.jsonl" write_manifest( [ ManifestEntry( sample_id="omni-1", dataset="opendatalab/OmniDocBench", split="eval", image_path=str(image), text="hello", sha256=sha256_file(image), metadata={"slices": ["document_parsing"]}, ) ], manifest, ) audit = audit_public_benchmark_manifest(manifest, tmp_path / "audit", min_samples=1) assert not audit.passed assert any("metadata.public_benchmark" in issue for issue in audit.issues) assert any("metadata.reference_path" in issue for issue in audit.issues) assert any("required public benchmark slice missing: public" in issue for issue in audit.issues) def test_prepare_public_benchmark_converts_verified_omnidocbench_schema(tmp_path: Path) -> None: root = tmp_path / "source" images = root / "images" images.mkdir(parents=True) image = images / "page-1.png" image.write_bytes(b"fake-png") annotation = root / "OmniDocBench.json" annotation.write_text( json.dumps( [ { "page_info": { "image_path": "page-1.png", "page_no": 0, "height": 640, "width": 480, "page_attribute": { "language": "english", "subset": "table_hard", "data_source": "research_report", }, }, "layout_dets": [ { "category_type": "title", "text": "Quarterly Result", "ignore": False, "order": 1, }, { "category_type": "table", "html": "
Revenue
Nepal42
", "ignore": False, "order": 2, "anno_id": "tbl-1", }, ], } ] ), encoding="utf-8", ) report = prepare_public_benchmark( "opendatalab/OmniDocBench", tmp_path / "prepared", limit=1, language="english", require_tables=True, annotation_path=annotation, local_image_root=root, ) manifest_path = Path(report.manifest_path) entries = load_manifest(manifest_path) assert report.sample_count == 1 assert len(entries) == 1 assert entries[0].metadata["public_benchmark"] is True assert "public" in entries[0].metadata["slices"] assert "table" in entries[0].metadata["slices"] reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8")) assert reference["text"] == "Quarterly Result\nRevenue\nNepal 42" assert reference["tables"][0]["cells"][0]["text"] == "Revenue" assert reference["tables"][0]["cells"][0]["colspan"] == 2 audit = audit_public_benchmark_manifest(manifest_path, tmp_path / "audit", min_samples=1) assert audit.passed def test_prepare_public_benchmark_filters_omnidocbench_rows(tmp_path: Path) -> None: root = tmp_path / "source" images = root / "images" images.mkdir(parents=True) (images / "english.png").write_bytes(b"english") (images / "chinese.png").write_bytes(b"chinese") annotation = root / "OmniDocBench.json" rows = [] for image_name, language in [("english.png", "english"), ("chinese.png", "simplified_chinese")]: rows.append( { "page_info": { "image_path": image_name, "page_no": 0, "height": 100, "width": 100, "page_attribute": { "language": language, "subset": "v1.5", "data_source": "book", }, }, "layout_dets": [{"category_type": "text_block", "text": language, "ignore": False, "order": 1}], } ) annotation.write_text(json.dumps(rows), encoding="utf-8") report = prepare_public_benchmark( "opendatalab/OmniDocBench", tmp_path / "prepared", language="english", annotation_path=annotation, local_image_root=root, ) entries = load_manifest(report.manifest_path) assert [entry.text for entry in entries] == ["english"] def test_prepare_public_benchmark_fails_on_missing_omnidocbench_schema(tmp_path: Path) -> None: root = tmp_path / "source" root.mkdir() annotation = root / "OmniDocBench.json" annotation.write_text(json.dumps([{"page_info": {}}]), encoding="utf-8") with pytest.raises(DataValidationError, match="page_info.page_attribute"): prepare_public_benchmark( "opendatalab/OmniDocBench", tmp_path / "prepared", annotation_path=annotation, local_image_root=root, )