ocr-tech-k2 / tests /test_public_benchmarks.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
7.54 kB
from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech.errors import DataValidationError
from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest
from ocrtech.public_benchmarks import audit_public_benchmark_manifest, list_public_benchmarks, prepare_public_benchmark
def test_list_public_benchmarks_includes_document_parsing_targets() -> None:
datasets = {spec.dataset for spec in list_public_benchmarks()}
assert "opendatalab/OmniDocBench" in datasets
assert "allenai/olmOCR-bench" in datasets
assert "PaddlePaddle/Real5-OmniDocBench" in datasets
def test_audit_public_benchmark_manifest_passes_with_provenance_and_references(tmp_path: Path) -> None:
image = tmp_path / "page.png"
image.write_bytes(b"image")
reference = tmp_path / "page.ref.json"
reference.write_text('{"text":"hello"}\n', encoding="utf-8")
manifest = tmp_path / "public.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="omni-1",
dataset="opendatalab/OmniDocBench",
split="eval",
image_path=str(image),
text="hello",
sha256=sha256_file(image),
metadata={
"benchmark_name": "opendatalab/OmniDocBench",
"public_benchmark": True,
"source_url": "https://huggingface.co/datasets/opendatalab/OmniDocBench",
"reference_path": str(reference),
"slices": ["public", "document_parsing", "table"],
"text_sha256": sha256_text("hello"),
},
)
],
manifest,
)
audit = audit_public_benchmark_manifest(manifest, tmp_path / "audit", min_samples=1)
assert audit.passed
assert audit.benchmark == "opendatalab/OmniDocBench"
assert audit.slice_counts["public"] == 1
assert (tmp_path / "audit" / "public-benchmark-audit.json").exists()
def test_audit_public_benchmark_manifest_fails_without_required_metadata(tmp_path: Path) -> None:
image = tmp_path / "page.png"
image.write_bytes(b"image")
manifest = tmp_path / "public.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="omni-1",
dataset="opendatalab/OmniDocBench",
split="eval",
image_path=str(image),
text="hello",
sha256=sha256_file(image),
metadata={"slices": ["document_parsing"]},
)
],
manifest,
)
audit = audit_public_benchmark_manifest(manifest, tmp_path / "audit", min_samples=1)
assert not audit.passed
assert any("metadata.public_benchmark" in issue for issue in audit.issues)
assert any("metadata.reference_path" in issue for issue in audit.issues)
assert any("required public benchmark slice missing: public" in issue for issue in audit.issues)
def test_prepare_public_benchmark_converts_verified_omnidocbench_schema(tmp_path: Path) -> None:
root = tmp_path / "source"
images = root / "images"
images.mkdir(parents=True)
image = images / "page-1.png"
image.write_bytes(b"fake-png")
annotation = root / "OmniDocBench.json"
annotation.write_text(
json.dumps(
[
{
"page_info": {
"image_path": "page-1.png",
"page_no": 0,
"height": 640,
"width": 480,
"page_attribute": {
"language": "english",
"subset": "table_hard",
"data_source": "research_report",
},
},
"layout_dets": [
{
"category_type": "title",
"text": "Quarterly Result",
"ignore": False,
"order": 1,
},
{
"category_type": "table",
"html": "<table><tr><th colspan=\"2\">Revenue</th></tr><tr><td>Nepal</td><td>42</td></tr></table>",
"ignore": False,
"order": 2,
"anno_id": "tbl-1",
},
],
}
]
),
encoding="utf-8",
)
report = prepare_public_benchmark(
"opendatalab/OmniDocBench",
tmp_path / "prepared",
limit=1,
language="english",
require_tables=True,
annotation_path=annotation,
local_image_root=root,
)
manifest_path = Path(report.manifest_path)
entries = load_manifest(manifest_path)
assert report.sample_count == 1
assert len(entries) == 1
assert entries[0].metadata["public_benchmark"] is True
assert "public" in entries[0].metadata["slices"]
assert "table" in entries[0].metadata["slices"]
reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8"))
assert reference["text"] == "Quarterly Result\nRevenue\nNepal 42"
assert reference["tables"][0]["cells"][0]["text"] == "Revenue"
assert reference["tables"][0]["cells"][0]["colspan"] == 2
audit = audit_public_benchmark_manifest(manifest_path, tmp_path / "audit", min_samples=1)
assert audit.passed
def test_prepare_public_benchmark_filters_omnidocbench_rows(tmp_path: Path) -> None:
root = tmp_path / "source"
images = root / "images"
images.mkdir(parents=True)
(images / "english.png").write_bytes(b"english")
(images / "chinese.png").write_bytes(b"chinese")
annotation = root / "OmniDocBench.json"
rows = []
for image_name, language in [("english.png", "english"), ("chinese.png", "simplified_chinese")]:
rows.append(
{
"page_info": {
"image_path": image_name,
"page_no": 0,
"height": 100,
"width": 100,
"page_attribute": {
"language": language,
"subset": "v1.5",
"data_source": "book",
},
},
"layout_dets": [{"category_type": "text_block", "text": language, "ignore": False, "order": 1}],
}
)
annotation.write_text(json.dumps(rows), encoding="utf-8")
report = prepare_public_benchmark(
"opendatalab/OmniDocBench",
tmp_path / "prepared",
language="english",
annotation_path=annotation,
local_image_root=root,
)
entries = load_manifest(report.manifest_path)
assert [entry.text for entry in entries] == ["english"]
def test_prepare_public_benchmark_fails_on_missing_omnidocbench_schema(tmp_path: Path) -> None:
root = tmp_path / "source"
root.mkdir()
annotation = root / "OmniDocBench.json"
annotation.write_text(json.dumps([{"page_info": {}}]), encoding="utf-8")
with pytest.raises(DataValidationError, match="page_info.page_attribute"):
prepare_public_benchmark(
"opendatalab/OmniDocBench",
tmp_path / "prepared",
annotation_path=annotation,
local_image_root=root,
)