ocr-tech-k2 / tests /test_manifest.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
12 kB
from __future__ import annotations
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from ocrtech.errors import DataValidationError
from ocrtech.manifest import (
ManifestEntry,
inspect_hf_dataset,
load_manifest,
normalize_manifest_images,
prepare_hf_correction_pairs,
prepare_hf_dataset,
prepare_local_dataset,
read_jsonl,
sha256_file,
write_manifest,
)
def test_prepare_local_dataset_verifies_fields_and_images(tmp_path: Path) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"fake image bytes")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8")
report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train")
entries = load_manifest(report.manifest_path)
assert len(entries) == 1
assert entries[0].sample_id == "local-train-00000000"
assert entries[0].text == "नेपाल"
assert entries[0].sha256
assert report.rejected_count == 0
def test_prepare_local_dataset_fails_on_missing_image(tmp_path: Path) -> None:
labels = tmp_path / "labels.jsonl"
labels.write_text(json.dumps({"image_path": "missing.png", "text": "नेपाल"}, ensure_ascii=False) + "\n", encoding="utf-8")
with pytest.raises(DataValidationError, match="image does not exist"):
prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train")
def test_prepare_local_dataset_fails_on_missing_text_field(tmp_path: Path) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"fake image bytes")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,caption\nline.png,नेपाल\n", encoding="utf-8")
with pytest.raises(DataValidationError, match="Could not infer text field"):
prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train")
def test_prepare_local_dataset_fails_on_empty_text_value(tmp_path: Path) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"fake image bytes")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline.png,\n", encoding="utf-8")
with pytest.raises(DataValidationError, match="text field 'text' is empty"):
prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train")
def test_prepare_hf_prashant_dataset_downloads_only_needed_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
image = tmp_path / "line_00001.png"
image.write_bytes(b"fake image bytes")
metadata = tmp_path / "metadata.jsonl"
metadata.write_text(
'{"file_name":"line_00001.png","text":"नेपाल सरकार"}\n{"file_name":"line_99999.png","text":"unused"}\n',
encoding="utf-8",
)
requested_files: list[str] = []
monkeypatch.setattr(
"ocrtech.manifest._list_dataset_files",
lambda *args, **kwargs: ["train/metadata.jsonl", "train/line_00001.png", "train/line_99999.png"],
)
def fake_download(dataset_id: str, dataset_file: str) -> Path:
requested_files.append(dataset_file)
if dataset_file == "train/metadata.jsonl":
return metadata
if dataset_file == "train/line_00001.png":
return image
raise AssertionError(f"unexpected dataset download: {dataset_file}")
monkeypatch.setattr("ocrtech.manifest._download_dataset_file", fake_download)
report = prepare_hf_dataset(
"prashant0919/nepali-synthetic-ocr-lines",
tmp_path / "manifests",
split="train",
limit=1,
)
entries = load_manifest(report.manifest_path)
assert len(entries) == 1
assert entries[0].text == "नेपाल सरकार"
assert Path(entries[0].image_path) == image
assert requested_files == ["train/metadata.jsonl", "train/line_00001.png"]
def test_prepare_hf_dataset_accepts_non_hashable_image_objects(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class FakeImage:
__hash__ = None
def save(self, path: Path) -> None:
Path(path).write_bytes(b"fake image bytes")
fake_module = SimpleNamespace(
load_dataset=lambda dataset_id, split, streaming=False: [{"id": 7, "image": FakeImage(), "text": "नेपाल"}]
)
monkeypatch.setitem(sys.modules, "datasets", fake_module)
report = prepare_hf_dataset("gauravgiri/nepali-ocr-dataset", tmp_path / "manifests", split="train")
entries = load_manifest(report.manifest_path)
assert len(entries) == 1
assert entries[0].sample_id == "gauravgiri__nepali-ocr-dataset-train-7"
assert entries[0].text == "नेपाल"
assert Path(entries[0].image_path).exists()
assert "nepali" in entries[0].metadata["slices"]
assert "real" in entries[0].metadata["slices"]
def test_prepare_local_dataset_can_stamp_slice_metadata(tmp_path: Path) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"fake image bytes")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8")
report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train", slices=["nepali", "real"])
entries = load_manifest(report.manifest_path)
assert entries[0].metadata["slices"] == ["nepali", "real"]
def test_normalize_manifest_images_converts_unsupported_suffix_and_updates_hash(tmp_path: Path) -> None:
source = tmp_path / "line.img"
source.write_bytes(b"raw image bytes")
manifest = tmp_path / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample:1",
dataset="fixture",
split="eval",
image_path=str(source),
text="नेपाल",
sha256=sha256_file(source),
metadata={"slices": ["nepali"]},
)
],
manifest,
)
class FakeImage:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb) -> None:
return None
def save(self, target: Path, format: str) -> None:
assert format == "PNG"
target.write_bytes(b"converted png bytes")
fake_pil = SimpleNamespace(Image=SimpleNamespace(open=lambda path: FakeImage()))
monkeypatch = pytest.MonkeyPatch()
monkeypatch.setitem(sys.modules, "PIL", fake_pil)
try:
report = normalize_manifest_images(manifest, tmp_path / "normalized")
finally:
monkeypatch.undo()
entries = load_manifest(report.manifest_path)
normalized = Path(entries[0].image_path)
assert report.sample_count == 1
assert report.copied_count == 0
assert report.converted_count == 1
assert report.unsupported_suffixes == {".img": 1}
assert normalized.suffix == ".png"
assert normalized.exists()
assert entries[0].sha256 == sha256_file(normalized)
assert entries[0].metadata["source_image_path"] == str(source)
assert entries[0].metadata["image_normalization"] == "converted_to_png"
def test_normalize_manifest_images_resolves_paths_relative_to_cwd_before_manifest_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
image = tmp_path / "assets" / "line.png"
image.parent.mkdir()
image.write_bytes(b"png")
manifest_dir = tmp_path / "manifests"
manifest_dir.mkdir()
manifest = manifest_dir / "eval.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-1",
dataset="fixture",
split="eval",
image_path="assets/line.png",
text="नेपाल",
sha256=sha256_file(image),
)
],
manifest,
)
monkeypatch.chdir(tmp_path)
report = normalize_manifest_images(manifest, tmp_path / "normalized")
entries = load_manifest(report.manifest_path)
assert report.copied_count == 1
assert Path(entries[0].image_path).exists()
assert entries[0].metadata["source_image_path"] == "assets/line.png"
def test_prepare_local_dataset_can_skip_invalid_rows_with_reject_report(tmp_path: Path) -> None:
first = tmp_path / "line-a.png"
second = tmp_path / "line-b.png"
first.write_bytes(b"line-a")
second.write_bytes(b"line-b")
labels = tmp_path / "labels.csv"
labels.write_text("image_path,text\nline-a.png,नेपाल\nline-b.png,\nmissing.png,English\n", encoding="utf-8")
report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train", skip_invalid=True)
entries = load_manifest(report.manifest_path)
rejects = list(read_jsonl(Path(report.rejects_path or "")))
assert len(entries) == 1
assert report.sample_count == 1
assert report.rejected_count == 2
assert len(rejects) == 2
assert "text field 'text' is empty" in rejects[0]["reason"]
assert "image does not exist" in rejects[1]["reason"]
def test_prepare_hf_correction_pairs_cfilt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
fake_module = SimpleNamespace(load_dataset=lambda dataset_id, split, streaming=False: [{"ocr": "गलत पाठ", "correct": "सही पाठ", "font": "Kalimati"}])
monkeypatch.setitem(sys.modules, "datasets", fake_module)
pairs_path = prepare_hf_correction_pairs("cfilt/RoundTripOCR-nepali", tmp_path / "pairs.jsonl", split="train")
rows = list(read_jsonl(pairs_path))
assert rows[0]["noisy_text"] == "गलत पाठ"
assert rows[0]["clean_text"] == "सही पाठ"
assert rows[0]["font"] == "Kalimati"
def test_inspect_hf_dataset_records_schema_without_writing_manifest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class FakeImage:
size = (120, 24)
mode = "RGB"
def save(self, path: Path) -> None:
Path(path).write_bytes(b"fake image bytes")
fake_module = SimpleNamespace(
load_dataset=lambda dataset_id, split, streaming=False: [
{"id": 1, "image": FakeImage(), "text": "नेपाल"},
{"id": 2, "image": FakeImage(), "text": "English"},
]
)
monkeypatch.setitem(sys.modules, "datasets", fake_module)
report = inspect_hf_dataset("fixture/hf-ocr", tmp_path / "inspect", split="train", limit=2)
assert report.passed is True
assert report.inspection_mode == "datasets.load_dataset"
assert report.fields == ["id", "image", "text"]
assert report.inferred_image_field == "image"
assert report.inferred_text_field == "text"
assert report.inspected_count == 2
assert report.samples[0]["image_supported_by_prepare"] is True
assert report.samples[0]["text_preview"] == "नेपाल"
assert Path(report.output_json_path or "").exists()
assert Path(report.output_md_path or "").exists()
def test_inspect_hf_prashant_dataset_checks_metadata_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
metadata = tmp_path / "metadata.jsonl"
metadata.write_text(
'{"file_name":"line_00001.png","text":"नेपाल सरकार"}\n{"file_name":"line_99999.png","text":"छैन"}\n',
encoding="utf-8",
)
monkeypatch.setattr(
"ocrtech.manifest._list_dataset_files",
lambda *args, **kwargs: ["train/metadata.jsonl", "train/line_00001.png"],
)
monkeypatch.setattr("ocrtech.manifest._download_dataset_file", lambda dataset_id, dataset_file: metadata)
report = inspect_hf_dataset("prashant0919/nepali-synthetic-ocr-lines", tmp_path / "inspect", split="train", limit=2)
assert report.passed is False
assert report.inferred_image_field == "file_name"
assert report.inferred_text_field == "text"
assert report.samples[0]["image_exists_in_repo"] is True
assert report.samples[1]["image_exists_in_repo"] is False
assert "train/line_99999.png" in report.issues[0]