Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_manifest.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 12 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_manifest.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_manifest.py
-
curl -L -o test_manifest.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_manifest.py
12 kB
| from __future__ import annotations | |
| import json | |
| import sys | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| import pytest | |
| from ocrtech.errors import DataValidationError | |
| from ocrtech.manifest import ( | |
| ManifestEntry, | |
| inspect_hf_dataset, | |
| load_manifest, | |
| normalize_manifest_images, | |
| prepare_hf_correction_pairs, | |
| prepare_hf_dataset, | |
| prepare_local_dataset, | |
| read_jsonl, | |
| sha256_file, | |
| write_manifest, | |
| ) | |
| def test_prepare_local_dataset_verifies_fields_and_images(tmp_path: Path) -> None: | |
| image = tmp_path / "line.png" | |
| image.write_bytes(b"fake image bytes") | |
| labels = tmp_path / "labels.csv" | |
| labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8") | |
| report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") | |
| entries = load_manifest(report.manifest_path) | |
| assert len(entries) == 1 | |
| assert entries[0].sample_id == "local-train-00000000" | |
| assert entries[0].text == "नेपाल" | |
| assert entries[0].sha256 | |
| assert report.rejected_count == 0 | |
| def test_prepare_local_dataset_fails_on_missing_image(tmp_path: Path) -> None: | |
| labels = tmp_path / "labels.jsonl" | |
| labels.write_text(json.dumps({"image_path": "missing.png", "text": "नेपाल"}, ensure_ascii=False) + "\n", encoding="utf-8") | |
| with pytest.raises(DataValidationError, match="image does not exist"): | |
| prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") | |
| def test_prepare_local_dataset_fails_on_missing_text_field(tmp_path: Path) -> None: | |
| image = tmp_path / "line.png" | |
| image.write_bytes(b"fake image bytes") | |
| labels = tmp_path / "labels.csv" | |
| labels.write_text("image_path,caption\nline.png,नेपाल\n", encoding="utf-8") | |
| with pytest.raises(DataValidationError, match="Could not infer text field"): | |
| prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") | |
| def test_prepare_local_dataset_fails_on_empty_text_value(tmp_path: Path) -> None: | |
| image = tmp_path / "line.png" | |
| image.write_bytes(b"fake image bytes") | |
| labels = tmp_path / "labels.csv" | |
| labels.write_text("image_path,text\nline.png,\n", encoding="utf-8") | |
| with pytest.raises(DataValidationError, match="text field 'text' is empty"): | |
| prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") | |
| def test_prepare_hf_prashant_dataset_downloads_only_needed_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| image = tmp_path / "line_00001.png" | |
| image.write_bytes(b"fake image bytes") | |
| metadata = tmp_path / "metadata.jsonl" | |
| metadata.write_text( | |
| '{"file_name":"line_00001.png","text":"नेपाल सरकार"}\n{"file_name":"line_99999.png","text":"unused"}\n', | |
| encoding="utf-8", | |
| ) | |
| requested_files: list[str] = [] | |
| monkeypatch.setattr( | |
| "ocrtech.manifest._list_dataset_files", | |
| lambda *args, **kwargs: ["train/metadata.jsonl", "train/line_00001.png", "train/line_99999.png"], | |
| ) | |
| def fake_download(dataset_id: str, dataset_file: str) -> Path: | |
| requested_files.append(dataset_file) | |
| if dataset_file == "train/metadata.jsonl": | |
| return metadata | |
| if dataset_file == "train/line_00001.png": | |
| return image | |
| raise AssertionError(f"unexpected dataset download: {dataset_file}") | |
| monkeypatch.setattr("ocrtech.manifest._download_dataset_file", fake_download) | |
| report = prepare_hf_dataset( | |
| "prashant0919/nepali-synthetic-ocr-lines", | |
| tmp_path / "manifests", | |
| split="train", | |
| limit=1, | |
| ) | |
| entries = load_manifest(report.manifest_path) | |
| assert len(entries) == 1 | |
| assert entries[0].text == "नेपाल सरकार" | |
| assert Path(entries[0].image_path) == image | |
| assert requested_files == ["train/metadata.jsonl", "train/line_00001.png"] | |
| def test_prepare_hf_dataset_accepts_non_hashable_image_objects(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| class FakeImage: | |
| __hash__ = None | |
| def save(self, path: Path) -> None: | |
| Path(path).write_bytes(b"fake image bytes") | |
| fake_module = SimpleNamespace( | |
| load_dataset=lambda dataset_id, split, streaming=False: [{"id": 7, "image": FakeImage(), "text": "नेपाल"}] | |
| ) | |
| monkeypatch.setitem(sys.modules, "datasets", fake_module) | |
| report = prepare_hf_dataset("gauravgiri/nepali-ocr-dataset", tmp_path / "manifests", split="train") | |
| entries = load_manifest(report.manifest_path) | |
| assert len(entries) == 1 | |
| assert entries[0].sample_id == "gauravgiri__nepali-ocr-dataset-train-7" | |
| assert entries[0].text == "नेपाल" | |
| assert Path(entries[0].image_path).exists() | |
| assert "nepali" in entries[0].metadata["slices"] | |
| assert "real" in entries[0].metadata["slices"] | |
| def test_prepare_local_dataset_can_stamp_slice_metadata(tmp_path: Path) -> None: | |
| image = tmp_path / "line.png" | |
| image.write_bytes(b"fake image bytes") | |
| labels = tmp_path / "labels.csv" | |
| labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8") | |
| report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train", slices=["nepali", "real"]) | |
| entries = load_manifest(report.manifest_path) | |
| assert entries[0].metadata["slices"] == ["nepali", "real"] | |
| def test_normalize_manifest_images_converts_unsupported_suffix_and_updates_hash(tmp_path: Path) -> None: | |
| source = tmp_path / "line.img" | |
| source.write_bytes(b"raw image bytes") | |
| manifest = tmp_path / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample:1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(source), | |
| text="नेपाल", | |
| sha256=sha256_file(source), | |
| metadata={"slices": ["nepali"]}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| class FakeImage: | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, exc_type, exc, tb) -> None: | |
| return None | |
| def save(self, target: Path, format: str) -> None: | |
| assert format == "PNG" | |
| target.write_bytes(b"converted png bytes") | |
| fake_pil = SimpleNamespace(Image=SimpleNamespace(open=lambda path: FakeImage())) | |
| monkeypatch = pytest.MonkeyPatch() | |
| monkeypatch.setitem(sys.modules, "PIL", fake_pil) | |
| try: | |
| report = normalize_manifest_images(manifest, tmp_path / "normalized") | |
| finally: | |
| monkeypatch.undo() | |
| entries = load_manifest(report.manifest_path) | |
| normalized = Path(entries[0].image_path) | |
| assert report.sample_count == 1 | |
| assert report.copied_count == 0 | |
| assert report.converted_count == 1 | |
| assert report.unsupported_suffixes == {".img": 1} | |
| assert normalized.suffix == ".png" | |
| assert normalized.exists() | |
| assert entries[0].sha256 == sha256_file(normalized) | |
| assert entries[0].metadata["source_image_path"] == str(source) | |
| assert entries[0].metadata["image_normalization"] == "converted_to_png" | |
| def test_normalize_manifest_images_resolves_paths_relative_to_cwd_before_manifest_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| image = tmp_path / "assets" / "line.png" | |
| image.parent.mkdir() | |
| image.write_bytes(b"png") | |
| manifest_dir = tmp_path / "manifests" | |
| manifest_dir.mkdir() | |
| manifest = manifest_dir / "eval.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path="assets/line.png", | |
| text="नेपाल", | |
| sha256=sha256_file(image), | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| monkeypatch.chdir(tmp_path) | |
| report = normalize_manifest_images(manifest, tmp_path / "normalized") | |
| entries = load_manifest(report.manifest_path) | |
| assert report.copied_count == 1 | |
| assert Path(entries[0].image_path).exists() | |
| assert entries[0].metadata["source_image_path"] == "assets/line.png" | |
| def test_prepare_local_dataset_can_skip_invalid_rows_with_reject_report(tmp_path: Path) -> None: | |
| first = tmp_path / "line-a.png" | |
| second = tmp_path / "line-b.png" | |
| first.write_bytes(b"line-a") | |
| second.write_bytes(b"line-b") | |
| labels = tmp_path / "labels.csv" | |
| labels.write_text("image_path,text\nline-a.png,नेपाल\nline-b.png,\nmissing.png,English\n", encoding="utf-8") | |
| report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train", skip_invalid=True) | |
| entries = load_manifest(report.manifest_path) | |
| rejects = list(read_jsonl(Path(report.rejects_path or ""))) | |
| assert len(entries) == 1 | |
| assert report.sample_count == 1 | |
| assert report.rejected_count == 2 | |
| assert len(rejects) == 2 | |
| assert "text field 'text' is empty" in rejects[0]["reason"] | |
| assert "image does not exist" in rejects[1]["reason"] | |
| def test_prepare_hf_correction_pairs_cfilt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| fake_module = SimpleNamespace(load_dataset=lambda dataset_id, split, streaming=False: [{"ocr": "गलत पाठ", "correct": "सही पाठ", "font": "Kalimati"}]) | |
| monkeypatch.setitem(sys.modules, "datasets", fake_module) | |
| pairs_path = prepare_hf_correction_pairs("cfilt/RoundTripOCR-nepali", tmp_path / "pairs.jsonl", split="train") | |
| rows = list(read_jsonl(pairs_path)) | |
| assert rows[0]["noisy_text"] == "गलत पाठ" | |
| assert rows[0]["clean_text"] == "सही पाठ" | |
| assert rows[0]["font"] == "Kalimati" | |
| def test_inspect_hf_dataset_records_schema_without_writing_manifest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| class FakeImage: | |
| size = (120, 24) | |
| mode = "RGB" | |
| def save(self, path: Path) -> None: | |
| Path(path).write_bytes(b"fake image bytes") | |
| fake_module = SimpleNamespace( | |
| load_dataset=lambda dataset_id, split, streaming=False: [ | |
| {"id": 1, "image": FakeImage(), "text": "नेपाल"}, | |
| {"id": 2, "image": FakeImage(), "text": "English"}, | |
| ] | |
| ) | |
| monkeypatch.setitem(sys.modules, "datasets", fake_module) | |
| report = inspect_hf_dataset("fixture/hf-ocr", tmp_path / "inspect", split="train", limit=2) | |
| assert report.passed is True | |
| assert report.inspection_mode == "datasets.load_dataset" | |
| assert report.fields == ["id", "image", "text"] | |
| assert report.inferred_image_field == "image" | |
| assert report.inferred_text_field == "text" | |
| assert report.inspected_count == 2 | |
| assert report.samples[0]["image_supported_by_prepare"] is True | |
| assert report.samples[0]["text_preview"] == "नेपाल" | |
| assert Path(report.output_json_path or "").exists() | |
| assert Path(report.output_md_path or "").exists() | |
| def test_inspect_hf_prashant_dataset_checks_metadata_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| metadata = tmp_path / "metadata.jsonl" | |
| metadata.write_text( | |
| '{"file_name":"line_00001.png","text":"नेपाल सरकार"}\n{"file_name":"line_99999.png","text":"छैन"}\n', | |
| encoding="utf-8", | |
| ) | |
| monkeypatch.setattr( | |
| "ocrtech.manifest._list_dataset_files", | |
| lambda *args, **kwargs: ["train/metadata.jsonl", "train/line_00001.png"], | |
| ) | |
| monkeypatch.setattr("ocrtech.manifest._download_dataset_file", lambda dataset_id, dataset_file: metadata) | |
| report = inspect_hf_dataset("prashant0919/nepali-synthetic-ocr-lines", tmp_path / "inspect", split="train", limit=2) | |
| assert report.passed is False | |
| assert report.inferred_image_field == "file_name" | |
| assert report.inferred_text_field == "text" | |
| assert report.samples[0]["image_exists_in_repo"] is True | |
| assert report.samples[1]["image_exists_in_repo"] is False | |
| assert "train/line_99999.png" in report.issues[0] | |