from __future__ import annotations import json import sys from pathlib import Path from types import SimpleNamespace import pytest from ocrtech.errors import DataValidationError from ocrtech.manifest import ( ManifestEntry, inspect_hf_dataset, load_manifest, normalize_manifest_images, prepare_hf_correction_pairs, prepare_hf_dataset, prepare_local_dataset, read_jsonl, sha256_file, write_manifest, ) def test_prepare_local_dataset_verifies_fields_and_images(tmp_path: Path) -> None: image = tmp_path / "line.png" image.write_bytes(b"fake image bytes") labels = tmp_path / "labels.csv" labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8") report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") entries = load_manifest(report.manifest_path) assert len(entries) == 1 assert entries[0].sample_id == "local-train-00000000" assert entries[0].text == "नेपाल" assert entries[0].sha256 assert report.rejected_count == 0 def test_prepare_local_dataset_fails_on_missing_image(tmp_path: Path) -> None: labels = tmp_path / "labels.jsonl" labels.write_text(json.dumps({"image_path": "missing.png", "text": "नेपाल"}, ensure_ascii=False) + "\n", encoding="utf-8") with pytest.raises(DataValidationError, match="image does not exist"): prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") def test_prepare_local_dataset_fails_on_missing_text_field(tmp_path: Path) -> None: image = tmp_path / "line.png" image.write_bytes(b"fake image bytes") labels = tmp_path / "labels.csv" labels.write_text("image_path,caption\nline.png,नेपाल\n", encoding="utf-8") with pytest.raises(DataValidationError, match="Could not infer text field"): prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") def test_prepare_local_dataset_fails_on_empty_text_value(tmp_path: Path) -> None: image = tmp_path / "line.png" image.write_bytes(b"fake image bytes") labels = tmp_path / "labels.csv" labels.write_text("image_path,text\nline.png,\n", encoding="utf-8") with pytest.raises(DataValidationError, match="text field 'text' is empty"): prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train") def test_prepare_hf_prashant_dataset_downloads_only_needed_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: image = tmp_path / "line_00001.png" image.write_bytes(b"fake image bytes") metadata = tmp_path / "metadata.jsonl" metadata.write_text( '{"file_name":"line_00001.png","text":"नेपाल सरकार"}\n{"file_name":"line_99999.png","text":"unused"}\n', encoding="utf-8", ) requested_files: list[str] = [] monkeypatch.setattr( "ocrtech.manifest._list_dataset_files", lambda *args, **kwargs: ["train/metadata.jsonl", "train/line_00001.png", "train/line_99999.png"], ) def fake_download(dataset_id: str, dataset_file: str) -> Path: requested_files.append(dataset_file) if dataset_file == "train/metadata.jsonl": return metadata if dataset_file == "train/line_00001.png": return image raise AssertionError(f"unexpected dataset download: {dataset_file}") monkeypatch.setattr("ocrtech.manifest._download_dataset_file", fake_download) report = prepare_hf_dataset( "prashant0919/nepali-synthetic-ocr-lines", tmp_path / "manifests", split="train", limit=1, ) entries = load_manifest(report.manifest_path) assert len(entries) == 1 assert entries[0].text == "नेपाल सरकार" assert Path(entries[0].image_path) == image assert requested_files == ["train/metadata.jsonl", "train/line_00001.png"] def test_prepare_hf_dataset_accepts_non_hashable_image_objects(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: class FakeImage: __hash__ = None def save(self, path: Path) -> None: Path(path).write_bytes(b"fake image bytes") fake_module = SimpleNamespace( load_dataset=lambda dataset_id, split, streaming=False: [{"id": 7, "image": FakeImage(), "text": "नेपाल"}] ) monkeypatch.setitem(sys.modules, "datasets", fake_module) report = prepare_hf_dataset("gauravgiri/nepali-ocr-dataset", tmp_path / "manifests", split="train") entries = load_manifest(report.manifest_path) assert len(entries) == 1 assert entries[0].sample_id == "gauravgiri__nepali-ocr-dataset-train-7" assert entries[0].text == "नेपाल" assert Path(entries[0].image_path).exists() assert "nepali" in entries[0].metadata["slices"] assert "real" in entries[0].metadata["slices"] def test_prepare_local_dataset_can_stamp_slice_metadata(tmp_path: Path) -> None: image = tmp_path / "line.png" image.write_bytes(b"fake image bytes") labels = tmp_path / "labels.csv" labels.write_text("image_path,text\nline.png,नेपाल\n", encoding="utf-8") report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train", slices=["nepali", "real"]) entries = load_manifest(report.manifest_path) assert entries[0].metadata["slices"] == ["nepali", "real"] def test_normalize_manifest_images_converts_unsupported_suffix_and_updates_hash(tmp_path: Path) -> None: source = tmp_path / "line.img" source.write_bytes(b"raw image bytes") manifest = tmp_path / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample:1", dataset="fixture", split="eval", image_path=str(source), text="नेपाल", sha256=sha256_file(source), metadata={"slices": ["nepali"]}, ) ], manifest, ) class FakeImage: def __enter__(self): return self def __exit__(self, exc_type, exc, tb) -> None: return None def save(self, target: Path, format: str) -> None: assert format == "PNG" target.write_bytes(b"converted png bytes") fake_pil = SimpleNamespace(Image=SimpleNamespace(open=lambda path: FakeImage())) monkeypatch = pytest.MonkeyPatch() monkeypatch.setitem(sys.modules, "PIL", fake_pil) try: report = normalize_manifest_images(manifest, tmp_path / "normalized") finally: monkeypatch.undo() entries = load_manifest(report.manifest_path) normalized = Path(entries[0].image_path) assert report.sample_count == 1 assert report.copied_count == 0 assert report.converted_count == 1 assert report.unsupported_suffixes == {".img": 1} assert normalized.suffix == ".png" assert normalized.exists() assert entries[0].sha256 == sha256_file(normalized) assert entries[0].metadata["source_image_path"] == str(source) assert entries[0].metadata["image_normalization"] == "converted_to_png" def test_normalize_manifest_images_resolves_paths_relative_to_cwd_before_manifest_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: image = tmp_path / "assets" / "line.png" image.parent.mkdir() image.write_bytes(b"png") manifest_dir = tmp_path / "manifests" manifest_dir.mkdir() manifest = manifest_dir / "eval.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-1", dataset="fixture", split="eval", image_path="assets/line.png", text="नेपाल", sha256=sha256_file(image), ) ], manifest, ) monkeypatch.chdir(tmp_path) report = normalize_manifest_images(manifest, tmp_path / "normalized") entries = load_manifest(report.manifest_path) assert report.copied_count == 1 assert Path(entries[0].image_path).exists() assert entries[0].metadata["source_image_path"] == "assets/line.png" def test_prepare_local_dataset_can_skip_invalid_rows_with_reject_report(tmp_path: Path) -> None: first = tmp_path / "line-a.png" second = tmp_path / "line-b.png" first.write_bytes(b"line-a") second.write_bytes(b"line-b") labels = tmp_path / "labels.csv" labels.write_text("image_path,text\nline-a.png,नेपाल\nline-b.png,\nmissing.png,English\n", encoding="utf-8") report = prepare_local_dataset(labels, tmp_path / "manifests", dataset="local", split="train", skip_invalid=True) entries = load_manifest(report.manifest_path) rejects = list(read_jsonl(Path(report.rejects_path or ""))) assert len(entries) == 1 assert report.sample_count == 1 assert report.rejected_count == 2 assert len(rejects) == 2 assert "text field 'text' is empty" in rejects[0]["reason"] assert "image does not exist" in rejects[1]["reason"] def test_prepare_hf_correction_pairs_cfilt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: fake_module = SimpleNamespace(load_dataset=lambda dataset_id, split, streaming=False: [{"ocr": "गलत पाठ", "correct": "सही पाठ", "font": "Kalimati"}]) monkeypatch.setitem(sys.modules, "datasets", fake_module) pairs_path = prepare_hf_correction_pairs("cfilt/RoundTripOCR-nepali", tmp_path / "pairs.jsonl", split="train") rows = list(read_jsonl(pairs_path)) assert rows[0]["noisy_text"] == "गलत पाठ" assert rows[0]["clean_text"] == "सही पाठ" assert rows[0]["font"] == "Kalimati" def test_inspect_hf_dataset_records_schema_without_writing_manifest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: class FakeImage: size = (120, 24) mode = "RGB" def save(self, path: Path) -> None: Path(path).write_bytes(b"fake image bytes") fake_module = SimpleNamespace( load_dataset=lambda dataset_id, split, streaming=False: [ {"id": 1, "image": FakeImage(), "text": "नेपाल"}, {"id": 2, "image": FakeImage(), "text": "English"}, ] ) monkeypatch.setitem(sys.modules, "datasets", fake_module) report = inspect_hf_dataset("fixture/hf-ocr", tmp_path / "inspect", split="train", limit=2) assert report.passed is True assert report.inspection_mode == "datasets.load_dataset" assert report.fields == ["id", "image", "text"] assert report.inferred_image_field == "image" assert report.inferred_text_field == "text" assert report.inspected_count == 2 assert report.samples[0]["image_supported_by_prepare"] is True assert report.samples[0]["text_preview"] == "नेपाल" assert Path(report.output_json_path or "").exists() assert Path(report.output_md_path or "").exists() def test_inspect_hf_prashant_dataset_checks_metadata_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: metadata = tmp_path / "metadata.jsonl" metadata.write_text( '{"file_name":"line_00001.png","text":"नेपाल सरकार"}\n{"file_name":"line_99999.png","text":"छैन"}\n', encoding="utf-8", ) monkeypatch.setattr( "ocrtech.manifest._list_dataset_files", lambda *args, **kwargs: ["train/metadata.jsonl", "train/line_00001.png"], ) monkeypatch.setattr("ocrtech.manifest._download_dataset_file", lambda dataset_id, dataset_file: metadata) report = inspect_hf_dataset("prashant0919/nepali-synthetic-ocr-lines", tmp_path / "inspect", split="train", limit=2) assert report.passed is False assert report.inferred_image_field == "file_name" assert report.inferred_text_field == "text" assert report.samples[0]["image_exists_in_repo"] is True assert report.samples[1]["image_exists_in_repo"] is False assert "train/line_99999.png" in report.issues[0]