from __future__ import annotations import json from pathlib import Path import pytest from ocrtech.datasets import audit_manifest, audit_recognizer_corpus, build_recognizer_corpus, filter_manifest, merge_manifests, rebalance_manifest, split_manifest from ocrtech.errors import DataValidationError from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_path, sha256_text, write_manifest from ocrtech.validation import check_leakage def test_audit_manifest_reports_counts_and_missing_slice(tmp_path: Path) -> None: manifest = _write_combined_manifest(tmp_path, count=4) audit = audit_manifest(manifest, tmp_path / "audit", require_slices=["nepali", "table", "missing"]) assert not audit.passed assert audit.sample_count == 4 assert audit.slice_counts["nepali"] == 2 assert "required slice missing: missing" in audit.issues assert (tmp_path / "audit" / "manifest-audit.json").exists() def test_split_manifest_is_deterministic_and_leakage_safe(tmp_path: Path) -> None: manifest = _write_combined_manifest(tmp_path, count=10) first = split_manifest(manifest, tmp_path / "split-a", eval_ratio=0.3, seed=7) second = split_manifest(manifest, tmp_path / "split-b", eval_ratio=0.3, seed=7) first_eval_ids = [entry.sample_id for entry in load_manifest(first.eval_manifest)] second_eval_ids = [entry.sample_id for entry in load_manifest(second.eval_manifest)] assert first_eval_ids == second_eval_ids assert first.train_count + first.eval_count == 10 assert check_leakage([first.train_manifest], first.eval_manifest).passed def test_split_manifest_keeps_duplicate_sample_signature_in_same_split(tmp_path: Path) -> None: image = tmp_path / "shared.png" image.write_bytes(b"same-image") image_hash = sha256_file(image) text_hash = sha256_text("same text") sample_hash = sha256_text(f"{image_hash}\n{text_hash}") manifest = tmp_path / "combined.jsonl" write_manifest( [ ManifestEntry( sample_id="sample-a", dataset="local", split="all", image_path=str(image), text="same text", sha256=image_hash, metadata={"slices": ["nepali"], "text_sha256": text_hash, "sample_sha256": sample_hash}, ), ManifestEntry( sample_id="sample-b", dataset="local", split="all", image_path=str(image), text="same text", sha256=image_hash, metadata={"slices": ["nepali"], "text_sha256": text_hash, "sample_sha256": sample_hash}, ), *_entries(tmp_path, start=2, count=4), ], manifest, ) summary = split_manifest(manifest, tmp_path / "split", eval_ratio=0.5, seed=3) train_ids = {entry.sample_id for entry in load_manifest(summary.train_manifest)} eval_ids = {entry.sample_id for entry in load_manifest(summary.eval_manifest)} assert {"sample-a", "sample-b"}.issubset(train_ids) or {"sample-a", "sample-b"}.issubset(eval_ids) assert check_leakage([summary.train_manifest], summary.eval_manifest).passed def test_split_manifest_can_group_by_text_sha256(tmp_path: Path) -> None: manifest = tmp_path / "combined.jsonl" entries = [] for index in range(6): text = "shared text" if index < 3 else f"text {index}" image = tmp_path / f"sample-{index}.png" image.write_bytes(f"image-{index}".encode("utf-8")) text_hash = sha256_text(text) entries.append( ManifestEntry( sample_id=f"sample-{index}", dataset="local", split="all", image_path=str(image), text=text, sha256=sha256_file(image), metadata={"slices": ["nepali"], "text_sha256": text_hash}, ) ) write_manifest(entries, manifest) summary = split_manifest(manifest, tmp_path / "split", eval_ratio=0.5, seed=1, group_by="text_sha256") train_entries = load_manifest(summary.train_manifest) eval_entries = load_manifest(summary.eval_manifest) train_shared = {entry.sample_id for entry in train_entries if entry.text == "shared text"} eval_shared = {entry.sample_id for entry in eval_entries if entry.text == "shared text"} assert summary.group_by == "text_sha256" assert not (train_shared and eval_shared) def test_audit_manifest_fails_on_hash_mismatch(tmp_path: Path) -> None: image = tmp_path / "line.png" image.write_bytes(b"actual") manifest = tmp_path / "manifest.jsonl" manifest.write_text( json.dumps( { "sample_id": "bad", "dataset": "local", "split": "train", "image_path": str(image), "text": "नेपाल", "sha256": "not-the-real-hash", "metadata": {"slices": ["nepali"]}, }, ensure_ascii=False, ) + "\n", encoding="utf-8", ) audit = audit_manifest(manifest) assert not audit.passed assert any("sha256 mismatch" in issue for issue in audit.issues) def test_audit_manifest_supports_directory_inputs(tmp_path: Path) -> None: bundle = tmp_path / "bundle" bundle.mkdir() (bundle / "page-0001.png").write_bytes(b"page-1") (bundle / "page-0002.png").write_bytes(b"page-2") manifest = tmp_path / "manifest.jsonl" write_manifest( [ ManifestEntry( sample_id="bundle-sample", dataset="local", split="eval", image_path=str(bundle), text="नेपाल सरकार\nApproved by: Sita Rana", sha256=sha256_path(bundle), metadata={"slices": ["nepali", "english", "multi_page"], "text_sha256": sha256_text("नेपाल सरकार\nApproved by: Sita Rana")}, ) ], manifest, ) audit = audit_manifest(manifest) assert audit.passed assert audit.sample_count == 1 def test_merge_manifests_combines_inputs_and_writes_summary(tmp_path: Path) -> None: first = tmp_path / "first.jsonl" second = tmp_path / "second.jsonl" write_manifest(_entries(tmp_path, start=0, count=2), first) write_manifest(_entries(tmp_path, start=2, count=2), second) summary = merge_manifests([first, second], tmp_path / "combined.jsonl") merged = load_manifest(summary.output_manifest) assert len(merged) == 4 assert summary.sample_count == 4 assert (tmp_path / "combined-merge.json").exists() assert (tmp_path / "combined-merge.md").exists() def test_rebalance_manifest_upsamples_requested_slice_and_writes_summary(tmp_path: Path) -> None: manifest = _write_combined_manifest(tmp_path, count=6) summary = rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["english", "nepali"], seed=5) rebalanced = load_manifest(summary.output_manifest) assert summary.original_count == 6 assert summary.target_count == 3 assert summary.slice_counts_before == {"english": 3, "nepali": 3} assert summary.slice_counts_after == {"english": 3, "nepali": 3} assert len(rebalanced) == 6 assert (tmp_path / "rebalanced-rebalance.json").exists() assert (tmp_path / "rebalanced-rebalance.md").exists() def test_rebalance_manifest_can_fill_underrepresented_slice(tmp_path: Path) -> None: manifest = tmp_path / "combined.jsonl" write_manifest(_entries(tmp_path, start=0, count=5), manifest) summary = rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["english", "nepali"], target_count=4, seed=7) rebalanced = load_manifest(summary.output_manifest) duplicated = [entry for entry in rebalanced if entry.metadata.get("rebalance_duplicate")] assert summary.original_count == 5 assert summary.rebalanced_count == 8 assert summary.slice_counts_before == {"english": 2, "nepali": 3} assert summary.slice_counts_after == {"english": 4, "nepali": 4} assert len(duplicated) == 3 assert all(entry.sample_id.startswith("sample-") and "~rebalance-english-" in entry.sample_id for entry in duplicated[:2]) assert all(entry.metadata["rebalance_source_sample_id"].startswith("sample-") for entry in duplicated) def test_rebalance_manifest_requires_present_slice(tmp_path: Path) -> None: manifest = _write_combined_manifest(tmp_path, count=4) with pytest.raises(DataValidationError, match="missing requested slices: missing"): rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["missing"], target_count=5) def test_audit_recognizer_corpus_passes_balanced_train_eval_manifests(tmp_path: Path) -> None: train = _write_recognizer_manifest( tmp_path, "train.jsonl", [ ("tr-en-1", "Invoice total", ["english", "real"]), ("tr-en-2", "Account number", ["english", "synthetic"]), ("tr-ne-1", "नेपाल सरकार", ["nepali", "real"]), ("tr-ne-2", "जम्मा रकम", ["nepali", "synthetic"]), ("tr-mix-1", "नाम Sita", ["mixed_script", "real"]), ("tr-mix-2", "रकम USD", ["mixed_script", "synthetic"]), ], ) eval_manifest = _write_recognizer_manifest( tmp_path, "eval.jsonl", [ ("ev-en", "Receipt date", ["english", "hard_eval"]), ("ev-ne", "प्रदेश कार्यालय", ["nepali", "hard_eval"]), ("ev-mix", "बिल No 7", ["mixed_script", "hard_eval"]), ], ) audit = audit_recognizer_corpus( train, eval_manifest, tmp_path / "audit", min_train_samples=6, min_eval_samples=3, min_train_english=3, min_train_devanagari=4, min_train_mixed=2, min_eval_english=2, min_eval_devanagari=2, min_eval_mixed=1, min_train_latin_only=2, min_train_devanagari_only=2, min_eval_latin_only=1, min_eval_devanagari_only=1, min_train_real=3, min_train_synthetic=3, ) assert audit.passed assert audit.leakage_passed assert audit.train_profile.script_sample_counts["mixed_devanagari_latin"] == 2 assert (tmp_path / "audit" / "recognizer-corpus-audit.json").exists() assert (tmp_path / "audit" / "recognizer-corpus-audit.md").exists() def test_audit_recognizer_corpus_fails_missing_english_and_leakage(tmp_path: Path) -> None: train = _write_recognizer_manifest( tmp_path, "train.jsonl", [ ("shared", "नेपाल सरकार", ["nepali", "real"]), ("tr-ne-2", "जम्मा रकम", ["nepali", "synthetic"]), ], ) eval_manifest = _write_recognizer_manifest( tmp_path, "eval.jsonl", [ ("shared", "नेपाल सरकार", ["nepali", "hard_eval"]), ("ev-ne-2", "प्रदेश कार्यालय", ["nepali", "hard_eval"]), ], ) audit = audit_recognizer_corpus( train, eval_manifest, min_train_samples=2, min_eval_samples=2, min_train_english=1, min_train_devanagari=2, min_train_mixed=1, min_eval_english=1, min_eval_devanagari=2, min_eval_mixed=0, min_train_latin_only=1, min_train_devanagari_only=1, min_eval_latin_only=1, min_eval_devanagari_only=1, min_train_real=1, min_train_synthetic=1, ) assert not audit.passed assert not audit.leakage_passed assert "train latin-script samples 0 < required 1" in audit.issues assert "train mixed-script samples 0 < required 1" in audit.issues assert "train latin-only samples 0 < required 1" in audit.issues assert any(issue.startswith("train/eval leakage detected") for issue in audit.issues) def test_audit_recognizer_corpus_can_require_isolated_script_samples(tmp_path: Path) -> None: train = _write_recognizer_manifest( tmp_path, "train.jsonl", [ ("tr-mix-1", "नाम Ram", ["mixed_script", "real"]), ("tr-mix-2", "रकम USD", ["mixed_script", "synthetic"]), ], ) eval_manifest = _write_recognizer_manifest( tmp_path, "eval.jsonl", [("ev-mix", "बिल No", ["mixed_script", "hard_eval"])], ) audit = audit_recognizer_corpus( train, eval_manifest, min_train_samples=2, min_eval_samples=1, min_train_english=2, min_train_devanagari=2, min_train_mixed=2, min_eval_english=1, min_eval_devanagari=1, min_eval_mixed=1, min_train_latin_only=1, min_train_devanagari_only=1, min_eval_latin_only=1, min_eval_devanagari_only=1, min_train_real=1, min_train_synthetic=1, ) assert not audit.passed assert "train latin-only samples 0 < required 1" in audit.issues assert "train devanagari-only samples 0 < required 1" in audit.issues markdown_dir = tmp_path / "audit" audit = audit_recognizer_corpus( train, eval_manifest, markdown_dir, min_train_samples=2, min_eval_samples=1, min_train_english=2, min_train_devanagari=2, min_train_mixed=2, min_eval_english=1, min_eval_devanagari=1, min_eval_mixed=1, min_train_latin_only=1, min_train_devanagari_only=1, min_eval_latin_only=1, min_eval_devanagari_only=1, min_train_real=1, min_train_synthetic=1, ) markdown = (markdown_dir / "recognizer-corpus-audit.md").read_text(encoding="utf-8") assert "add Latin-only English samples to protect English preservation" in markdown assert "add Devanagari-only Nepali samples to prevent mixed-script-only overfitting" in markdown def test_build_recognizer_corpus_upsamples_script_and_source_buckets(tmp_path: Path) -> None: source = _write_recognizer_manifest( tmp_path, "source.jsonl", [ ("en", "Invoice total", ["english", "real"]), ("ne", "नेपाल सरकार", ["nepali", "real"]), ("mix", "नाम Ram", ["mixed_script", "synthetic"]), ], ) summary = build_recognizer_corpus( [source], tmp_path / "built.jsonl", target_latin_only=2, target_devanagari_only=2, target_mixed=2, target_real=4, target_synthetic=2, seed=3, ) built = load_manifest(summary.output_manifest) assert summary.source_count == 3 assert summary.output_count >= 6 assert summary.bucket_counts_after["latin_only"] >= 2 assert summary.bucket_counts_after["devanagari_only"] >= 2 assert summary.bucket_counts_after["mixed_devanagari_latin"] >= 2 assert summary.bucket_counts_after["real"] >= 4 assert summary.bucket_counts_after["synthetic"] >= 2 assert any(entry.metadata.get("recognizer_corpus_duplicate") for entry in built) assert (tmp_path / "built-recognizer-corpus-build.json").exists() assert (tmp_path / "built-recognizer-corpus-build.md").exists() def test_build_recognizer_corpus_fails_when_required_bucket_is_missing(tmp_path: Path) -> None: source = _write_recognizer_manifest( tmp_path, "source.jsonl", [("ne", "नेपाल सरकार", ["nepali", "real"])], ) with pytest.raises(DataValidationError, match="missing bucket: latin_only"): build_recognizer_corpus([source], tmp_path / "built.jsonl", target_latin_only=1) def test_filter_manifest_selects_by_sample_id_and_metadata(tmp_path: Path) -> None: manifest = tmp_path / "combined.jsonl" write_manifest(_entries(tmp_path, start=0, count=6), manifest) summary = filter_manifest( manifest, tmp_path / "filtered.jsonl", sample_ids=["sample-0", "sample-3"], document_types=["table"], degradations=["scan"], ) filtered = load_manifest(summary.output_manifest) assert summary.source_count == 6 assert summary.selected_count == 2 assert [entry.sample_id for entry in filtered] == ["sample-0", "sample-3"] assert (tmp_path / "filtered-filter.json").exists() assert (tmp_path / "filtered-filter.md").exists() def test_filter_manifest_requires_existing_sample_ids(tmp_path: Path) -> None: manifest = tmp_path / "combined.jsonl" write_manifest(_entries(tmp_path, start=0, count=4), manifest) with pytest.raises(DataValidationError, match="requested sample_ids not found"): filter_manifest(manifest, tmp_path / "filtered.jsonl", sample_ids=["missing-sample"]) def _write_combined_manifest(tmp_path: Path, *, count: int) -> Path: manifest = tmp_path / "combined.jsonl" write_manifest(_entries(tmp_path, start=0, count=count), manifest) return manifest def _write_recognizer_manifest(tmp_path: Path, name: str, rows: list[tuple[str, str, list[str]]]) -> Path: manifest = tmp_path / name entries: list[ManifestEntry] = [] for sample_id, text, slices in rows: image = tmp_path / f"{name}-{sample_id}.png" image.write_bytes(f"image-{name}-{sample_id}".encode("utf-8")) image_hash = sha256_file(image) text_hash = sha256_text(text) entries.append( ManifestEntry( sample_id=sample_id, dataset="fixture", split="train" if name.startswith("train") else "eval", image_path=str(image), text=text, sha256=image_hash, metadata={"slices": slices, "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, ) ) write_manifest(entries, manifest) return manifest def _entries(tmp_path: Path, *, start: int, count: int) -> list[ManifestEntry]: entries: list[ManifestEntry] = [] for index in range(start, start + count): image = tmp_path / f"line-{index}.png" image.write_bytes(f"image-{index}".encode("utf-8")) image_hash = sha256_file(image) text = "नेपाल" if index % 2 == 0 else "English" text_hash = sha256_text(text) slices = ["nepali"] if index % 2 == 0 else ["english"] document_type = "table" if index % 3 == 0 else "paragraph" degradation = "scan" if index % 3 == 0 else "clean" if document_type == "table": slices.append("table") entries.append( ManifestEntry( sample_id=f"sample-{index}", dataset="local", split="all", image_path=str(image), text=text, sha256=image_hash, metadata={ "slices": slices, "document_type": document_type, "degradation": degradation, "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), }, ) ) return entries