Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_datasets.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 19.4 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_datasets.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_datasets.py
-
curl -L -o test_datasets.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_datasets.py
19.4 kB
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import pytest | |
| from ocrtech.datasets import audit_manifest, audit_recognizer_corpus, build_recognizer_corpus, filter_manifest, merge_manifests, rebalance_manifest, split_manifest | |
| from ocrtech.errors import DataValidationError | |
| from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_path, sha256_text, write_manifest | |
| from ocrtech.validation import check_leakage | |
| def test_audit_manifest_reports_counts_and_missing_slice(tmp_path: Path) -> None: | |
| manifest = _write_combined_manifest(tmp_path, count=4) | |
| audit = audit_manifest(manifest, tmp_path / "audit", require_slices=["nepali", "table", "missing"]) | |
| assert not audit.passed | |
| assert audit.sample_count == 4 | |
| assert audit.slice_counts["nepali"] == 2 | |
| assert "required slice missing: missing" in audit.issues | |
| assert (tmp_path / "audit" / "manifest-audit.json").exists() | |
| def test_split_manifest_is_deterministic_and_leakage_safe(tmp_path: Path) -> None: | |
| manifest = _write_combined_manifest(tmp_path, count=10) | |
| first = split_manifest(manifest, tmp_path / "split-a", eval_ratio=0.3, seed=7) | |
| second = split_manifest(manifest, tmp_path / "split-b", eval_ratio=0.3, seed=7) | |
| first_eval_ids = [entry.sample_id for entry in load_manifest(first.eval_manifest)] | |
| second_eval_ids = [entry.sample_id for entry in load_manifest(second.eval_manifest)] | |
| assert first_eval_ids == second_eval_ids | |
| assert first.train_count + first.eval_count == 10 | |
| assert check_leakage([first.train_manifest], first.eval_manifest).passed | |
| def test_split_manifest_keeps_duplicate_sample_signature_in_same_split(tmp_path: Path) -> None: | |
| image = tmp_path / "shared.png" | |
| image.write_bytes(b"same-image") | |
| image_hash = sha256_file(image) | |
| text_hash = sha256_text("same text") | |
| sample_hash = sha256_text(f"{image_hash}\n{text_hash}") | |
| manifest = tmp_path / "combined.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="sample-a", | |
| dataset="local", | |
| split="all", | |
| image_path=str(image), | |
| text="same text", | |
| sha256=image_hash, | |
| metadata={"slices": ["nepali"], "text_sha256": text_hash, "sample_sha256": sample_hash}, | |
| ), | |
| ManifestEntry( | |
| sample_id="sample-b", | |
| dataset="local", | |
| split="all", | |
| image_path=str(image), | |
| text="same text", | |
| sha256=image_hash, | |
| metadata={"slices": ["nepali"], "text_sha256": text_hash, "sample_sha256": sample_hash}, | |
| ), | |
| *_entries(tmp_path, start=2, count=4), | |
| ], | |
| manifest, | |
| ) | |
| summary = split_manifest(manifest, tmp_path / "split", eval_ratio=0.5, seed=3) | |
| train_ids = {entry.sample_id for entry in load_manifest(summary.train_manifest)} | |
| eval_ids = {entry.sample_id for entry in load_manifest(summary.eval_manifest)} | |
| assert {"sample-a", "sample-b"}.issubset(train_ids) or {"sample-a", "sample-b"}.issubset(eval_ids) | |
| assert check_leakage([summary.train_manifest], summary.eval_manifest).passed | |
| def test_split_manifest_can_group_by_text_sha256(tmp_path: Path) -> None: | |
| manifest = tmp_path / "combined.jsonl" | |
| entries = [] | |
| for index in range(6): | |
| text = "shared text" if index < 3 else f"text {index}" | |
| image = tmp_path / f"sample-{index}.png" | |
| image.write_bytes(f"image-{index}".encode("utf-8")) | |
| text_hash = sha256_text(text) | |
| entries.append( | |
| ManifestEntry( | |
| sample_id=f"sample-{index}", | |
| dataset="local", | |
| split="all", | |
| image_path=str(image), | |
| text=text, | |
| sha256=sha256_file(image), | |
| metadata={"slices": ["nepali"], "text_sha256": text_hash}, | |
| ) | |
| ) | |
| write_manifest(entries, manifest) | |
| summary = split_manifest(manifest, tmp_path / "split", eval_ratio=0.5, seed=1, group_by="text_sha256") | |
| train_entries = load_manifest(summary.train_manifest) | |
| eval_entries = load_manifest(summary.eval_manifest) | |
| train_shared = {entry.sample_id for entry in train_entries if entry.text == "shared text"} | |
| eval_shared = {entry.sample_id for entry in eval_entries if entry.text == "shared text"} | |
| assert summary.group_by == "text_sha256" | |
| assert not (train_shared and eval_shared) | |
| def test_audit_manifest_fails_on_hash_mismatch(tmp_path: Path) -> None: | |
| image = tmp_path / "line.png" | |
| image.write_bytes(b"actual") | |
| manifest = tmp_path / "manifest.jsonl" | |
| manifest.write_text( | |
| json.dumps( | |
| { | |
| "sample_id": "bad", | |
| "dataset": "local", | |
| "split": "train", | |
| "image_path": str(image), | |
| "text": "नेपाल", | |
| "sha256": "not-the-real-hash", | |
| "metadata": {"slices": ["nepali"]}, | |
| }, | |
| ensure_ascii=False, | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| audit = audit_manifest(manifest) | |
| assert not audit.passed | |
| assert any("sha256 mismatch" in issue for issue in audit.issues) | |
| def test_audit_manifest_supports_directory_inputs(tmp_path: Path) -> None: | |
| bundle = tmp_path / "bundle" | |
| bundle.mkdir() | |
| (bundle / "page-0001.png").write_bytes(b"page-1") | |
| (bundle / "page-0002.png").write_bytes(b"page-2") | |
| manifest = tmp_path / "manifest.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="bundle-sample", | |
| dataset="local", | |
| split="eval", | |
| image_path=str(bundle), | |
| text="नेपाल सरकार\nApproved by: Sita Rana", | |
| sha256=sha256_path(bundle), | |
| metadata={"slices": ["nepali", "english", "multi_page"], "text_sha256": sha256_text("नेपाल सरकार\nApproved by: Sita Rana")}, | |
| ) | |
| ], | |
| manifest, | |
| ) | |
| audit = audit_manifest(manifest) | |
| assert audit.passed | |
| assert audit.sample_count == 1 | |
| def test_merge_manifests_combines_inputs_and_writes_summary(tmp_path: Path) -> None: | |
| first = tmp_path / "first.jsonl" | |
| second = tmp_path / "second.jsonl" | |
| write_manifest(_entries(tmp_path, start=0, count=2), first) | |
| write_manifest(_entries(tmp_path, start=2, count=2), second) | |
| summary = merge_manifests([first, second], tmp_path / "combined.jsonl") | |
| merged = load_manifest(summary.output_manifest) | |
| assert len(merged) == 4 | |
| assert summary.sample_count == 4 | |
| assert (tmp_path / "combined-merge.json").exists() | |
| assert (tmp_path / "combined-merge.md").exists() | |
| def test_rebalance_manifest_upsamples_requested_slice_and_writes_summary(tmp_path: Path) -> None: | |
| manifest = _write_combined_manifest(tmp_path, count=6) | |
| summary = rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["english", "nepali"], seed=5) | |
| rebalanced = load_manifest(summary.output_manifest) | |
| assert summary.original_count == 6 | |
| assert summary.target_count == 3 | |
| assert summary.slice_counts_before == {"english": 3, "nepali": 3} | |
| assert summary.slice_counts_after == {"english": 3, "nepali": 3} | |
| assert len(rebalanced) == 6 | |
| assert (tmp_path / "rebalanced-rebalance.json").exists() | |
| assert (tmp_path / "rebalanced-rebalance.md").exists() | |
| def test_rebalance_manifest_can_fill_underrepresented_slice(tmp_path: Path) -> None: | |
| manifest = tmp_path / "combined.jsonl" | |
| write_manifest(_entries(tmp_path, start=0, count=5), manifest) | |
| summary = rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["english", "nepali"], target_count=4, seed=7) | |
| rebalanced = load_manifest(summary.output_manifest) | |
| duplicated = [entry for entry in rebalanced if entry.metadata.get("rebalance_duplicate")] | |
| assert summary.original_count == 5 | |
| assert summary.rebalanced_count == 8 | |
| assert summary.slice_counts_before == {"english": 2, "nepali": 3} | |
| assert summary.slice_counts_after == {"english": 4, "nepali": 4} | |
| assert len(duplicated) == 3 | |
| assert all(entry.sample_id.startswith("sample-") and "~rebalance-english-" in entry.sample_id for entry in duplicated[:2]) | |
| assert all(entry.metadata["rebalance_source_sample_id"].startswith("sample-") for entry in duplicated) | |
| def test_rebalance_manifest_requires_present_slice(tmp_path: Path) -> None: | |
| manifest = _write_combined_manifest(tmp_path, count=4) | |
| with pytest.raises(DataValidationError, match="missing requested slices: missing"): | |
| rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["missing"], target_count=5) | |
| def test_audit_recognizer_corpus_passes_balanced_train_eval_manifests(tmp_path: Path) -> None: | |
| train = _write_recognizer_manifest( | |
| tmp_path, | |
| "train.jsonl", | |
| [ | |
| ("tr-en-1", "Invoice total", ["english", "real"]), | |
| ("tr-en-2", "Account number", ["english", "synthetic"]), | |
| ("tr-ne-1", "नेपाल सरकार", ["nepali", "real"]), | |
| ("tr-ne-2", "जम्मा रकम", ["nepali", "synthetic"]), | |
| ("tr-mix-1", "नाम Sita", ["mixed_script", "real"]), | |
| ("tr-mix-2", "रकम USD", ["mixed_script", "synthetic"]), | |
| ], | |
| ) | |
| eval_manifest = _write_recognizer_manifest( | |
| tmp_path, | |
| "eval.jsonl", | |
| [ | |
| ("ev-en", "Receipt date", ["english", "hard_eval"]), | |
| ("ev-ne", "प्रदेश कार्यालय", ["nepali", "hard_eval"]), | |
| ("ev-mix", "बिल No 7", ["mixed_script", "hard_eval"]), | |
| ], | |
| ) | |
| audit = audit_recognizer_corpus( | |
| train, | |
| eval_manifest, | |
| tmp_path / "audit", | |
| min_train_samples=6, | |
| min_eval_samples=3, | |
| min_train_english=3, | |
| min_train_devanagari=4, | |
| min_train_mixed=2, | |
| min_eval_english=2, | |
| min_eval_devanagari=2, | |
| min_eval_mixed=1, | |
| min_train_latin_only=2, | |
| min_train_devanagari_only=2, | |
| min_eval_latin_only=1, | |
| min_eval_devanagari_only=1, | |
| min_train_real=3, | |
| min_train_synthetic=3, | |
| ) | |
| assert audit.passed | |
| assert audit.leakage_passed | |
| assert audit.train_profile.script_sample_counts["mixed_devanagari_latin"] == 2 | |
| assert (tmp_path / "audit" / "recognizer-corpus-audit.json").exists() | |
| assert (tmp_path / "audit" / "recognizer-corpus-audit.md").exists() | |
| def test_audit_recognizer_corpus_fails_missing_english_and_leakage(tmp_path: Path) -> None: | |
| train = _write_recognizer_manifest( | |
| tmp_path, | |
| "train.jsonl", | |
| [ | |
| ("shared", "नेपाल सरकार", ["nepali", "real"]), | |
| ("tr-ne-2", "जम्मा रकम", ["nepali", "synthetic"]), | |
| ], | |
| ) | |
| eval_manifest = _write_recognizer_manifest( | |
| tmp_path, | |
| "eval.jsonl", | |
| [ | |
| ("shared", "नेपाल सरकार", ["nepali", "hard_eval"]), | |
| ("ev-ne-2", "प्रदेश कार्यालय", ["nepali", "hard_eval"]), | |
| ], | |
| ) | |
| audit = audit_recognizer_corpus( | |
| train, | |
| eval_manifest, | |
| min_train_samples=2, | |
| min_eval_samples=2, | |
| min_train_english=1, | |
| min_train_devanagari=2, | |
| min_train_mixed=1, | |
| min_eval_english=1, | |
| min_eval_devanagari=2, | |
| min_eval_mixed=0, | |
| min_train_latin_only=1, | |
| min_train_devanagari_only=1, | |
| min_eval_latin_only=1, | |
| min_eval_devanagari_only=1, | |
| min_train_real=1, | |
| min_train_synthetic=1, | |
| ) | |
| assert not audit.passed | |
| assert not audit.leakage_passed | |
| assert "train latin-script samples 0 < required 1" in audit.issues | |
| assert "train mixed-script samples 0 < required 1" in audit.issues | |
| assert "train latin-only samples 0 < required 1" in audit.issues | |
| assert any(issue.startswith("train/eval leakage detected") for issue in audit.issues) | |
| def test_audit_recognizer_corpus_can_require_isolated_script_samples(tmp_path: Path) -> None: | |
| train = _write_recognizer_manifest( | |
| tmp_path, | |
| "train.jsonl", | |
| [ | |
| ("tr-mix-1", "नाम Ram", ["mixed_script", "real"]), | |
| ("tr-mix-2", "रकम USD", ["mixed_script", "synthetic"]), | |
| ], | |
| ) | |
| eval_manifest = _write_recognizer_manifest( | |
| tmp_path, | |
| "eval.jsonl", | |
| [("ev-mix", "बिल No", ["mixed_script", "hard_eval"])], | |
| ) | |
| audit = audit_recognizer_corpus( | |
| train, | |
| eval_manifest, | |
| min_train_samples=2, | |
| min_eval_samples=1, | |
| min_train_english=2, | |
| min_train_devanagari=2, | |
| min_train_mixed=2, | |
| min_eval_english=1, | |
| min_eval_devanagari=1, | |
| min_eval_mixed=1, | |
| min_train_latin_only=1, | |
| min_train_devanagari_only=1, | |
| min_eval_latin_only=1, | |
| min_eval_devanagari_only=1, | |
| min_train_real=1, | |
| min_train_synthetic=1, | |
| ) | |
| assert not audit.passed | |
| assert "train latin-only samples 0 < required 1" in audit.issues | |
| assert "train devanagari-only samples 0 < required 1" in audit.issues | |
| markdown_dir = tmp_path / "audit" | |
| audit = audit_recognizer_corpus( | |
| train, | |
| eval_manifest, | |
| markdown_dir, | |
| min_train_samples=2, | |
| min_eval_samples=1, | |
| min_train_english=2, | |
| min_train_devanagari=2, | |
| min_train_mixed=2, | |
| min_eval_english=1, | |
| min_eval_devanagari=1, | |
| min_eval_mixed=1, | |
| min_train_latin_only=1, | |
| min_train_devanagari_only=1, | |
| min_eval_latin_only=1, | |
| min_eval_devanagari_only=1, | |
| min_train_real=1, | |
| min_train_synthetic=1, | |
| ) | |
| markdown = (markdown_dir / "recognizer-corpus-audit.md").read_text(encoding="utf-8") | |
| assert "add Latin-only English samples to protect English preservation" in markdown | |
| assert "add Devanagari-only Nepali samples to prevent mixed-script-only overfitting" in markdown | |
| def test_build_recognizer_corpus_upsamples_script_and_source_buckets(tmp_path: Path) -> None: | |
| source = _write_recognizer_manifest( | |
| tmp_path, | |
| "source.jsonl", | |
| [ | |
| ("en", "Invoice total", ["english", "real"]), | |
| ("ne", "नेपाल सरकार", ["nepali", "real"]), | |
| ("mix", "नाम Ram", ["mixed_script", "synthetic"]), | |
| ], | |
| ) | |
| summary = build_recognizer_corpus( | |
| [source], | |
| tmp_path / "built.jsonl", | |
| target_latin_only=2, | |
| target_devanagari_only=2, | |
| target_mixed=2, | |
| target_real=4, | |
| target_synthetic=2, | |
| seed=3, | |
| ) | |
| built = load_manifest(summary.output_manifest) | |
| assert summary.source_count == 3 | |
| assert summary.output_count >= 6 | |
| assert summary.bucket_counts_after["latin_only"] >= 2 | |
| assert summary.bucket_counts_after["devanagari_only"] >= 2 | |
| assert summary.bucket_counts_after["mixed_devanagari_latin"] >= 2 | |
| assert summary.bucket_counts_after["real"] >= 4 | |
| assert summary.bucket_counts_after["synthetic"] >= 2 | |
| assert any(entry.metadata.get("recognizer_corpus_duplicate") for entry in built) | |
| assert (tmp_path / "built-recognizer-corpus-build.json").exists() | |
| assert (tmp_path / "built-recognizer-corpus-build.md").exists() | |
| def test_build_recognizer_corpus_fails_when_required_bucket_is_missing(tmp_path: Path) -> None: | |
| source = _write_recognizer_manifest( | |
| tmp_path, | |
| "source.jsonl", | |
| [("ne", "नेपाल सरकार", ["nepali", "real"])], | |
| ) | |
| with pytest.raises(DataValidationError, match="missing bucket: latin_only"): | |
| build_recognizer_corpus([source], tmp_path / "built.jsonl", target_latin_only=1) | |
| def test_filter_manifest_selects_by_sample_id_and_metadata(tmp_path: Path) -> None: | |
| manifest = tmp_path / "combined.jsonl" | |
| write_manifest(_entries(tmp_path, start=0, count=6), manifest) | |
| summary = filter_manifest( | |
| manifest, | |
| tmp_path / "filtered.jsonl", | |
| sample_ids=["sample-0", "sample-3"], | |
| document_types=["table"], | |
| degradations=["scan"], | |
| ) | |
| filtered = load_manifest(summary.output_manifest) | |
| assert summary.source_count == 6 | |
| assert summary.selected_count == 2 | |
| assert [entry.sample_id for entry in filtered] == ["sample-0", "sample-3"] | |
| assert (tmp_path / "filtered-filter.json").exists() | |
| assert (tmp_path / "filtered-filter.md").exists() | |
| def test_filter_manifest_requires_existing_sample_ids(tmp_path: Path) -> None: | |
| manifest = tmp_path / "combined.jsonl" | |
| write_manifest(_entries(tmp_path, start=0, count=4), manifest) | |
| with pytest.raises(DataValidationError, match="requested sample_ids not found"): | |
| filter_manifest(manifest, tmp_path / "filtered.jsonl", sample_ids=["missing-sample"]) | |
| def _write_combined_manifest(tmp_path: Path, *, count: int) -> Path: | |
| manifest = tmp_path / "combined.jsonl" | |
| write_manifest(_entries(tmp_path, start=0, count=count), manifest) | |
| return manifest | |
| def _write_recognizer_manifest(tmp_path: Path, name: str, rows: list[tuple[str, str, list[str]]]) -> Path: | |
| manifest = tmp_path / name | |
| entries: list[ManifestEntry] = [] | |
| for sample_id, text, slices in rows: | |
| image = tmp_path / f"{name}-{sample_id}.png" | |
| image.write_bytes(f"image-{name}-{sample_id}".encode("utf-8")) | |
| image_hash = sha256_file(image) | |
| text_hash = sha256_text(text) | |
| entries.append( | |
| ManifestEntry( | |
| sample_id=sample_id, | |
| dataset="fixture", | |
| split="train" if name.startswith("train") else "eval", | |
| image_path=str(image), | |
| text=text, | |
| sha256=image_hash, | |
| metadata={"slices": slices, "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")}, | |
| ) | |
| ) | |
| write_manifest(entries, manifest) | |
| return manifest | |
| def _entries(tmp_path: Path, *, start: int, count: int) -> list[ManifestEntry]: | |
| entries: list[ManifestEntry] = [] | |
| for index in range(start, start + count): | |
| image = tmp_path / f"line-{index}.png" | |
| image.write_bytes(f"image-{index}".encode("utf-8")) | |
| image_hash = sha256_file(image) | |
| text = "नेपाल" if index % 2 == 0 else "English" | |
| text_hash = sha256_text(text) | |
| slices = ["nepali"] if index % 2 == 0 else ["english"] | |
| document_type = "table" if index % 3 == 0 else "paragraph" | |
| degradation = "scan" if index % 3 == 0 else "clean" | |
| if document_type == "table": | |
| slices.append("table") | |
| entries.append( | |
| ManifestEntry( | |
| sample_id=f"sample-{index}", | |
| dataset="local", | |
| split="all", | |
| image_path=str(image), | |
| text=text, | |
| sha256=image_hash, | |
| metadata={ | |
| "slices": slices, | |
| "document_type": document_type, | |
| "degradation": degradation, | |
| "text_sha256": text_hash, | |
| "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"), | |
| }, | |
| ) | |
| ) | |
| return entries | |