ocr-tech-k2 / tests /test_datasets.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
19.4 kB
from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech.datasets import audit_manifest, audit_recognizer_corpus, build_recognizer_corpus, filter_manifest, merge_manifests, rebalance_manifest, split_manifest
from ocrtech.errors import DataValidationError
from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, sha256_path, sha256_text, write_manifest
from ocrtech.validation import check_leakage
def test_audit_manifest_reports_counts_and_missing_slice(tmp_path: Path) -> None:
manifest = _write_combined_manifest(tmp_path, count=4)
audit = audit_manifest(manifest, tmp_path / "audit", require_slices=["nepali", "table", "missing"])
assert not audit.passed
assert audit.sample_count == 4
assert audit.slice_counts["nepali"] == 2
assert "required slice missing: missing" in audit.issues
assert (tmp_path / "audit" / "manifest-audit.json").exists()
def test_split_manifest_is_deterministic_and_leakage_safe(tmp_path: Path) -> None:
manifest = _write_combined_manifest(tmp_path, count=10)
first = split_manifest(manifest, tmp_path / "split-a", eval_ratio=0.3, seed=7)
second = split_manifest(manifest, tmp_path / "split-b", eval_ratio=0.3, seed=7)
first_eval_ids = [entry.sample_id for entry in load_manifest(first.eval_manifest)]
second_eval_ids = [entry.sample_id for entry in load_manifest(second.eval_manifest)]
assert first_eval_ids == second_eval_ids
assert first.train_count + first.eval_count == 10
assert check_leakage([first.train_manifest], first.eval_manifest).passed
def test_split_manifest_keeps_duplicate_sample_signature_in_same_split(tmp_path: Path) -> None:
image = tmp_path / "shared.png"
image.write_bytes(b"same-image")
image_hash = sha256_file(image)
text_hash = sha256_text("same text")
sample_hash = sha256_text(f"{image_hash}\n{text_hash}")
manifest = tmp_path / "combined.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="sample-a",
dataset="local",
split="all",
image_path=str(image),
text="same text",
sha256=image_hash,
metadata={"slices": ["nepali"], "text_sha256": text_hash, "sample_sha256": sample_hash},
),
ManifestEntry(
sample_id="sample-b",
dataset="local",
split="all",
image_path=str(image),
text="same text",
sha256=image_hash,
metadata={"slices": ["nepali"], "text_sha256": text_hash, "sample_sha256": sample_hash},
),
*_entries(tmp_path, start=2, count=4),
],
manifest,
)
summary = split_manifest(manifest, tmp_path / "split", eval_ratio=0.5, seed=3)
train_ids = {entry.sample_id for entry in load_manifest(summary.train_manifest)}
eval_ids = {entry.sample_id for entry in load_manifest(summary.eval_manifest)}
assert {"sample-a", "sample-b"}.issubset(train_ids) or {"sample-a", "sample-b"}.issubset(eval_ids)
assert check_leakage([summary.train_manifest], summary.eval_manifest).passed
def test_split_manifest_can_group_by_text_sha256(tmp_path: Path) -> None:
manifest = tmp_path / "combined.jsonl"
entries = []
for index in range(6):
text = "shared text" if index < 3 else f"text {index}"
image = tmp_path / f"sample-{index}.png"
image.write_bytes(f"image-{index}".encode("utf-8"))
text_hash = sha256_text(text)
entries.append(
ManifestEntry(
sample_id=f"sample-{index}",
dataset="local",
split="all",
image_path=str(image),
text=text,
sha256=sha256_file(image),
metadata={"slices": ["nepali"], "text_sha256": text_hash},
)
)
write_manifest(entries, manifest)
summary = split_manifest(manifest, tmp_path / "split", eval_ratio=0.5, seed=1, group_by="text_sha256")
train_entries = load_manifest(summary.train_manifest)
eval_entries = load_manifest(summary.eval_manifest)
train_shared = {entry.sample_id for entry in train_entries if entry.text == "shared text"}
eval_shared = {entry.sample_id for entry in eval_entries if entry.text == "shared text"}
assert summary.group_by == "text_sha256"
assert not (train_shared and eval_shared)
def test_audit_manifest_fails_on_hash_mismatch(tmp_path: Path) -> None:
image = tmp_path / "line.png"
image.write_bytes(b"actual")
manifest = tmp_path / "manifest.jsonl"
manifest.write_text(
json.dumps(
{
"sample_id": "bad",
"dataset": "local",
"split": "train",
"image_path": str(image),
"text": "नेपाल",
"sha256": "not-the-real-hash",
"metadata": {"slices": ["nepali"]},
},
ensure_ascii=False,
)
+ "\n",
encoding="utf-8",
)
audit = audit_manifest(manifest)
assert not audit.passed
assert any("sha256 mismatch" in issue for issue in audit.issues)
def test_audit_manifest_supports_directory_inputs(tmp_path: Path) -> None:
bundle = tmp_path / "bundle"
bundle.mkdir()
(bundle / "page-0001.png").write_bytes(b"page-1")
(bundle / "page-0002.png").write_bytes(b"page-2")
manifest = tmp_path / "manifest.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="bundle-sample",
dataset="local",
split="eval",
image_path=str(bundle),
text="नेपाल सरकार\nApproved by: Sita Rana",
sha256=sha256_path(bundle),
metadata={"slices": ["nepali", "english", "multi_page"], "text_sha256": sha256_text("नेपाल सरकार\nApproved by: Sita Rana")},
)
],
manifest,
)
audit = audit_manifest(manifest)
assert audit.passed
assert audit.sample_count == 1
def test_merge_manifests_combines_inputs_and_writes_summary(tmp_path: Path) -> None:
first = tmp_path / "first.jsonl"
second = tmp_path / "second.jsonl"
write_manifest(_entries(tmp_path, start=0, count=2), first)
write_manifest(_entries(tmp_path, start=2, count=2), second)
summary = merge_manifests([first, second], tmp_path / "combined.jsonl")
merged = load_manifest(summary.output_manifest)
assert len(merged) == 4
assert summary.sample_count == 4
assert (tmp_path / "combined-merge.json").exists()
assert (tmp_path / "combined-merge.md").exists()
def test_rebalance_manifest_upsamples_requested_slice_and_writes_summary(tmp_path: Path) -> None:
manifest = _write_combined_manifest(tmp_path, count=6)
summary = rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["english", "nepali"], seed=5)
rebalanced = load_manifest(summary.output_manifest)
assert summary.original_count == 6
assert summary.target_count == 3
assert summary.slice_counts_before == {"english": 3, "nepali": 3}
assert summary.slice_counts_after == {"english": 3, "nepali": 3}
assert len(rebalanced) == 6
assert (tmp_path / "rebalanced-rebalance.json").exists()
assert (tmp_path / "rebalanced-rebalance.md").exists()
def test_rebalance_manifest_can_fill_underrepresented_slice(tmp_path: Path) -> None:
manifest = tmp_path / "combined.jsonl"
write_manifest(_entries(tmp_path, start=0, count=5), manifest)
summary = rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["english", "nepali"], target_count=4, seed=7)
rebalanced = load_manifest(summary.output_manifest)
duplicated = [entry for entry in rebalanced if entry.metadata.get("rebalance_duplicate")]
assert summary.original_count == 5
assert summary.rebalanced_count == 8
assert summary.slice_counts_before == {"english": 2, "nepali": 3}
assert summary.slice_counts_after == {"english": 4, "nepali": 4}
assert len(duplicated) == 3
assert all(entry.sample_id.startswith("sample-") and "~rebalance-english-" in entry.sample_id for entry in duplicated[:2])
assert all(entry.metadata["rebalance_source_sample_id"].startswith("sample-") for entry in duplicated)
def test_rebalance_manifest_requires_present_slice(tmp_path: Path) -> None:
manifest = _write_combined_manifest(tmp_path, count=4)
with pytest.raises(DataValidationError, match="missing requested slices: missing"):
rebalance_manifest(manifest, tmp_path / "rebalanced.jsonl", slices=["missing"], target_count=5)
def test_audit_recognizer_corpus_passes_balanced_train_eval_manifests(tmp_path: Path) -> None:
train = _write_recognizer_manifest(
tmp_path,
"train.jsonl",
[
("tr-en-1", "Invoice total", ["english", "real"]),
("tr-en-2", "Account number", ["english", "synthetic"]),
("tr-ne-1", "नेपाल सरकार", ["nepali", "real"]),
("tr-ne-2", "जम्मा रकम", ["nepali", "synthetic"]),
("tr-mix-1", "नाम Sita", ["mixed_script", "real"]),
("tr-mix-2", "रकम USD", ["mixed_script", "synthetic"]),
],
)
eval_manifest = _write_recognizer_manifest(
tmp_path,
"eval.jsonl",
[
("ev-en", "Receipt date", ["english", "hard_eval"]),
("ev-ne", "प्रदेश कार्यालय", ["nepali", "hard_eval"]),
("ev-mix", "बिल No 7", ["mixed_script", "hard_eval"]),
],
)
audit = audit_recognizer_corpus(
train,
eval_manifest,
tmp_path / "audit",
min_train_samples=6,
min_eval_samples=3,
min_train_english=3,
min_train_devanagari=4,
min_train_mixed=2,
min_eval_english=2,
min_eval_devanagari=2,
min_eval_mixed=1,
min_train_latin_only=2,
min_train_devanagari_only=2,
min_eval_latin_only=1,
min_eval_devanagari_only=1,
min_train_real=3,
min_train_synthetic=3,
)
assert audit.passed
assert audit.leakage_passed
assert audit.train_profile.script_sample_counts["mixed_devanagari_latin"] == 2
assert (tmp_path / "audit" / "recognizer-corpus-audit.json").exists()
assert (tmp_path / "audit" / "recognizer-corpus-audit.md").exists()
def test_audit_recognizer_corpus_fails_missing_english_and_leakage(tmp_path: Path) -> None:
train = _write_recognizer_manifest(
tmp_path,
"train.jsonl",
[
("shared", "नेपाल सरकार", ["nepali", "real"]),
("tr-ne-2", "जम्मा रकम", ["nepali", "synthetic"]),
],
)
eval_manifest = _write_recognizer_manifest(
tmp_path,
"eval.jsonl",
[
("shared", "नेपाल सरकार", ["nepali", "hard_eval"]),
("ev-ne-2", "प्रदेश कार्यालय", ["nepali", "hard_eval"]),
],
)
audit = audit_recognizer_corpus(
train,
eval_manifest,
min_train_samples=2,
min_eval_samples=2,
min_train_english=1,
min_train_devanagari=2,
min_train_mixed=1,
min_eval_english=1,
min_eval_devanagari=2,
min_eval_mixed=0,
min_train_latin_only=1,
min_train_devanagari_only=1,
min_eval_latin_only=1,
min_eval_devanagari_only=1,
min_train_real=1,
min_train_synthetic=1,
)
assert not audit.passed
assert not audit.leakage_passed
assert "train latin-script samples 0 < required 1" in audit.issues
assert "train mixed-script samples 0 < required 1" in audit.issues
assert "train latin-only samples 0 < required 1" in audit.issues
assert any(issue.startswith("train/eval leakage detected") for issue in audit.issues)
def test_audit_recognizer_corpus_can_require_isolated_script_samples(tmp_path: Path) -> None:
train = _write_recognizer_manifest(
tmp_path,
"train.jsonl",
[
("tr-mix-1", "नाम Ram", ["mixed_script", "real"]),
("tr-mix-2", "रकम USD", ["mixed_script", "synthetic"]),
],
)
eval_manifest = _write_recognizer_manifest(
tmp_path,
"eval.jsonl",
[("ev-mix", "बिल No", ["mixed_script", "hard_eval"])],
)
audit = audit_recognizer_corpus(
train,
eval_manifest,
min_train_samples=2,
min_eval_samples=1,
min_train_english=2,
min_train_devanagari=2,
min_train_mixed=2,
min_eval_english=1,
min_eval_devanagari=1,
min_eval_mixed=1,
min_train_latin_only=1,
min_train_devanagari_only=1,
min_eval_latin_only=1,
min_eval_devanagari_only=1,
min_train_real=1,
min_train_synthetic=1,
)
assert not audit.passed
assert "train latin-only samples 0 < required 1" in audit.issues
assert "train devanagari-only samples 0 < required 1" in audit.issues
markdown_dir = tmp_path / "audit"
audit = audit_recognizer_corpus(
train,
eval_manifest,
markdown_dir,
min_train_samples=2,
min_eval_samples=1,
min_train_english=2,
min_train_devanagari=2,
min_train_mixed=2,
min_eval_english=1,
min_eval_devanagari=1,
min_eval_mixed=1,
min_train_latin_only=1,
min_train_devanagari_only=1,
min_eval_latin_only=1,
min_eval_devanagari_only=1,
min_train_real=1,
min_train_synthetic=1,
)
markdown = (markdown_dir / "recognizer-corpus-audit.md").read_text(encoding="utf-8")
assert "add Latin-only English samples to protect English preservation" in markdown
assert "add Devanagari-only Nepali samples to prevent mixed-script-only overfitting" in markdown
def test_build_recognizer_corpus_upsamples_script_and_source_buckets(tmp_path: Path) -> None:
source = _write_recognizer_manifest(
tmp_path,
"source.jsonl",
[
("en", "Invoice total", ["english", "real"]),
("ne", "नेपाल सरकार", ["nepali", "real"]),
("mix", "नाम Ram", ["mixed_script", "synthetic"]),
],
)
summary = build_recognizer_corpus(
[source],
tmp_path / "built.jsonl",
target_latin_only=2,
target_devanagari_only=2,
target_mixed=2,
target_real=4,
target_synthetic=2,
seed=3,
)
built = load_manifest(summary.output_manifest)
assert summary.source_count == 3
assert summary.output_count >= 6
assert summary.bucket_counts_after["latin_only"] >= 2
assert summary.bucket_counts_after["devanagari_only"] >= 2
assert summary.bucket_counts_after["mixed_devanagari_latin"] >= 2
assert summary.bucket_counts_after["real"] >= 4
assert summary.bucket_counts_after["synthetic"] >= 2
assert any(entry.metadata.get("recognizer_corpus_duplicate") for entry in built)
assert (tmp_path / "built-recognizer-corpus-build.json").exists()
assert (tmp_path / "built-recognizer-corpus-build.md").exists()
def test_build_recognizer_corpus_fails_when_required_bucket_is_missing(tmp_path: Path) -> None:
source = _write_recognizer_manifest(
tmp_path,
"source.jsonl",
[("ne", "नेपाल सरकार", ["nepali", "real"])],
)
with pytest.raises(DataValidationError, match="missing bucket: latin_only"):
build_recognizer_corpus([source], tmp_path / "built.jsonl", target_latin_only=1)
def test_filter_manifest_selects_by_sample_id_and_metadata(tmp_path: Path) -> None:
manifest = tmp_path / "combined.jsonl"
write_manifest(_entries(tmp_path, start=0, count=6), manifest)
summary = filter_manifest(
manifest,
tmp_path / "filtered.jsonl",
sample_ids=["sample-0", "sample-3"],
document_types=["table"],
degradations=["scan"],
)
filtered = load_manifest(summary.output_manifest)
assert summary.source_count == 6
assert summary.selected_count == 2
assert [entry.sample_id for entry in filtered] == ["sample-0", "sample-3"]
assert (tmp_path / "filtered-filter.json").exists()
assert (tmp_path / "filtered-filter.md").exists()
def test_filter_manifest_requires_existing_sample_ids(tmp_path: Path) -> None:
manifest = tmp_path / "combined.jsonl"
write_manifest(_entries(tmp_path, start=0, count=4), manifest)
with pytest.raises(DataValidationError, match="requested sample_ids not found"):
filter_manifest(manifest, tmp_path / "filtered.jsonl", sample_ids=["missing-sample"])
def _write_combined_manifest(tmp_path: Path, *, count: int) -> Path:
manifest = tmp_path / "combined.jsonl"
write_manifest(_entries(tmp_path, start=0, count=count), manifest)
return manifest
def _write_recognizer_manifest(tmp_path: Path, name: str, rows: list[tuple[str, str, list[str]]]) -> Path:
manifest = tmp_path / name
entries: list[ManifestEntry] = []
for sample_id, text, slices in rows:
image = tmp_path / f"{name}-{sample_id}.png"
image.write_bytes(f"image-{name}-{sample_id}".encode("utf-8"))
image_hash = sha256_file(image)
text_hash = sha256_text(text)
entries.append(
ManifestEntry(
sample_id=sample_id,
dataset="fixture",
split="train" if name.startswith("train") else "eval",
image_path=str(image),
text=text,
sha256=image_hash,
metadata={"slices": slices, "text_sha256": text_hash, "sample_sha256": sha256_text(f"{image_hash}\n{text_hash}")},
)
)
write_manifest(entries, manifest)
return manifest
def _entries(tmp_path: Path, *, start: int, count: int) -> list[ManifestEntry]:
entries: list[ManifestEntry] = []
for index in range(start, start + count):
image = tmp_path / f"line-{index}.png"
image.write_bytes(f"image-{index}".encode("utf-8"))
image_hash = sha256_file(image)
text = "नेपाल" if index % 2 == 0 else "English"
text_hash = sha256_text(text)
slices = ["nepali"] if index % 2 == 0 else ["english"]
document_type = "table" if index % 3 == 0 else "paragraph"
degradation = "scan" if index % 3 == 0 else "clean"
if document_type == "table":
slices.append("table")
entries.append(
ManifestEntry(
sample_id=f"sample-{index}",
dataset="local",
split="all",
image_path=str(image),
text=text,
sha256=image_hash,
metadata={
"slices": slices,
"document_type": document_type,
"degradation": degradation,
"text_sha256": text_hash,
"sample_sha256": sha256_text(f"{image_hash}\n{text_hash}"),
},
)
)
return entries