ocr-projects-archive / ocr-tech /tests /test_audit_newar_devanagari_normalized_pack.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
7.47 kB
from __future__ import annotations
import importlib.util
import json
import shutil
from pathlib import Path
import pytest
from local_artifacts import require_local_artifacts
@pytest.fixture(scope="module")
def tool():
path = Path("tools/audit_newar_devanagari_normalized_pack.py")
spec = importlib.util.spec_from_file_location(
"audit_newar_devanagari_normalized_pack", path
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_independent_normalizer_handles_only_the_two_documented_classes(tool):
normalized, counts = tool.independent_normalize("संस् थापक न ़ दुई शब्द")
assert normalized == "संस्थापक ऩ दुई शब्द"
assert counts == {
"class_a_removals": 1,
"class_b_removals": 1,
"class_a_characters_removed": 2,
"class_b_characters_removed": 2,
}
def test_real_normalized_labels_reproduce_exactly_and_stats_reconcile(tool):
original_dir = Path("outputs/newar-devanagari-silver-pack-v1-recovered")
normalized_dir = Path("outputs/newar-pack-v1-normalized-2026-07-15")
require_local_artifacts(original_dir, normalized_dir, reason="reconciles the local Newar silver packs")
stats = json.loads((normalized_dir / "stats.json").read_text(encoding="utf-8"))
reports, rows = tool.reconcile_labels(original_dir, normalized_dir, stats)
assert {split: len(rows[split]) for split in tool.SPLITS} == {
"train": 4096,
"dev": 1528,
"test": 1139,
}
assert reports["train"]["lines_changed"] == 1622
assert reports["dev"]["lines_changed"] == 700
assert reports["test"]["lines_changed"] == 396
assert all(
report["targeted_artifact_runs_after_normalization"]
== {"class_a_runs": 0, "class_b_runs": 0}
for report in reports.values()
)
def test_changed_stats_are_rejected(tool):
original_dir = Path("outputs/newar-devanagari-silver-pack-v1-recovered")
normalized_dir = Path("outputs/newar-pack-v1-normalized-2026-07-15")
require_local_artifacts(original_dir, normalized_dir, reason="reconciles the local Newar silver packs")
stats = json.loads((normalized_dir / "stats.json").read_text(encoding="utf-8"))
stats["test"]["lines_changed"] += 1
with pytest.raises(ValueError, match="stats.json counts do not reconcile"):
tool.reconcile_labels(original_dir, normalized_dir, stats)
def test_leakage_audit_blocks_exact_and_long_substring_overlap(tool):
def row(text: str):
return {"text": text, "normalized_text_sha256": tool._sha256_text(text)}
split_rows = {
"train": [row("unique"), row("abcdefghijklmnop plus")],
"dev": [row("abcdefghijklmnop other")],
"test": [row("unrelated")],
}
report = tool.audit_leakage(split_rows, 16)
assert report["train_vs_dev_plus_test"]["status"] == "FAIL"
assert report["train_vs_dev_plus_test"][
"substring_overlap_left_rows_including_exact"
] == 1
assert report["dev_vs_test"]["status"] == "PASS"
def test_host_gate_requires_exact_omac_and_memory(tool, monkeypatch):
monkeypatch.setattr(tool.socket, "gethostname", lambda: "wrong-host")
monkeypatch.setattr(tool, "_read_physical_memory", lambda: 64 * 1024**3)
with pytest.raises(RuntimeError, match="host gate failed"):
tool.require_omac()
monkeypatch.setattr(tool.socket, "gethostname", lambda: tool.EXPECTED_HOSTNAME)
monkeypatch.setattr(tool, "_read_physical_memory", lambda: 18 * 1024**3)
with pytest.raises(RuntimeError, match="memory gate failed"):
tool.require_omac()
def _write_tiny_pack(tool, root: Path):
from PIL import Image, ImageDraw
original_dir = root / "original"
normalized_dir = root / "normalized"
normalized_dir.mkdir(parents=True)
source_texts = {
"train": "संस् ता",
"dev": "न ़",
"test": "अलग",
}
stats = {}
for index, split in enumerate(tool.SPLITS, start=1):
image_dir = original_dir / split / "images"
image_dir.mkdir(parents=True)
image = Image.new("L", (20 + index, 10), 255)
draw = ImageDraw.Draw(image)
draw.rectangle((index, 2, index + 3, 7), fill=index * 20)
image.save(image_dir / f"{split}.png")
source = source_texts[split]
normalized, counts = tool.independent_normalize(source)
(original_dir / split / "labels.txt").write_text(
f"images/{split}.png\t{source}\n", encoding="utf-8"
)
(normalized_dir / f"{split}-labels.txt").write_text(
f"images/{split}.png\t{normalized}\n", encoding="utf-8"
)
stats[split] = {
"total_lines": 1,
"lines_changed": int(source != normalized),
**counts,
}
(normalized_dir / "stats.json").write_text(
json.dumps(stats), encoding="utf-8"
)
licensing = root / "DATA_LICENSING.md"
licensing.write_text(
"| Gorkhapatra | unresolved | RISK | RISK | RISK/NO | RISK |\n",
encoding="utf-8",
)
attributions = root / "ATTRIBUTIONS.md"
attributions.write_text("# Attributions\n", encoding="utf-8")
return original_dir, normalized_dir, licensing, attributions
def test_full_tiny_run_writes_hash_only_not_admitted_artifact(
tool, monkeypatch, tmp_path
):
original, normalized, licensing, attributions = _write_tiny_pack(tool, tmp_path)
monkeypatch.setattr(
tool,
"require_omac",
lambda: {"hostname": tool.EXPECTED_HOSTNAME, "physical_memory_bytes": 64 * 1024**3},
)
output_dir = tmp_path / "audit"
payload = tool.run(
original, normalized, licensing, attributions, output_dir
)
assert payload["overall"]["status"] == "NOT_ADMITTED"
assert payload["normalization"]["status"] == (
"PASS_EXACT_REPRODUCTION_OF_NARROW_REWRITE"
)
assert payload["images"]["total_rows"] == 3
assert payload["images"]["missing_images"] == 0
assert payload["safety"]["source_or_label_text_written"] == 0
report = json.loads(
(output_dir / "claim-boundary-audit.json").read_text(encoding="utf-8")
)
assert next(iter(report)) == "license_and_use_boundary"
inventory = (output_dir / "image-label-hash-inventory.jsonl").read_text(
encoding="utf-8"
)
assert "संस्" not in inventory
assert "ऩ" not in inventory
assert (output_dir / "PROGRESS.md").is_file()
assert (output_dir / "SHA256SUMS").is_file()
def test_duplicate_image_groups_and_conflicting_labels_are_counted(tool, tmp_path):
original, normalized, _licensing, _attributions = _write_tiny_pack(tool, tmp_path)
shutil.copyfile(
original / "train/images/train.png",
original / "dev/images/dev.png",
)
stats = json.loads((normalized / "stats.json").read_text(encoding="utf-8"))
_reports, rows = tool.reconcile_labels(original, normalized, stats)
report, _inventory = tool.audit_images(original, rows)
assert report["duplicate_image_hash_groups"] == 1
assert report["rows_with_a_duplicated_image_hash"] == 2
assert report["duplicate_image_rows_beyond_first"] == 1
assert report["image_hashes_shared_across_splits"] == 1
assert report[
"duplicate_image_hash_groups_with_conflicting_label_hashes"
] == 1