from __future__ import annotations import importlib.util import json import shutil from pathlib import Path import pytest from local_artifacts import require_local_artifacts @pytest.fixture(scope="module") def tool(): path = Path("tools/audit_newar_devanagari_normalized_pack.py") spec = importlib.util.spec_from_file_location( "audit_newar_devanagari_normalized_pack", path ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def test_independent_normalizer_handles_only_the_two_documented_classes(tool): normalized, counts = tool.independent_normalize("संस् थापक न ़ दुई शब्द") assert normalized == "संस्थापक ऩ दुई शब्द" assert counts == { "class_a_removals": 1, "class_b_removals": 1, "class_a_characters_removed": 2, "class_b_characters_removed": 2, } def test_real_normalized_labels_reproduce_exactly_and_stats_reconcile(tool): original_dir = Path("outputs/newar-devanagari-silver-pack-v1-recovered") normalized_dir = Path("outputs/newar-pack-v1-normalized-2026-07-15") require_local_artifacts(original_dir, normalized_dir, reason="reconciles the local Newar silver packs") stats = json.loads((normalized_dir / "stats.json").read_text(encoding="utf-8")) reports, rows = tool.reconcile_labels(original_dir, normalized_dir, stats) assert {split: len(rows[split]) for split in tool.SPLITS} == { "train": 4096, "dev": 1528, "test": 1139, } assert reports["train"]["lines_changed"] == 1622 assert reports["dev"]["lines_changed"] == 700 assert reports["test"]["lines_changed"] == 396 assert all( report["targeted_artifact_runs_after_normalization"] == {"class_a_runs": 0, "class_b_runs": 0} for report in reports.values() ) def test_changed_stats_are_rejected(tool): original_dir = Path("outputs/newar-devanagari-silver-pack-v1-recovered") normalized_dir = Path("outputs/newar-pack-v1-normalized-2026-07-15") require_local_artifacts(original_dir, normalized_dir, reason="reconciles the local Newar silver packs") stats = json.loads((normalized_dir / "stats.json").read_text(encoding="utf-8")) stats["test"]["lines_changed"] += 1 with pytest.raises(ValueError, match="stats.json counts do not reconcile"): tool.reconcile_labels(original_dir, normalized_dir, stats) def test_leakage_audit_blocks_exact_and_long_substring_overlap(tool): def row(text: str): return {"text": text, "normalized_text_sha256": tool._sha256_text(text)} split_rows = { "train": [row("unique"), row("abcdefghijklmnop plus")], "dev": [row("abcdefghijklmnop other")], "test": [row("unrelated")], } report = tool.audit_leakage(split_rows, 16) assert report["train_vs_dev_plus_test"]["status"] == "FAIL" assert report["train_vs_dev_plus_test"][ "substring_overlap_left_rows_including_exact" ] == 1 assert report["dev_vs_test"]["status"] == "PASS" def test_host_gate_requires_exact_omac_and_memory(tool, monkeypatch): monkeypatch.setattr(tool.socket, "gethostname", lambda: "wrong-host") monkeypatch.setattr(tool, "_read_physical_memory", lambda: 64 * 1024**3) with pytest.raises(RuntimeError, match="host gate failed"): tool.require_omac() monkeypatch.setattr(tool.socket, "gethostname", lambda: tool.EXPECTED_HOSTNAME) monkeypatch.setattr(tool, "_read_physical_memory", lambda: 18 * 1024**3) with pytest.raises(RuntimeError, match="memory gate failed"): tool.require_omac() def _write_tiny_pack(tool, root: Path): from PIL import Image, ImageDraw original_dir = root / "original" normalized_dir = root / "normalized" normalized_dir.mkdir(parents=True) source_texts = { "train": "संस् ता", "dev": "न ़", "test": "अलग", } stats = {} for index, split in enumerate(tool.SPLITS, start=1): image_dir = original_dir / split / "images" image_dir.mkdir(parents=True) image = Image.new("L", (20 + index, 10), 255) draw = ImageDraw.Draw(image) draw.rectangle((index, 2, index + 3, 7), fill=index * 20) image.save(image_dir / f"{split}.png") source = source_texts[split] normalized, counts = tool.independent_normalize(source) (original_dir / split / "labels.txt").write_text( f"images/{split}.png\t{source}\n", encoding="utf-8" ) (normalized_dir / f"{split}-labels.txt").write_text( f"images/{split}.png\t{normalized}\n", encoding="utf-8" ) stats[split] = { "total_lines": 1, "lines_changed": int(source != normalized), **counts, } (normalized_dir / "stats.json").write_text( json.dumps(stats), encoding="utf-8" ) licensing = root / "DATA_LICENSING.md" licensing.write_text( "| Gorkhapatra | unresolved | RISK | RISK | RISK/NO | RISK |\n", encoding="utf-8", ) attributions = root / "ATTRIBUTIONS.md" attributions.write_text("# Attributions\n", encoding="utf-8") return original_dir, normalized_dir, licensing, attributions def test_full_tiny_run_writes_hash_only_not_admitted_artifact( tool, monkeypatch, tmp_path ): original, normalized, licensing, attributions = _write_tiny_pack(tool, tmp_path) monkeypatch.setattr( tool, "require_omac", lambda: {"hostname": tool.EXPECTED_HOSTNAME, "physical_memory_bytes": 64 * 1024**3}, ) output_dir = tmp_path / "audit" payload = tool.run( original, normalized, licensing, attributions, output_dir ) assert payload["overall"]["status"] == "NOT_ADMITTED" assert payload["normalization"]["status"] == ( "PASS_EXACT_REPRODUCTION_OF_NARROW_REWRITE" ) assert payload["images"]["total_rows"] == 3 assert payload["images"]["missing_images"] == 0 assert payload["safety"]["source_or_label_text_written"] == 0 report = json.loads( (output_dir / "claim-boundary-audit.json").read_text(encoding="utf-8") ) assert next(iter(report)) == "license_and_use_boundary" inventory = (output_dir / "image-label-hash-inventory.jsonl").read_text( encoding="utf-8" ) assert "संस्" not in inventory assert "ऩ" not in inventory assert (output_dir / "PROGRESS.md").is_file() assert (output_dir / "SHA256SUMS").is_file() def test_duplicate_image_groups_and_conflicting_labels_are_counted(tool, tmp_path): original, normalized, _licensing, _attributions = _write_tiny_pack(tool, tmp_path) shutil.copyfile( original / "train/images/train.png", original / "dev/images/dev.png", ) stats = json.loads((normalized / "stats.json").read_text(encoding="utf-8")) _reports, rows = tool.reconcile_labels(original, normalized, stats) report, _inventory = tool.audit_images(original, rows) assert report["duplicate_image_hash_groups"] == 1 assert report["rows_with_a_duplicated_image_hash"] == 2 assert report["duplicate_image_rows_beyond_first"] == 1 assert report["image_hashes_shared_across_splits"] == 1 assert report[ "duplicate_image_hash_groups_with_conflicting_label_hashes" ] == 1