ksbyte/normalize/guard.py

"""Runtime invariants over the vendored KashmiriNormalizer mappings.

The vendored `constants.py` is a third-party file we may re-pull to a newer
commit. A careless upstream edit could start *deleting* genuine Kashmiri
letters (e.g. by listing one under the empty-string key `''`, which the
normalizer treats as "map to nothing"). For Kashmiri that is catastrophic:
ۄ and ۍ carry vowels, ؠ is a real consonant.

`run_guards()` flattens the character map exactly the way the normalizer does
(source-char -> target-char) and asserts that no protected Kashmiri letter is
ever a *source* that maps to deletion or to a different base letter. It is
cheap and runs at the start of data preparation, so a bad vendor bump fails
loudly instead of silently corrupting the corpus.
"""
from __future__ import annotations

from typing import Dict, List

from ._upstream.constants import KASHMIRI_CHARACTER_MAPPING, PUNCTUATION_MAP

# Kashmiri / Perso-Arabic letters that MUST survive normalization, by codepoint
# (codepoints, not literals, so this file is encoding-safe on every platform).
_PROTECTED_CODEPOINTS = [
    0x0620,  # ؠ  ARABIC LETTER KASHMIRI YEH
    0x06C4,  # ۄ  ARABIC LETTER WAW WITH RING
    0x06C2,  # ۂ  HEH GOAL WITH HAMZA ABOVE
    0x06C3,  # ۃ  TEH MARBUTA GOAL
    0x06D3,  # ۓ  YEH BARREE WITH HAMZA ABOVE
    0x06CD,  # ۍ  YEH WITH TAIL
    0x0679,  # ٹ  TTEH
    0x0688,  # ڈ  DDAL
    0x0691,  # ڑ  RREH
    0x0698,  # ژ  JEH
    0x067E,  # پ  PEH
    0x0686,  # چ  TCHEH
    0x06AF,  # گ  GAF
    0x06A9,  # ک  KEHEH
    0x06BE,  # ھ  HEH DOACHASHMEE
    0x06C1,  # ہ  HEH GOAL
    0x06BA,  # ں  NOON GHUNNA
]
PROTECTED_LETTERS = frozenset(chr(cp) for cp in _PROTECTED_CODEPOINTS)

ZWNJ = "‌"
TATWEEL = "ـ"


class NormalizerGuardError(RuntimeError):
    """Raised when a vendored mapping would corrupt protected Kashmiri letters."""


def _flatten(char_map: Dict[str, List[str]]) -> Dict[str, str]:
    """Invert `{target: [sources...]}` to `{source: target}` (upstream's logic)."""
    flat: Dict[str, str] = {}
    for target, sources in char_map.items():
        for src in sources:
            flat[src] = target
    return flat


def run_guards(strict: bool = True) -> dict:
    """Validate the vendored maps. Returns a report; raises on hard violations.

    Args:
        strict: if True (default) raise NormalizerGuardError on any violation;
                if False, only collect violations into the returned report.
    """
    flat = _flatten(KASHMIRI_CHARACTER_MAPPING)
    flat.update(_flatten(PUNCTUATION_MAP))

    violations: List[str] = []
    for letter in sorted(PROTECTED_LETTERS):
        if letter in flat:
            target = flat[letter]
            if target == "":
                violations.append(
                    f"protected letter U+{ord(letter):04X} maps to DELETION ('')"
                )
            elif target != letter:
                violations.append(
                    f"protected letter U+{ord(letter):04X} maps to "
                    f"U+{ord(target):04X} (would be rewritten)"
                )

    report = {
        "protected_count": len(PROTECTED_LETTERS),
        "violations": violations,
        # Surface the two policy-sensitive mappings so callers can log them.
        "tatweel_maps_to": repr(_flatten(KASHMIRI_CHARACTER_MAPPING).get(TATWEEL, TATWEEL)),
        "zwnj_maps_to": repr(_flatten(KASHMIRI_CHARACTER_MAPPING).get(ZWNJ, ZWNJ)),
    }

    if violations and strict:
        raise NormalizerGuardError(
            "Vendored KashmiriNormalizer would corrupt protected letters:\n  - "
            + "\n  - ".join(violations)
            + "\nRefusing to normalize. Inspect ksbyte/normalize/_upstream/constants.py."
        )
    return report


if __name__ == "__main__":  # quick manual check
    import json

    print(json.dumps(run_guards(strict=True), ensure_ascii=False, indent=2))