"""Runtime invariants over the vendored KashmiriNormalizer mappings. The vendored `constants.py` is a third-party file we may re-pull to a newer commit. A careless upstream edit could start *deleting* genuine Kashmiri letters (e.g. by listing one under the empty-string key `''`, which the normalizer treats as "map to nothing"). For Kashmiri that is catastrophic: ۄ and ۍ carry vowels, ؠ is a real consonant. `run_guards()` flattens the character map exactly the way the normalizer does (source-char -> target-char) and asserts that no protected Kashmiri letter is ever a *source* that maps to deletion or to a different base letter. It is cheap and runs at the start of data preparation, so a bad vendor bump fails loudly instead of silently corrupting the corpus. """ from __future__ import annotations from typing import Dict, List from ._upstream.constants import KASHMIRI_CHARACTER_MAPPING, PUNCTUATION_MAP # Kashmiri / Perso-Arabic letters that MUST survive normalization, by codepoint # (codepoints, not literals, so this file is encoding-safe on every platform). _PROTECTED_CODEPOINTS = [ 0x0620, # ؠ ARABIC LETTER KASHMIRI YEH 0x06C4, # ۄ ARABIC LETTER WAW WITH RING 0x06C2, # ۂ HEH GOAL WITH HAMZA ABOVE 0x06C3, # ۃ TEH MARBUTA GOAL 0x06D3, # ۓ YEH BARREE WITH HAMZA ABOVE 0x06CD, # ۍ YEH WITH TAIL 0x0679, # ٹ TTEH 0x0688, # ڈ DDAL 0x0691, # ڑ RREH 0x0698, # ژ JEH 0x067E, # پ PEH 0x0686, # چ TCHEH 0x06AF, # گ GAF 0x06A9, # ک KEHEH 0x06BE, # ھ HEH DOACHASHMEE 0x06C1, # ہ HEH GOAL 0x06BA, # ں NOON GHUNNA ] PROTECTED_LETTERS = frozenset(chr(cp) for cp in _PROTECTED_CODEPOINTS) ZWNJ = "‌" TATWEEL = "ـ" class NormalizerGuardError(RuntimeError): """Raised when a vendored mapping would corrupt protected Kashmiri letters.""" def _flatten(char_map: Dict[str, List[str]]) -> Dict[str, str]: """Invert `{target: [sources...]}` to `{source: target}` (upstream's logic).""" flat: Dict[str, str] = {} for target, sources in char_map.items(): for src in sources: flat[src] = target return flat def run_guards(strict: bool = True) -> dict: """Validate the vendored maps. Returns a report; raises on hard violations. Args: strict: if True (default) raise NormalizerGuardError on any violation; if False, only collect violations into the returned report. """ flat = _flatten(KASHMIRI_CHARACTER_MAPPING) flat.update(_flatten(PUNCTUATION_MAP)) violations: List[str] = [] for letter in sorted(PROTECTED_LETTERS): if letter in flat: target = flat[letter] if target == "": violations.append( f"protected letter U+{ord(letter):04X} maps to DELETION ('')" ) elif target != letter: violations.append( f"protected letter U+{ord(letter):04X} maps to " f"U+{ord(target):04X} (would be rewritten)" ) report = { "protected_count": len(PROTECTED_LETTERS), "violations": violations, # Surface the two policy-sensitive mappings so callers can log them. "tatweel_maps_to": repr(_flatten(KASHMIRI_CHARACTER_MAPPING).get(TATWEEL, TATWEEL)), "zwnj_maps_to": repr(_flatten(KASHMIRI_CHARACTER_MAPPING).get(ZWNJ, ZWNJ)), } if violations and strict: raise NormalizerGuardError( "Vendored KashmiriNormalizer would corrupt protected letters:\n - " + "\n - ".join(violations) + "\nRefusing to normalize. Inspect ksbyte/normalize/_upstream/constants.py." ) return report if __name__ == "__main__": # quick manual check import json print(json.dumps(run_guards(strict=True), ensure_ascii=False, indent=2))