| """Text normalisation, Unicode validation and tokenisation shared by data prep, baselines and eval.""" |
| import re |
| import unicodedata |
|
|
| ZW = "\u200c\u200d\u200b\ufeff\u00ad" |
| _ZW_RE = re.compile(f"[{ZW}]") |
| _WS_RE = re.compile(r"\s+") |
|
|
| |
| CONS = set(chr(c) for c in range(0x0915, 0x093A)) | set(chr(c) for c in range(0x0958, 0x0960)) | set("ॹॺॻॼॽॾॿ") |
| INDEP_V = set(chr(c) for c in range(0x0904, 0x0915)) | set("ॠॡॲॳॴॵॶॷ") |
| MATRA = set(chr(c) for c in range(0x093E, 0x094D)) | set("ॢॣॏॕॖॗ\u093a\u093b\u094e") |
| VIRAMA = "\u094d" |
| NUKTA = "\u093c" |
| NASAL = set("\u0901\u0902") |
| VISARGA = "\u0903" |
| DEVA_DIGITS = "०१२३४५६७८९" |
| DEVA_RANGE = re.compile(r"[\u0900-\u097F]") |
| DANDA = "।॥" |
|
|
|
|
| def nfc(s: str) -> str: |
| return unicodedata.normalize("NFC", s) |
|
|
|
|
| def is_malformed(s: str) -> bool: |
| """True if the string contains characters we treat as malformed/garbage: |
| replacement char, private use, surrogates, unassigned, or control chars other than tab/newline.""" |
| for ch in s: |
| if ch == "\ufffd": |
| return True |
| cat = unicodedata.category(ch) |
| if cat in ("Co", "Cs", "Cn"): |
| return True |
| if cat == "Cc" and ch not in "\t\n": |
| return True |
| return False |
|
|
|
|
| def clean(s: str) -> str: |
| """NFC + strip zero-width/format chars + collapse whitespace.""" |
| s = nfc(s) |
| s = _ZW_RE.sub("", s) |
| s = "".join(ch for ch in s if unicodedata.category(ch) != "Cf") |
| return _WS_RE.sub(" ", s).strip() |
|
|
|
|
| def valid_devanagari_word(w: str) -> bool: |
| """Orthographic sanity check for a single Devanagari word (letters only). |
| Rejects: non-Devanagari chars, leading combining marks, matra/virama/nukta after a non-consonant, |
| two dependent vowel signs in a row, nukta not on a consonant, dangling virama+matra.""" |
| if not w: |
| return False |
| prev = None |
| for i, ch in enumerate(w): |
| if not ("\u0900" <= ch <= "\u097F"): |
| return False |
| if ch in DEVA_DIGITS or ch in DANDA: |
| return False |
| if ch in MATRA or ch == VIRAMA or ch == NUKTA: |
| if prev is None: |
| return False |
| if ch == NUKTA and prev not in CONS: |
| return False |
| if ch in MATRA and not (prev in CONS or prev == NUKTA): |
| return False |
| if ch == VIRAMA and not (prev in CONS or prev == NUKTA): |
| return False |
| if ch in NASAL or ch == VISARGA: |
| if prev is None or prev in NASAL or prev == VIRAMA: |
| return False |
| prev = ch |
| return True |
|
|
|
|
| def valid_roman_word(w: str) -> bool: |
| return bool(re.fullmatch(r"[a-z]+", w)) |
|
|
|
|
| |
| |
| TOKEN_RE = re.compile( |
| r"(https?://\S+|www\.\S+|[\w.+-]+@[\w-]+\.[\w.]+|[@#][\w_]+" |
| r"|[A-Za-z]+(?:'[A-Za-z]+)?" |
| r"|[\u0900-\u0963\u0971-\u097F]+" |
| r"|[0-9\u0966-\u096F]+(?:[.,:][0-9\u0966-\u096F]+)*" |
| r"|[\u0964\u0965]|[^\sA-Za-z0-9\u0900-\u097F])" |
| ) |
|
|
|
|
| def tokenize(s: str): |
| return TOKEN_RE.findall(s) |
|
|
|
|
| PUNCT_RE = re.compile(r"[^\w\u0900-\u097F]+", re.UNICODE) |
|
|
|
|
| |
| _DIGIT_TR = str.maketrans(DEVA_DIGITS, "0123456789") |
|
|
|
|
| def norm_for_score(s: str) -> str: |
| """'norm' scoring view: NFC, no zero-width chars, Devanagari digits -> ASCII, punctuation removed |
| (incl. danda), whitespace collapsed. Spelling is otherwise untouched (strict on letters).""" |
| s = clean(s).translate(_DIGIT_TR) |
| s = re.sub(r"[^\w\s\u0900-\u097F]|[\u0964\u0965]", " ", s) |
| return _WS_RE.sub(" ", s).strip() |
|
|
|
|
| _FOLD_MAP = { |
| "\u0901": "\u0902", |
| NUKTA: "", |
| "\u0945": "\u0947", |
| "\u0949": "\u094b", |
| "\u0911": "\u0913", |
| "\u090d": "\u090f", |
| } |
|
|
|
|
| def fold(s: str) -> str: |
| """'fold' scoring view (variant-tolerant, reported separately): norm_for_score + nukta dropped, |
| chandrabindu->anusvara, candra vowels -> plain. Never used as the primary metric.""" |
| s = unicodedata.normalize("NFD", norm_for_score(s)) |
| s = "".join(_FOLD_MAP.get(ch, ch) for ch in s) |
| return unicodedata.normalize("NFC", s) |
|
|
