File size: 4,612 Bytes
483b250 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | """Text normalisation, Unicode validation and tokenisation shared by data prep, baselines and eval."""
import re
import unicodedata
ZW = "\u200c\u200d\u200b\ufeff\u00ad" # ZWNJ, ZWJ, ZWSP, BOM, soft hyphen
_ZW_RE = re.compile(f"[{ZW}]")
_WS_RE = re.compile(r"\s+")
# Devanagari character classes
CONS = set(chr(c) for c in range(0x0915, 0x093A)) | set(chr(c) for c in range(0x0958, 0x0960)) | set("ॹॺॻॼॽॾॿ")
INDEP_V = set(chr(c) for c in range(0x0904, 0x0915)) | set("ॠॡॲॳॴॵॶॷ")
MATRA = set(chr(c) for c in range(0x093E, 0x094D)) | set("ॢॣॏॕॖॗ\u093a\u093b\u094e")
VIRAMA = "\u094d"
NUKTA = "\u093c"
NASAL = set("\u0901\u0902") # chandrabindu, anusvara
VISARGA = "\u0903"
DEVA_DIGITS = "०१२३४५६७८९"
DEVA_RANGE = re.compile(r"[\u0900-\u097F]")
DANDA = "।॥"
def nfc(s: str) -> str:
return unicodedata.normalize("NFC", s)
def is_malformed(s: str) -> bool:
"""True if the string contains characters we treat as malformed/garbage:
replacement char, private use, surrogates, unassigned, or control chars other than tab/newline."""
for ch in s:
if ch == "\ufffd":
return True
cat = unicodedata.category(ch)
if cat in ("Co", "Cs", "Cn"):
return True
if cat == "Cc" and ch not in "\t\n":
return True
return False
def clean(s: str) -> str:
"""NFC + strip zero-width/format chars + collapse whitespace."""
s = nfc(s)
s = _ZW_RE.sub("", s)
s = "".join(ch for ch in s if unicodedata.category(ch) != "Cf")
return _WS_RE.sub(" ", s).strip()
def valid_devanagari_word(w: str) -> bool:
"""Orthographic sanity check for a single Devanagari word (letters only).
Rejects: non-Devanagari chars, leading combining marks, matra/virama/nukta after a non-consonant,
two dependent vowel signs in a row, nukta not on a consonant, dangling virama+matra."""
if not w:
return False
prev = None
for i, ch in enumerate(w):
if not ("\u0900" <= ch <= "\u097F"):
return False
if ch in DEVA_DIGITS or ch in DANDA:
return False
if ch in MATRA or ch == VIRAMA or ch == NUKTA:
if prev is None:
return False
if ch == NUKTA and prev not in CONS:
return False
if ch in MATRA and not (prev in CONS or prev == NUKTA):
return False
if ch == VIRAMA and not (prev in CONS or prev == NUKTA):
return False
if ch in NASAL or ch == VISARGA:
if prev is None or prev in NASAL or prev == VIRAMA:
return False
prev = ch
return True
def valid_roman_word(w: str) -> bool:
return bool(re.fullmatch(r"[a-z]+", w))
# --- tokenisation -----------------------------------------------------------
# Words (Latin or Devanagari letters incl. marks), numbers, URLs/emails/handles, punctuation.
TOKEN_RE = re.compile(
r"(https?://\S+|www\.\S+|[\w.+-]+@[\w-]+\.[\w.]+|[@#][\w_]+"
r"|[A-Za-z]+(?:'[A-Za-z]+)?"
r"|[\u0900-\u0963\u0971-\u097F]+"
r"|[0-9\u0966-\u096F]+(?:[.,:][0-9\u0966-\u096F]+)*"
r"|[\u0964\u0965]|[^\sA-Za-z0-9\u0900-\u097F])"
)
def tokenize(s: str):
return TOKEN_RE.findall(s)
PUNCT_RE = re.compile(r"[^\w\u0900-\u097F]+", re.UNICODE)
# --- scoring normalisation --------------------------------------------------
_DIGIT_TR = str.maketrans(DEVA_DIGITS, "0123456789")
def norm_for_score(s: str) -> str:
"""'norm' scoring view: NFC, no zero-width chars, Devanagari digits -> ASCII, punctuation removed
(incl. danda), whitespace collapsed. Spelling is otherwise untouched (strict on letters)."""
s = clean(s).translate(_DIGIT_TR)
s = re.sub(r"[^\w\s\u0900-\u097F]|[\u0964\u0965]", " ", s)
return _WS_RE.sub(" ", s).strip()
_FOLD_MAP = {
"\u0901": "\u0902", # chandrabindu -> anusvara (हूँ ~ हूं)
NUKTA: "", # drop nukta (ज़ ~ ज)
"\u0945": "\u0947", # candra e (ॅ) -> e
"\u0949": "\u094b", # candra o (ॉ) -> o (ऑफिस ~ ओफिस)
"\u0911": "\u0913", # independent candra O -> O
"\u090d": "\u090f",
}
def fold(s: str) -> str:
"""'fold' scoring view (variant-tolerant, reported separately): norm_for_score + nukta dropped,
chandrabindu->anusvara, candra vowels -> plain. Never used as the primary metric."""
s = unicodedata.normalize("NFD", norm_for_score(s)) # decompose precomposed nukta letters
s = "".join(_FOLD_MAP.get(ch, ch) for ch in s)
return unicodedata.normalize("NFC", s)
|