""" Memoni Roman/Urdu normalization rules. Used by: - labeling app (sidebar reference for volunteers) - Module 3.5 pipeline (normalize pseudo-labeled text before mT5 training) Rule: one canonical Roman form per word. Urdu script form is the equivalent for volunteers who prefer to write in Urdu. """ # Roman variant → canonical Roman # All keys lowercase. Apply after lowercasing input. CANONICAL = { # I / in 'main': 'mein', 'me': 'mein', 'mn': 'mein', 'mien': 'mein', # is 'hy': 'hai', 'he': 'hai', 'hay': 'hai', # are 'hen': 'hain', 'hein': 'hain', # and 'or': 'aur', 'ur': 'aur', 'oor': 'aur', # no / not 'nahin': 'nahi', 'nai': 'nahi', 'ni': 'nahi', 'nae': 'nahi', 'nhi': 'nahi', # what / did 'kia': 'kya', 'kiya': 'kya', 'ke': 'kya', # come 'aavo': 'aao', 'aaw': 'aao', 'aawo': 'aao', # go 'jaavo': 'jao', 'jaw': 'jao', 'jawo': 'jao', # this 'ye': 'yeh', 'ya': 'yeh', 'yae': 'yeh', # that 'wo': 'woh', 'wu': 'woh', # house 'gher': 'ghar', 'ghur': 'ghar', # say / said 'keha': 'kaha', 'kehya': 'kaha', # person 'bunda': 'banda', 'bende': 'banda', # now 'ub': 'ab', 'abb': 'ab', # very 'buhat': 'bohat', 'bhut': 'bohat', 'bahut': 'bohat', # good 'accha': 'acha', 'atcha': 'acha', 'achha': 'acha', # yes 'haan': 'han', 'hun': 'han', # today 'aj': 'aaj', # come (imperative, singular) 'awo': 'ao', 'aavo': 'ao', # with 'saath': 'sath', 'saath': 'sath', # here 'idhar': 'ider', 'udhar': 'uder', # how 'kaise': 'kese', 'kaisa': 'kesa', # why 'kyun': 'kyun', 'kiun': 'kyun', # where 'kahan': 'kahan', 'kahan': 'kahan', # when 'kab': 'kab', # want 'chahiye': 'chahye', 'chahie': 'chahye', # money 'paisa': 'pesa', 'paesa': 'pesa', # brother 'bhai': 'bhai', 'bhi': 'bhai', # sister 'bhen': 'bhen', 'behan': 'bhen', } # Reference table shown to volunteers in the labeling UI # Format: (canonical_roman, urdu_script, english_meaning) REFERENCE_TABLE = [ ('mein', 'میں', 'I / in'), ('hai', 'ہے', 'is'), ('hain', 'ہیں', 'are'), ('aur', 'اور', 'and'), ('nahi', 'نہیں', 'no / not'), ('kya', 'کیا', 'what / did'), ('aao', 'آؤ', 'come (plural)'), ('ao', 'آو', 'come (singular)'), ('jao', 'جاؤ', 'go'), ('yeh', 'یہ', 'this'), ('woh', 'وہ', 'that'), ('ghar', 'گھر', 'house'), ('bohat', 'بہت', 'very'), ('acha', 'اچھا', 'good / okay'), ('han', 'ہاں', 'yes'), ('na', 'نہ', 'no (soft)'), ('aaj', 'آج', 'today'), ('kab', 'کب', 'when'), ('kahan', 'کہاں', 'where'), ('kyun', 'کیوں', 'why'), ('kese', 'کیسے', 'how'), ('sath', 'ساتھ', 'with'), ('ab', 'اب', 'now'), ('bhai', 'بھائی','brother'), ('bhen', 'بہن', 'sister'), ('kaha', 'کہا', 'said'), ('banda', 'بندہ', 'person / guy'), ('pesa', 'پیسہ', 'money'), ] def normalize(text: str) -> str: """Normalize a Roman Memoni transcript to canonical forms.""" words = text.lower().split() return ' '.join(CANONICAL.get(w, w) for w in words) # ── Urdu script → Roman transliteration ────────────────────────────────────── _U2R = { 'ا': 'a', 'آ': 'aa', 'ب': 'b', 'پ': 'p', 'ت': 't', 'ٹ': 't', 'ث': 's', 'ج': 'j', 'چ': 'ch', 'ح': 'h', 'خ': 'kh', 'د': 'd', 'ڈ': 'd', 'ذ': 'z', 'ر': 'r', 'ڑ': 'r', 'ز': 'z', 'ژ': 'zh', 'س': 's', 'ش': 'sh', 'ص': 's', 'ض': 'z', 'ط': 't', 'ظ': 'z', 'ع': '', 'غ': 'gh', 'ف': 'f', 'ق': 'q', 'ک': 'k', 'گ': 'g', 'ل': 'l', 'م': 'm', 'ن': 'n', 'ں': 'n', 'و': 'w', 'ہ': 'h', 'ھ': 'h', 'ی': 'y', 'ے': 'e', 'ء': '', 'ئ': 'y', 'ؤ': 'w', 'ة': 't', 'إ': 'i', 'أ': 'a', 'ٰ': '', '،': ',', '؟': '?', # diacritics 'َ': 'a', 'ِ': 'i', 'ُ': 'u', 'ّ': '', 'ْ': '', 'ً': 'an', 'ٍ': 'in', 'ٌ': 'un', ' ': ' ', } def urdu_to_roman(text: str) -> str: """Best-effort character-level transliteration of Urdu script to Roman.""" out = [] for ch in text: out.append(_U2R.get(ch, ch if ord(ch) < 128 else '')) result = ''.join(out).strip() # Collapse multiple spaces import re return re.sub(r' +', ' ', result)