normalization.py

"""
Memoni Roman/Urdu normalization rules.
Used by:
  - labeling app (sidebar reference for volunteers)
  - Module 3.5 pipeline (normalize pseudo-labeled text before mT5 training)

Rule: one canonical Roman form per word.
Urdu script form is the equivalent for volunteers who prefer to write in Urdu.
"""

# Roman variant → canonical Roman
# All keys lowercase. Apply after lowercasing input.
CANONICAL = {
    # I / in
    'main': 'mein', 'me': 'mein', 'mn': 'mein', 'mien': 'mein',

    # is
    'hy': 'hai', 'he': 'hai', 'hay': 'hai',

    # are
    'hen': 'hain', 'hein': 'hain',

    # and
    'or': 'aur', 'ur': 'aur', 'oor': 'aur',

    # no / not
    'nahin': 'nahi', 'nai': 'nahi', 'ni': 'nahi', 'nae': 'nahi', 'nhi': 'nahi',

    # what / did
    'kia': 'kya', 'kiya': 'kya', 'ke': 'kya',

    # come
    'aavo': 'aao', 'aaw': 'aao', 'aawo': 'aao',

    # go
    'jaavo': 'jao', 'jaw': 'jao', 'jawo': 'jao',

    # this
    'ye': 'yeh', 'ya': 'yeh', 'yae': 'yeh',

    # that
    'wo': 'woh', 'wu': 'woh',

    # house
    'gher': 'ghar', 'ghur': 'ghar',

    # say / said
    'keha': 'kaha', 'kehya': 'kaha',

    # person
    'bunda': 'banda', 'bende': 'banda',

    # now
    'ub': 'ab', 'abb': 'ab',

    # very
    'buhat': 'bohat', 'bhut': 'bohat', 'bahut': 'bohat',

    # good
    'accha': 'acha', 'atcha': 'acha', 'achha': 'acha',

    # yes
    'haan': 'han', 'hun': 'han',

    # today
    'aj': 'aaj',

    # come (imperative, singular)
    'awo': 'ao', 'aavo': 'ao',

    # with
    'saath': 'sath', 'saath': 'sath',

    # here
    'idhar': 'ider', 'udhar': 'uder',

    # how
    'kaise': 'kese', 'kaisa': 'kesa',

    # why
    'kyun': 'kyun', 'kiun': 'kyun',

    # where
    'kahan': 'kahan', 'kahan': 'kahan',

    # when
    'kab': 'kab',

    # want
    'chahiye': 'chahye', 'chahie': 'chahye',

    # money
    'paisa': 'pesa', 'paesa': 'pesa',

    # brother
    'bhai': 'bhai', 'bhi': 'bhai',

    # sister
    'bhen': 'bhen', 'behan': 'bhen',
}


# Reference table shown to volunteers in the labeling UI
# Format: (canonical_roman, urdu_script, english_meaning)
REFERENCE_TABLE = [
    ('mein',  'میں',  'I / in'),
    ('hai',   'ہے',   'is'),
    ('hain',  'ہیں',  'are'),
    ('aur',   'اور',  'and'),
    ('nahi',  'نہیں', 'no / not'),
    ('kya',   'کیا',  'what / did'),
    ('aao',   'آؤ',   'come (plural)'),
    ('ao',    'آو',   'come (singular)'),
    ('jao',   'جاؤ',  'go'),
    ('yeh',   'یہ',   'this'),
    ('woh',   'وہ',   'that'),
    ('ghar',  'گھر',  'house'),
    ('bohat', 'بہت',  'very'),
    ('acha',  'اچھا', 'good / okay'),
    ('han',   'ہاں',  'yes'),
    ('na',    'نہ',   'no (soft)'),
    ('aaj',   'آج',   'today'),
    ('kab',   'کب',   'when'),
    ('kahan', 'کہاں', 'where'),
    ('kyun',  'کیوں', 'why'),
    ('kese',  'کیسے', 'how'),
    ('sath',  'ساتھ', 'with'),
    ('ab',    'اب',   'now'),
    ('bhai',  'بھائی','brother'),
    ('bhen',  'بہن',  'sister'),
    ('kaha',  'کہا',  'said'),
    ('banda', 'بندہ', 'person / guy'),
    ('pesa',  'پیسہ', 'money'),
]


def normalize(text: str) -> str:
    """Normalize a Roman Memoni transcript to canonical forms."""
    words = text.lower().split()
    return ' '.join(CANONICAL.get(w, w) for w in words)


# ── Urdu script → Roman transliteration ──────────────────────────────────────
_U2R = {
    'ا': 'a',  'آ': 'aa', 'ب': 'b',  'پ': 'p',  'ت': 't',  'ٹ': 't',
    'ث': 's',  'ج': 'j',  'چ': 'ch', 'ح': 'h',  'خ': 'kh', 'د': 'd',
    'ڈ': 'd',  'ذ': 'z',  'ر': 'r',  'ڑ': 'r',  'ز': 'z',  'ژ': 'zh',
    'س': 's',  'ش': 'sh', 'ص': 's',  'ض': 'z',  'ط': 't',  'ظ': 'z',
    'ع': '',   'غ': 'gh', 'ف': 'f',  'ق': 'q',  'ک': 'k',  'گ': 'g',
    'ل': 'l',  'م': 'm',  'ن': 'n',  'ں': 'n',  'و': 'w',  'ہ': 'h',
    'ھ': 'h',  'ی': 'y',  'ے': 'e',  'ء': '',   'ئ': 'y',  'ؤ': 'w',
    'ة': 't',  'إ': 'i',  'أ': 'a',  'ٰ': '',   '،': ',',  '؟': '?',
    # diacritics
    'َ': 'a',  'ِ': 'i',  'ُ': 'u',  'ّ': '',   'ْ': '',   'ً': 'an',
    'ٍ': 'in', 'ٌ': 'un',
    ' ': ' ',
}

def urdu_to_roman(text: str) -> str:
    """Best-effort character-level transliteration of Urdu script to Roman."""
    out = []
    for ch in text:
        out.append(_U2R.get(ch, ch if ord(ch) < 128 else ''))
    result = ''.join(out).strip()
    # Collapse multiple spaces
    import re
    return re.sub(r' +', ' ', result)