normalizer.py

"""
GCAS Search Engine – Query Normalizer
======================================

Handles three classes of input noise before the query reaches FAISS:

  1. Language detection  – detects English / Hindi / Gujarati
  2. Script transliteration – Gujarati/Hindi Unicode → Latin (for fuzzy matching)
  3. Alias resolution – maps regional spellings, ASR phonetic variants,
     and common abbreviations to canonical forms used in the database
  4. Text normalisation – lowercase, collapse whitespace, strip punctuation

Pipeline
--------
  raw query
      → detect_language()
      → transliterate_to_latin()   (only if Gujarati/Hindi Unicode)
      → expand_abbreviations()
      → resolve_aliases()
      → normalize_text()
      → cleaned query string + metadata dict
"""
from __future__ import annotations

import re
import unicodedata
from typing import Dict, Optional, Tuple

# ---------------------------------------------------------------------------
# 1. Known entity alias tables
#    Keys are lowercase, stripped variants (incl. common ASR mistakes)
#    Values are the canonical form found in the database
# ---------------------------------------------------------------------------

# ── Districts & Cities ──────────────────────────────────────────────────────
CITY_ALIASES: Dict[str, str] = {
    # Ahmedabad variants (Gujarati: અમદાવાદ, Hindi: अहमदाबाद)
    "amdavad": "Ahmedabad",
    "amdabad": "Ahmedabad",
    "ahemadabad": "Ahmedabad",
    "ahemdabad": "Ahmedabad",
    "ahmadabad": "Ahmedabad",
    "ahmedabaad": "Ahmedabad",
    "ahmed abad": "Ahmedabad",
    "ahmedabad": "Ahmedabad",
    # Vadodara / Baroda
    "baroda": "Vadodara",
    "vadodra": "Vadodara",
    "vadodar": "Vadodara",
    "barod": "Vadodara",
    # Surat
    "soorat": "Surat",
    "sorat": "Surat",
    "suret": "Surat",
    # Rajkot
    "rajkote": "Rajkot",
    "rajkott": "Rajkot",
    "raj kot": "Rajkot",
    # Bhavnagar
    "bhawnagar": "Bhavnagar",
    "bhavanagar": "Bhavnagar",
    "bhavnagar": "Bhavnagar",
    "bhaunagar": "Bhavnagar",
    # Gandhinagar
    "gandhi nagar": "Gandhinagar",
    "gandhinagr": "Gandhinagar",
    # Mehsana
    "mahesana": "Mehsana",
    "mehsana": "Mehsana",
    "mehasana": "Mehsana",
    "mahisana": "Mehsana",
    # Anand
    "aanand": "Anand",
    "annad": "Anand",
    # Kutch / Kachchh
    "kutch": "Kutch",
    "kachchh": "Kutch",
    "kaach": "Kutch",
    "kachch": "Kutch",
    # Junagadh
    "junagarh": "Junagadh",
    "junagar": "Junagadh",
    "junaagadh": "Junagadh",
    # Jamnagar
    "jamnagar": "Jamnagar",
    "jamnagr": "Jamnagar",
    # Surendranagar
    "surendranagar": "Surendra Nagar",
    "surendra nagar": "Surendra Nagar",
    # GirSomnath
    "gir somnath": "GirSomnath",
    "girsomnath": "GirSomnath",
    # Banaskantha
    "banaskanta": "Banaskantha",
    "banas kantha": "Banaskantha",
    # Sabarkantha
    "sabar kantha": "Sabarkantha",
    "sabarkanta": "Sabarkantha",
    # Navsari
    "navsari": "Navsari",
    "navsarri": "Navsari",
    # Valsad
    "valsad": "Valsad",
    "valsaad": "Valsad",
    # Patan
    "patan": "Patan",
    # Bharuch
    "bharuch": "Bharuch",
    "bharooch": "Bharuch",
    "broach": "Bharuch",
    # Amreli
    "amreli": "Amreli",
    "amrely": "Amreli",
    # Porbandar
    "porbandar": "Porbandar",
    "porbander": "Porbandar",
    # Dahod
    "dahod": "Dahod",
    "daaod": "Dahod",
    # Panchmahal
    "panch mahal": "Panchmahal",
    "panchmahal": "Panchmahal",
    "panchmahals": "Panchmahal",
    # Chhota Udepur
    "chhota udepur": "Chhota Udepur",
    "chota udepur": "Chhota Udepur",
    # Narmada
    "narmada": "Narmada",
    # Navsari
    "navsari": "Navsari",
    # Tapi
    "tapi": "TAPI",
    # Aravalli
    "aravalli": "Aravalli",
    "aravali": "Aravalli",
    # Morbi
    "morbi": "Morbi",
    "morvi": "Morbi",
    # Botad
    "botad": "Botad",
    # Mahisagar
    "mahisagar": "Mahisagar",
    # Devbhumi Dwarka
    "dwarka": "DEVBHUMI DWARKA",
    "devbhumi dwarka": "DEVBHUMI DWARKA",
    "deobhumi dwarka": "DEVBHUMI DWARKA",
    # Dang
    "dang": "Dang",
    # Kheda
    "kheda": "Kheda",
    "kaira": "Kheda",
    "nadiad": "Kheda",
    "visnagar": "Mehsana",
    "vallabh vidyanagar": "Anand",
    "vallabhpur": "Anand",
    "vapi": "Valsad",
    "godhra": "Panchmahal",
    "palanpur": "Banaskantha",
    "ankleshwar": "Bharuch",
    "dahej": "Bharuch",
    "halol": "Panchmahal",
}

# ── University abbreviations & common mistakes ──────────────────────────────
UNIVERSITY_ALIASES: Dict[str, str] = {
    "gtu": "Gujarat Technological University",
    "gujarat tech university": "Gujarat Technological University",
    "gujarat technical university": "Gujarat Technological University",
    "vnsgu": "VEER NARMAD SOUTH GUJARAT UNIVERSITY",
    "veer narmad university": "VEER NARMAD SOUTH GUJARAT UNIVERSITY",
    "south gujarat university": "VEER NARMAD SOUTH GUJARAT UNIVERSITY",
    "spu": "SARDAR PATEL UNIVERSITY",
    "sardar patel university": "SARDAR PATEL UNIVERSITY",
    "hngu": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN",
    "north gujarat university": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN",
    "hemchandracharya university": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN",
    "msu": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA",
    "maharaja sayajirao university": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA",
    "ms university": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA",
    "msu baroda": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA",
    "gu": "GUJARAT UNIVERSITY",
    "gujarat university": "GUJARAT UNIVERSITY",
    "sau": "SAURASHTRA UNIVERSITY",
    "saurashtra university": "SAURASHTRA UNIVERSITY",
    "saurastra university": "SAURASHTRA UNIVERSITY",
    "ksv": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY",
    "kachchh university": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY",
    "kutch university": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY",
    "mkbu": "MAHARAJA KRISHNAKUMARSINHJI BHAVNAGAR UNIVERSITY",
    "bhavnagar university": "MAHARAJA KRISHNAKUMARSINHJI BHAVNAGAR UNIVERSITY",
    "bknmu": "BHAKTA KAVI NARSINH MEHTA UNIVERSITY",
    "narsinh mehta university": "BHAKTA KAVI NARSINH MEHTA UNIVERSITY",
    "junagadh university": "BHAKTA KAVI NARSINH MEHTA UNIVERSITY",
    "sggu": "SHRI GOVIND GURU UNIVERSITY",
    "govind guru university": "SHRI GOVIND GURU UNIVERSITY",
    "sssu": "SHREE SOMNATH SANSKRIT UNIVERSITY, VERAVAL",
    "somnath university": "SHREE SOMNATH SANSKRIT UNIVERSITY, VERAVAL",
    "iite": "INDIAN INSTITUTE OF TEACHER EDUCATION",
}

# ── Course / Program abbreviations ──────────────────────────────────────────
COURSE_ALIASES: Dict[str, str] = {
    # Engineering
    "btech": "B.TECH",
    "b tech": "B.TECH",
    "b.tech.": "B.TECH",
    "be": "BACHELOR OF ENGINEERING",
    "bachelor of engineering": "BACHELOR OF ENGINEERING",
    "mtech": "M.TECH",
    "m tech": "M.TECH",
    "me": "MASTER OF ENGINEERING",
    # Management
    "mba": "MASTER OF BUSINESS ADMINISTRATION",
    "bba": "BACHELOR OF BUSINESS ADMINISTRATION",
    "pgdm": "POST GRADUATE DIPLOMA IN MANAGEMENT",
    # Computer
    "mca": "MASTER OF COMPUTER APPLICATIONS",
    "bca": "BACHELOR OF COMPUTER APPLICATIONS",
    # Commerce
    "bcom": "B.COM.",
    "b com": "B.COM.",
    "b.com": "B.COM.",
    "mcom": "M.COM.",
    "m com": "M.COM.",
    # Science
    "bsc": "B.SC.",
    "b sc": "B.SC.",
    "b.sc": "B.SC.",
    "msc": "M.SC.",
    "m sc": "M.SC.",
    "m.sc": "M.SC.",
    # Arts
    "ba": "BACHELOR OF ARTS",
    "ma": "MASTER OF ARTS",
    # Law
    "llb": "BACHELOR OF LAWS",
    "ll.b": "BACHELOR OF LAWS",
    "llm": "MASTER OF LAWS",
    "ll.m": "MASTER OF LAWS",
    # Medicine / Pharmacy
    "mbbs": "BACHELOR OF MEDICINE AND BACHELOR OF SURGERY",
    "bpharm": "B.PHARM.",
    "b pharm": "B.PHARM.",
    "b.pharm": "B.PHARM.",
    "mpharm": "M.PHARM.",
    "m pharm": "M.PHARM.",
    "dpharm": "D.PHARM.",
    "d pharm": "D.PHARM.",
    # Architecture / Design
    "barch": "B.ARCH.",
    "b arch": "B.ARCH.",
    "march": "M.ARCH.",
    "bdes": "B.DES.",
    "mdes": "M.DES.",
    # Education
    "bed": "BACHELOR OF EDUCATION",
    "b ed": "BACHELOR OF EDUCATION",
    "med": "MASTER OF EDUCATION",
    "m ed": "MASTER OF EDUCATION",
    "bped": "BACHELOR OF PHYSICAL EDUCATION",
    # Social work
    "msw": "MASTER OF SOCIAL WORK",
    "bsw": "BACHELOR OF SOCIAL WORK",
    # Research
    "phd": "DOCTOR OF PHILOSOPHY",
    "ph d": "DOCTOR OF PHILOSOPHY",
    "ph.d": "DOCTOR OF PHILOSOPHY",
    "mphil": "MASTER OF PHILOSOPHY",
    # Hotel / Hospitality
    "bhmct": "BACHELOR OF HOTEL MANAGEMENT AND CATERING TECHNOLOGY",
    "mhmct": "MASTER OF HOTEL MANAGEMENT AND CATERING TECHNOLOGY",
    # Fine Arts
    "bfa": "BACHELOR OF FINE ARTS",
    "mfa": "MASTER OF FINE ARTS",
    # Nursing
    "bsc nursing": "B.SC. NURSING",
    "gnm": "GENERAL NURSING AND MIDWIFERY",
    # NOTE: Single common English words like "commerce", "arts", "science",
    # "law", "computer" are intentionally NOT expanded here because they
    # appear in college names (e.g. "Gujarat Commerce College") and
    # expanding them before college resolution corrupts the college name.
    # The query_planner._PROGRAM_ALIASES handles program detection from
    # these plain words independently after college resolution.
    "nursing": "B.SC. NURSING",
}

# ── Common ASR phonetic mistakes (English accent variants) ──────────────────
# These arise from Indian English speech recognition errors
ASR_PHONETIC_ALIASES: Dict[str, str] = {
    # University name phonetic errors
    "saurashtra": "SAURASHTRA UNIVERSITY",
    "sourastra": "SAURASHTRA UNIVERSITY",
    "saurastra": "SAURASHTRA UNIVERSITY",
    "saurashra": "SAURASHTRA UNIVERSITY",
    "hemchandracharya": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN",
    "hemchandra": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN",
    "krantiguru": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY",
    # Common English→Indian accent ASR variants
    "collage": "college",
    "colege": "college",
    "coledge": "college",
    "univercity": "university",
    "universty": "university",
    "univeristy": "university",
    "addmission": "admission",
    "admision": "admission",
    "cutof": "cutoff",
    "cut off": "cutoff",
    "enginering": "engineering",
    "engeenering": "engineering",
    "engeneering": "engineering",
    "manegment": "management",
    "managment": "management",
    "comercial": "commerce",
    "comerce": "commerce",
    "infrastucture": "infrastructure",
    "infrastrucutre": "infrastructure",
    "hostle": "hostel",
    "hosttel": "hostel",
    "scolarship": "scholarship",
    "sholarship": "scholarship",
    "faculity": "faculty",
    "facalty": "faculty",
    "labratry": "laboratory",
    "labratory": "laboratory",
    "librery": "library",
    "libary": "library",
    "placment": "placement",
    "plcament": "placement",
    "afiliation": "affiliation",
    "afilliation": "affiliation",
    "govenment": "government",
    "goverment": "government",
    "scince": "science",
    "sience": "science",
}

# ── Gujarati common words transliterated (ASR output from Gujarati speech) ──
# Maps common Gujarati words that may appear in queries to their English equivalents
GUJARATI_WORD_ALIASES: Dict[str, str] = {
    # Location prepositions / particles
    "ma": "in",           # "Ahmedabad ma" = "in Ahmedabad"
    "ni": "of",           # "GTU ni college" = "college of GTU"
    "no": "of",
    "na": "of",
    "thi": "from",
    "mate": "for",
    "ane": "and",
    "ke": "and",          # Hindi: "aur" / Gujarati "ane"
    # Common query words in Gujarati
    "college": "college",
    "vishwavidyalaya": "university",
    "pravesh": "admission",
    "merit": "merit",
    "shikshan": "education",
    "abhyas": "course",
    "jilla": "district",
    "shaher": "city",
    # Hindi multi-word phrases (processed before single words due to length-sorted matching)
    "ladkiyon ke liye": "for girls",
    "ladkon ke liye": "for boys",
    "ke liye": "for",     # "girls ke liye" = "for girls"
    "ki fees": "fees",
    "ki jankari": "information",
    "ki jaankari": "information",
    "se kam": "less than",
    "se jyada": "more than",
    "se adhik": "more than",
    "se zyada": "more than",
    "ke saath": "with",
    "ke sath": "with",
    "ke baad": "after",
    "ke upar": "above",
    "ke neeche": "below",
    "ke andar": "inside",
    "ke under": "under",   # "GTU ke under" = "under GTU"
    "ke baare mein": "",    # "M N College ke baare mein" → "M N College"
    "ke baare": "",
    "ke vishay mein": "",
    "milta hai": "available",
    "milti hai": "available",
    "milte hain": "available",
    "milega": "available",
    "milegi": "available",
    "kahan milta": "where available",
    "kahan milegi": "where available",
    "kahan hain": "where",
    "saal ka": "year",
    "saal ki": "year",
    "saal ke": "year",
    "kitne saal": "how many years",
    "kya hai": "",
    "kya hain": "",
    "kya hogi": "",
    "kitni hai": "",
    "kitna hai": "",
    "batao bhai": "",
    "bata do": "",
    # Hindi common query words (single tokens)
    "mein": "in",
    "ka": "of",
    "ki": "of",
    "ke": "of",
    "liye": "for",        # "ladkiyon liye" fallback
    "aur": "and",
    "wale": "",           # "Ahmedabad wale college" → "Ahmedabad college"
    "wali": "",
    "koi": "",
    "kaunse": "",
    "kaun": "",
    "kitna": "",
    "kitne": "",
    "fees": "fees",
    "kitni": "",
    "hai": "",
    "hain": "",
    "hogi": "",
    "kya": "",
    "kaise": "",
    "kaisi": "",
    "kab": "",
    "batao": "",
    "bataiye": "",
    "vidyalay": "school",
    "mahavidyalaya": "college",
    "vishvavidhyalay": "university",
    "pravesh": "admission",
    "ank": "marks",
    "ankpatti": "marksheet",
    "chhatravritti": "scholarship",
    "ladkiyon": "girls",
    "ladkon": "boys",
    "vanijya": "commerce",
    "vigyan": "science",
    "vidnyan": "science",
    "kala": "arts",
    "kanoon": "law",
    "kanun": "law",
    "prabandhan": "management",
    "shikshan": "education",
    "private": "self finance",
    "sarkari": "government",
    "aided": "grant in aid",
}

# ---------------------------------------------------------------------------
# 2. Language detection
# ---------------------------------------------------------------------------

def detect_language(text: str) -> str:
    """
    Detect the language of the input text.
    Returns: "en" | "hi" | "gu" | "unknown"

    Strategy:
      1. Unicode block analysis (fast, no dependencies)
      2. Optionally backed by langdetect library if installed
    """
    # Check for Devanagari (Hindi): U+0900–U+097F
    devanagari = sum(1 for ch in text if '\u0900' <= ch <= '\u097F')
    # Check for Gujarati script: U+0A80–U+0AFF
    gujarati   = sum(1 for ch in text if '\u0A80' <= ch <= '\u0AFF')
    total      = len(text.replace(" ", "")) or 1

    if gujarati / total > 0.15:
        return "gu"
    if devanagari / total > 0.15:
        return "hi"

    # Try langdetect if available (handles Romanised Hindi/Gujarati)
    try:
        from langdetect import detect
        lang = detect(text)
        if lang in ("hi", "gu", "en"):
            return lang
        # langdetect sometimes says "mr" or "ne" for Gujarati/Hindi Romanised
        return "en"
    except Exception:
        return "en"


# ---------------------------------------------------------------------------
# 3. Script transliteration (Gujarati/Hindi Unicode → Latin)
# ---------------------------------------------------------------------------

def transliterate_to_latin(text: str) -> str:
    """
    Convert Gujarati or Hindi Unicode text to approximate Latin representation
    using the indic-transliteration library.  Falls back to unicodedata
    normalization if the library is not installed.
    """
    try:
        from indic_transliteration import sanscript
        from indic_transliteration.sanscript import transliterate

        # Detect script
        gujarati_chars = sum(1 for ch in text if '\u0A80' <= ch <= '\u0AFF')
        devanagari_chars = sum(1 for ch in text if '\u0900' <= ch <= '\u097F')

        if gujarati_chars > devanagari_chars:
            return transliterate(text, sanscript.GUJARATI, sanscript.ITRANS).lower()
        elif devanagari_chars > 0:
            return transliterate(text, sanscript.DEVANAGARI, sanscript.ITRANS).lower()
        return text

    except ImportError:
        # Fallback: unicode normalization strips diacritics (partial help)
        nfkd = unicodedata.normalize("NFKD", text)
        ascii_text = nfkd.encode("ascii", "ignore").decode("ascii")
        return ascii_text if ascii_text.strip() else text


# ---------------------------------------------------------------------------
# 4. Abbreviation expansion
# ---------------------------------------------------------------------------

def expand_abbreviations(text: str) -> str:
    """
    Expand university / course abbreviations to their full canonical names.
    Operates on the lowercased version but returns title-cased expansions.
    """
    lower = text.lower().strip()

    # Check full-string match first (e.g., "GTU" alone)
    if lower in UNIVERSITY_ALIASES:
        return UNIVERSITY_ALIASES[lower]
    if lower in COURSE_ALIASES:
        return COURSE_ALIASES[lower]

    # Replace abbreviations that appear as whole words in the text
    tokens = re.split(r'(\s+)', lower)   # split preserving whitespace
    result_tokens = []
    i = 0
    while i < len(tokens):
        token = tokens[i].strip()
        if not token:
            result_tokens.append(tokens[i])
            i += 1
            continue

        # Try two-word phrases first (e.g., "b tech", "m tech")
        if i + 2 < len(tokens):
            two_word = (tokens[i] + " " + tokens[i+2]).strip()
            if two_word in COURSE_ALIASES:
                result_tokens.append(COURSE_ALIASES[two_word])
                i += 3
                continue

        # Single token
        if token in UNIVERSITY_ALIASES:
            result_tokens.append(UNIVERSITY_ALIASES[token])
        elif token in COURSE_ALIASES:
            result_tokens.append(COURSE_ALIASES[token])
        else:
            result_tokens.append(tokens[i])
        i += 1

    return "".join(result_tokens)


# ---------------------------------------------------------------------------
# 5. Alias resolution (city names, ASR phonetics, Gujarati/Hindi words)
# ---------------------------------------------------------------------------

def resolve_aliases(text: str) -> str:
    """
    Replace known regional / ASR variant spellings with canonical forms.
    Returns the text with substitutions applied.
    """
    lower = text.lower()

    # Merge all alias maps: city > asr_phonetic > gujarati_words
    # (process city/university aliases as multi-word phrases first)
    combined: Dict[str, str] = {}
    combined.update(ASR_PHONETIC_ALIASES)
    combined.update(GUJARATI_WORD_ALIASES)
    combined.update(CITY_ALIASES)

    # Sort by length descending so longer phrases are matched first
    for alias, canonical in sorted(combined.items(), key=lambda x: -len(x[0])):
        pattern = r'\b' + re.escape(alias) + r'\b'
        lower = re.sub(pattern, canonical.lower(), lower, flags=re.IGNORECASE)

    return lower


# ---------------------------------------------------------------------------
# 6. Text normalisation (final clean-up)
# ---------------------------------------------------------------------------

def normalize_text(text: str) -> str:
    """
    Lowercase, collapse whitespace, remove leading/trailing punctuation.
    Preserves dots inside abbreviations (B.Tech, M.Sc).
    Also deduplicates consecutive repeated words that arise from alias
    expansion (e.g. "GTU" → "Gujarat Technological University" + "university"
    remaining from the original query → "...University university...").
    """
    text = text.strip()
    # Collapse multiple spaces/newlines
    text = re.sub(r'\s+', ' ', text)
    # Remove leading/trailing punctuation that is not alpha-numeric
    text = text.strip('.,;:!?"\'-')
    # Deduplicate consecutive identical words (case-insensitive)
    text = re.sub(
        r'\b(\w+)\s+\1\b',
        r'\1',
        text,
        flags=re.IGNORECASE,
    )
    # One more whitespace collapse after dedup
    text = re.sub(r'\s+', ' ', text).strip()
    return text


# ---------------------------------------------------------------------------
# 7. Full pipeline
# ---------------------------------------------------------------------------

def process_query(raw_query: str) -> Tuple[str, str, Dict]:
    """
    Full pre-processing pipeline.

    Parameters
    ----------
    raw_query : raw user input (may be English, Hindi, or Gujarati)

    Returns
    -------
    (corrected_query, detected_language, metadata)
      corrected_query  – cleaned, normalised query ready for embedding
      detected_language – "en" | "hi" | "gu" | "unknown"
      metadata          – {"original": str, "transliterated": str,
                           "after_alias": str, "language": str}
    """
    original = raw_query

    # Step 1 – Detect language
    lang = detect_language(raw_query)

    # Step 2 – Transliterate if Indic script
    if lang in ("gu", "hi"):
        transliterated = transliterate_to_latin(raw_query)
    else:
        transliterated = raw_query

    # Step 3 – Resolve Gujarati/Hindi particles and city/ASR aliases
    after_alias = resolve_aliases(transliterated)

    # Step 4 – Expand abbreviations
    after_expand = expand_abbreviations(after_alias)

    # Step 5 – Final normalisation
    corrected = normalize_text(after_expand)

    metadata = {
        "original": original,
        "detected_language": lang,
        "transliterated": transliterated if lang in ("gu", "hi") else None,
        "after_alias_resolution": after_alias,
        "corrected_query": corrected,
    }

    return corrected, lang, metadata