""" GCAS Search Engine – Query Normalizer ====================================== Handles three classes of input noise before the query reaches FAISS: 1. Language detection – detects English / Hindi / Gujarati 2. Script transliteration – Gujarati/Hindi Unicode → Latin (for fuzzy matching) 3. Alias resolution – maps regional spellings, ASR phonetic variants, and common abbreviations to canonical forms used in the database 4. Text normalisation – lowercase, collapse whitespace, strip punctuation Pipeline -------- raw query → detect_language() → transliterate_to_latin() (only if Gujarati/Hindi Unicode) → expand_abbreviations() → resolve_aliases() → normalize_text() → cleaned query string + metadata dict """ from __future__ import annotations import re import unicodedata from typing import Dict, Optional, Tuple # --------------------------------------------------------------------------- # 1. Known entity alias tables # Keys are lowercase, stripped variants (incl. common ASR mistakes) # Values are the canonical form found in the database # --------------------------------------------------------------------------- # ── Districts & Cities ────────────────────────────────────────────────────── CITY_ALIASES: Dict[str, str] = { # Ahmedabad variants (Gujarati: અમદાવાદ, Hindi: अहमदाबाद) "amdavad": "Ahmedabad", "amdabad": "Ahmedabad", "ahemadabad": "Ahmedabad", "ahemdabad": "Ahmedabad", "ahmadabad": "Ahmedabad", "ahmedabaad": "Ahmedabad", "ahmed abad": "Ahmedabad", "ahmedabad": "Ahmedabad", # Vadodara / Baroda "baroda": "Vadodara", "vadodra": "Vadodara", "vadodar": "Vadodara", "barod": "Vadodara", # Surat "soorat": "Surat", "sorat": "Surat", "suret": "Surat", # Rajkot "rajkote": "Rajkot", "rajkott": "Rajkot", "raj kot": "Rajkot", # Bhavnagar "bhawnagar": "Bhavnagar", "bhavanagar": "Bhavnagar", "bhavnagar": "Bhavnagar", "bhaunagar": "Bhavnagar", # Gandhinagar "gandhi nagar": "Gandhinagar", "gandhinagr": "Gandhinagar", # Mehsana "mahesana": "Mehsana", "mehsana": "Mehsana", "mehasana": "Mehsana", "mahisana": "Mehsana", # Anand "aanand": "Anand", "annad": "Anand", # Kutch / Kachchh "kutch": "Kutch", "kachchh": "Kutch", "kaach": "Kutch", "kachch": "Kutch", # Junagadh "junagarh": "Junagadh", "junagar": "Junagadh", "junaagadh": "Junagadh", # Jamnagar "jamnagar": "Jamnagar", "jamnagr": "Jamnagar", # Surendranagar "surendranagar": "Surendra Nagar", "surendra nagar": "Surendra Nagar", # GirSomnath "gir somnath": "GirSomnath", "girsomnath": "GirSomnath", # Banaskantha "banaskanta": "Banaskantha", "banas kantha": "Banaskantha", # Sabarkantha "sabar kantha": "Sabarkantha", "sabarkanta": "Sabarkantha", # Navsari "navsari": "Navsari", "navsarri": "Navsari", # Valsad "valsad": "Valsad", "valsaad": "Valsad", # Patan "patan": "Patan", # Bharuch "bharuch": "Bharuch", "bharooch": "Bharuch", "broach": "Bharuch", # Amreli "amreli": "Amreli", "amrely": "Amreli", # Porbandar "porbandar": "Porbandar", "porbander": "Porbandar", # Dahod "dahod": "Dahod", "daaod": "Dahod", # Panchmahal "panch mahal": "Panchmahal", "panchmahal": "Panchmahal", "panchmahals": "Panchmahal", # Chhota Udepur "chhota udepur": "Chhota Udepur", "chota udepur": "Chhota Udepur", # Narmada "narmada": "Narmada", # Navsari "navsari": "Navsari", # Tapi "tapi": "TAPI", # Aravalli "aravalli": "Aravalli", "aravali": "Aravalli", # Morbi "morbi": "Morbi", "morvi": "Morbi", # Botad "botad": "Botad", # Mahisagar "mahisagar": "Mahisagar", # Devbhumi Dwarka "dwarka": "DEVBHUMI DWARKA", "devbhumi dwarka": "DEVBHUMI DWARKA", "deobhumi dwarka": "DEVBHUMI DWARKA", # Dang "dang": "Dang", # Kheda "kheda": "Kheda", "kaira": "Kheda", "nadiad": "Kheda", "visnagar": "Mehsana", "vallabh vidyanagar": "Anand", "vallabhpur": "Anand", "vapi": "Valsad", "godhra": "Panchmahal", "palanpur": "Banaskantha", "ankleshwar": "Bharuch", "dahej": "Bharuch", "halol": "Panchmahal", } # ── University abbreviations & common mistakes ────────────────────────────── UNIVERSITY_ALIASES: Dict[str, str] = { "gtu": "Gujarat Technological University", "gujarat tech university": "Gujarat Technological University", "gujarat technical university": "Gujarat Technological University", "vnsgu": "VEER NARMAD SOUTH GUJARAT UNIVERSITY", "veer narmad university": "VEER NARMAD SOUTH GUJARAT UNIVERSITY", "south gujarat university": "VEER NARMAD SOUTH GUJARAT UNIVERSITY", "spu": "SARDAR PATEL UNIVERSITY", "sardar patel university": "SARDAR PATEL UNIVERSITY", "hngu": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN", "north gujarat university": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN", "hemchandracharya university": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN", "msu": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA", "maharaja sayajirao university": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA", "ms university": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA", "msu baroda": "THE MAHARAJA SAYAJIRAO UNIVERSITY OF BARODA", "gu": "GUJARAT UNIVERSITY", "gujarat university": "GUJARAT UNIVERSITY", "sau": "SAURASHTRA UNIVERSITY", "saurashtra university": "SAURASHTRA UNIVERSITY", "saurastra university": "SAURASHTRA UNIVERSITY", "ksv": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY", "kachchh university": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY", "kutch university": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY", "mkbu": "MAHARAJA KRISHNAKUMARSINHJI BHAVNAGAR UNIVERSITY", "bhavnagar university": "MAHARAJA KRISHNAKUMARSINHJI BHAVNAGAR UNIVERSITY", "bknmu": "BHAKTA KAVI NARSINH MEHTA UNIVERSITY", "narsinh mehta university": "BHAKTA KAVI NARSINH MEHTA UNIVERSITY", "junagadh university": "BHAKTA KAVI NARSINH MEHTA UNIVERSITY", "sggu": "SHRI GOVIND GURU UNIVERSITY", "govind guru university": "SHRI GOVIND GURU UNIVERSITY", "sssu": "SHREE SOMNATH SANSKRIT UNIVERSITY, VERAVAL", "somnath university": "SHREE SOMNATH SANSKRIT UNIVERSITY, VERAVAL", "iite": "INDIAN INSTITUTE OF TEACHER EDUCATION", } # ── Course / Program abbreviations ────────────────────────────────────────── COURSE_ALIASES: Dict[str, str] = { # Engineering "btech": "B.TECH", "b tech": "B.TECH", "b.tech.": "B.TECH", "be": "BACHELOR OF ENGINEERING", "bachelor of engineering": "BACHELOR OF ENGINEERING", "mtech": "M.TECH", "m tech": "M.TECH", "me": "MASTER OF ENGINEERING", # Management "mba": "MASTER OF BUSINESS ADMINISTRATION", "bba": "BACHELOR OF BUSINESS ADMINISTRATION", "pgdm": "POST GRADUATE DIPLOMA IN MANAGEMENT", # Computer "mca": "MASTER OF COMPUTER APPLICATIONS", "bca": "BACHELOR OF COMPUTER APPLICATIONS", # Commerce "bcom": "B.COM.", "b com": "B.COM.", "b.com": "B.COM.", "mcom": "M.COM.", "m com": "M.COM.", # Science "bsc": "B.SC.", "b sc": "B.SC.", "b.sc": "B.SC.", "msc": "M.SC.", "m sc": "M.SC.", "m.sc": "M.SC.", # Arts "ba": "BACHELOR OF ARTS", "ma": "MASTER OF ARTS", # Law "llb": "BACHELOR OF LAWS", "ll.b": "BACHELOR OF LAWS", "llm": "MASTER OF LAWS", "ll.m": "MASTER OF LAWS", # Medicine / Pharmacy "mbbs": "BACHELOR OF MEDICINE AND BACHELOR OF SURGERY", "bpharm": "B.PHARM.", "b pharm": "B.PHARM.", "b.pharm": "B.PHARM.", "mpharm": "M.PHARM.", "m pharm": "M.PHARM.", "dpharm": "D.PHARM.", "d pharm": "D.PHARM.", # Architecture / Design "barch": "B.ARCH.", "b arch": "B.ARCH.", "march": "M.ARCH.", "bdes": "B.DES.", "mdes": "M.DES.", # Education "bed": "BACHELOR OF EDUCATION", "b ed": "BACHELOR OF EDUCATION", "med": "MASTER OF EDUCATION", "m ed": "MASTER OF EDUCATION", "bped": "BACHELOR OF PHYSICAL EDUCATION", # Social work "msw": "MASTER OF SOCIAL WORK", "bsw": "BACHELOR OF SOCIAL WORK", # Research "phd": "DOCTOR OF PHILOSOPHY", "ph d": "DOCTOR OF PHILOSOPHY", "ph.d": "DOCTOR OF PHILOSOPHY", "mphil": "MASTER OF PHILOSOPHY", # Hotel / Hospitality "bhmct": "BACHELOR OF HOTEL MANAGEMENT AND CATERING TECHNOLOGY", "mhmct": "MASTER OF HOTEL MANAGEMENT AND CATERING TECHNOLOGY", # Fine Arts "bfa": "BACHELOR OF FINE ARTS", "mfa": "MASTER OF FINE ARTS", # Nursing "bsc nursing": "B.SC. NURSING", "gnm": "GENERAL NURSING AND MIDWIFERY", # NOTE: Single common English words like "commerce", "arts", "science", # "law", "computer" are intentionally NOT expanded here because they # appear in college names (e.g. "Gujarat Commerce College") and # expanding them before college resolution corrupts the college name. # The query_planner._PROGRAM_ALIASES handles program detection from # these plain words independently after college resolution. "nursing": "B.SC. NURSING", } # ── Common ASR phonetic mistakes (English accent variants) ────────────────── # These arise from Indian English speech recognition errors ASR_PHONETIC_ALIASES: Dict[str, str] = { # University name phonetic errors "saurashtra": "SAURASHTRA UNIVERSITY", "sourastra": "SAURASHTRA UNIVERSITY", "saurastra": "SAURASHTRA UNIVERSITY", "saurashra": "SAURASHTRA UNIVERSITY", "hemchandracharya": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN", "hemchandra": "HEMCHANDRACHARYA NORTH GUJARAT UNIVERSITY, PATAN", "krantiguru": "KRANTIGURU SHYAMJI KRISHNA VERMA KACHCHH UNIVERSITY", # Common English→Indian accent ASR variants "collage": "college", "colege": "college", "coledge": "college", "univercity": "university", "universty": "university", "univeristy": "university", "addmission": "admission", "admision": "admission", "cutof": "cutoff", "cut off": "cutoff", "enginering": "engineering", "engeenering": "engineering", "engeneering": "engineering", "manegment": "management", "managment": "management", "comercial": "commerce", "comerce": "commerce", "infrastucture": "infrastructure", "infrastrucutre": "infrastructure", "hostle": "hostel", "hosttel": "hostel", "scolarship": "scholarship", "sholarship": "scholarship", "faculity": "faculty", "facalty": "faculty", "labratry": "laboratory", "labratory": "laboratory", "librery": "library", "libary": "library", "placment": "placement", "plcament": "placement", "afiliation": "affiliation", "afilliation": "affiliation", "govenment": "government", "goverment": "government", "scince": "science", "sience": "science", } # ── Gujarati common words transliterated (ASR output from Gujarati speech) ── # Maps common Gujarati words that may appear in queries to their English equivalents GUJARATI_WORD_ALIASES: Dict[str, str] = { # Location prepositions / particles "ma": "in", # "Ahmedabad ma" = "in Ahmedabad" "ni": "of", # "GTU ni college" = "college of GTU" "no": "of", "na": "of", "thi": "from", "mate": "for", "ane": "and", "ke": "and", # Hindi: "aur" / Gujarati "ane" # Common query words in Gujarati "college": "college", "vishwavidyalaya": "university", "pravesh": "admission", "merit": "merit", "shikshan": "education", "abhyas": "course", "jilla": "district", "shaher": "city", # Hindi multi-word phrases (processed before single words due to length-sorted matching) "ladkiyon ke liye": "for girls", "ladkon ke liye": "for boys", "ke liye": "for", # "girls ke liye" = "for girls" "ki fees": "fees", "ki jankari": "information", "ki jaankari": "information", "se kam": "less than", "se jyada": "more than", "se adhik": "more than", "se zyada": "more than", "ke saath": "with", "ke sath": "with", "ke baad": "after", "ke upar": "above", "ke neeche": "below", "ke andar": "inside", "ke under": "under", # "GTU ke under" = "under GTU" "ke baare mein": "", # "M N College ke baare mein" → "M N College" "ke baare": "", "ke vishay mein": "", "milta hai": "available", "milti hai": "available", "milte hain": "available", "milega": "available", "milegi": "available", "kahan milta": "where available", "kahan milegi": "where available", "kahan hain": "where", "saal ka": "year", "saal ki": "year", "saal ke": "year", "kitne saal": "how many years", "kya hai": "", "kya hain": "", "kya hogi": "", "kitni hai": "", "kitna hai": "", "batao bhai": "", "bata do": "", # Hindi common query words (single tokens) "mein": "in", "ka": "of", "ki": "of", "ke": "of", "liye": "for", # "ladkiyon liye" fallback "aur": "and", "wale": "", # "Ahmedabad wale college" → "Ahmedabad college" "wali": "", "koi": "", "kaunse": "", "kaun": "", "kitna": "", "kitne": "", "fees": "fees", "kitni": "", "hai": "", "hain": "", "hogi": "", "kya": "", "kaise": "", "kaisi": "", "kab": "", "batao": "", "bataiye": "", "vidyalay": "school", "mahavidyalaya": "college", "vishvavidhyalay": "university", "pravesh": "admission", "ank": "marks", "ankpatti": "marksheet", "chhatravritti": "scholarship", "ladkiyon": "girls", "ladkon": "boys", "vanijya": "commerce", "vigyan": "science", "vidnyan": "science", "kala": "arts", "kanoon": "law", "kanun": "law", "prabandhan": "management", "shikshan": "education", "private": "self finance", "sarkari": "government", "aided": "grant in aid", } # --------------------------------------------------------------------------- # 2. Language detection # --------------------------------------------------------------------------- def detect_language(text: str) -> str: """ Detect the language of the input text. Returns: "en" | "hi" | "gu" | "unknown" Strategy: 1. Unicode block analysis (fast, no dependencies) 2. Optionally backed by langdetect library if installed """ # Check for Devanagari (Hindi): U+0900–U+097F devanagari = sum(1 for ch in text if '\u0900' <= ch <= '\u097F') # Check for Gujarati script: U+0A80–U+0AFF gujarati = sum(1 for ch in text if '\u0A80' <= ch <= '\u0AFF') total = len(text.replace(" ", "")) or 1 if gujarati / total > 0.15: return "gu" if devanagari / total > 0.15: return "hi" # Try langdetect if available (handles Romanised Hindi/Gujarati) try: from langdetect import detect lang = detect(text) if lang in ("hi", "gu", "en"): return lang # langdetect sometimes says "mr" or "ne" for Gujarati/Hindi Romanised return "en" except Exception: return "en" # --------------------------------------------------------------------------- # 3. Script transliteration (Gujarati/Hindi Unicode → Latin) # --------------------------------------------------------------------------- def transliterate_to_latin(text: str) -> str: """ Convert Gujarati or Hindi Unicode text to approximate Latin representation using the indic-transliteration library. Falls back to unicodedata normalization if the library is not installed. """ try: from indic_transliteration import sanscript from indic_transliteration.sanscript import transliterate # Detect script gujarati_chars = sum(1 for ch in text if '\u0A80' <= ch <= '\u0AFF') devanagari_chars = sum(1 for ch in text if '\u0900' <= ch <= '\u097F') if gujarati_chars > devanagari_chars: return transliterate(text, sanscript.GUJARATI, sanscript.ITRANS).lower() elif devanagari_chars > 0: return transliterate(text, sanscript.DEVANAGARI, sanscript.ITRANS).lower() return text except ImportError: # Fallback: unicode normalization strips diacritics (partial help) nfkd = unicodedata.normalize("NFKD", text) ascii_text = nfkd.encode("ascii", "ignore").decode("ascii") return ascii_text if ascii_text.strip() else text # --------------------------------------------------------------------------- # 4. Abbreviation expansion # --------------------------------------------------------------------------- def expand_abbreviations(text: str) -> str: """ Expand university / course abbreviations to their full canonical names. Operates on the lowercased version but returns title-cased expansions. """ lower = text.lower().strip() # Check full-string match first (e.g., "GTU" alone) if lower in UNIVERSITY_ALIASES: return UNIVERSITY_ALIASES[lower] if lower in COURSE_ALIASES: return COURSE_ALIASES[lower] # Replace abbreviations that appear as whole words in the text tokens = re.split(r'(\s+)', lower) # split preserving whitespace result_tokens = [] i = 0 while i < len(tokens): token = tokens[i].strip() if not token: result_tokens.append(tokens[i]) i += 1 continue # Try two-word phrases first (e.g., "b tech", "m tech") if i + 2 < len(tokens): two_word = (tokens[i] + " " + tokens[i+2]).strip() if two_word in COURSE_ALIASES: result_tokens.append(COURSE_ALIASES[two_word]) i += 3 continue # Single token if token in UNIVERSITY_ALIASES: result_tokens.append(UNIVERSITY_ALIASES[token]) elif token in COURSE_ALIASES: result_tokens.append(COURSE_ALIASES[token]) else: result_tokens.append(tokens[i]) i += 1 return "".join(result_tokens) # --------------------------------------------------------------------------- # 5. Alias resolution (city names, ASR phonetics, Gujarati/Hindi words) # --------------------------------------------------------------------------- def resolve_aliases(text: str) -> str: """ Replace known regional / ASR variant spellings with canonical forms. Returns the text with substitutions applied. """ lower = text.lower() # Merge all alias maps: city > asr_phonetic > gujarati_words # (process city/university aliases as multi-word phrases first) combined: Dict[str, str] = {} combined.update(ASR_PHONETIC_ALIASES) combined.update(GUJARATI_WORD_ALIASES) combined.update(CITY_ALIASES) # Sort by length descending so longer phrases are matched first for alias, canonical in sorted(combined.items(), key=lambda x: -len(x[0])): pattern = r'\b' + re.escape(alias) + r'\b' lower = re.sub(pattern, canonical.lower(), lower, flags=re.IGNORECASE) return lower # --------------------------------------------------------------------------- # 6. Text normalisation (final clean-up) # --------------------------------------------------------------------------- def normalize_text(text: str) -> str: """ Lowercase, collapse whitespace, remove leading/trailing punctuation. Preserves dots inside abbreviations (B.Tech, M.Sc). Also deduplicates consecutive repeated words that arise from alias expansion (e.g. "GTU" → "Gujarat Technological University" + "university" remaining from the original query → "...University university..."). """ text = text.strip() # Collapse multiple spaces/newlines text = re.sub(r'\s+', ' ', text) # Remove leading/trailing punctuation that is not alpha-numeric text = text.strip('.,;:!?"\'-') # Deduplicate consecutive identical words (case-insensitive) text = re.sub( r'\b(\w+)\s+\1\b', r'\1', text, flags=re.IGNORECASE, ) # One more whitespace collapse after dedup text = re.sub(r'\s+', ' ', text).strip() return text # --------------------------------------------------------------------------- # 7. Full pipeline # --------------------------------------------------------------------------- def process_query(raw_query: str) -> Tuple[str, str, Dict]: """ Full pre-processing pipeline. Parameters ---------- raw_query : raw user input (may be English, Hindi, or Gujarati) Returns ------- (corrected_query, detected_language, metadata) corrected_query – cleaned, normalised query ready for embedding detected_language – "en" | "hi" | "gu" | "unknown" metadata – {"original": str, "transliterated": str, "after_alias": str, "language": str} """ original = raw_query # Step 1 – Detect language lang = detect_language(raw_query) # Step 2 – Transliterate if Indic script if lang in ("gu", "hi"): transliterated = transliterate_to_latin(raw_query) else: transliterated = raw_query # Step 3 – Resolve Gujarati/Hindi particles and city/ASR aliases after_alias = resolve_aliases(transliterated) # Step 4 – Expand abbreviations after_expand = expand_abbreviations(after_alias) # Step 5 – Final normalisation corrected = normalize_text(after_expand) metadata = { "original": original, "detected_language": lang, "transliterated": transliterated if lang in ("gu", "hi") else None, "after_alias_resolution": after_alias, "corrected_query": corrected, } return corrected, lang, metadata