Spaces:
Sleeping
Sleeping
File size: 3,900 Bytes
b336134 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | """
Synonyms and Hinglish normalization module.
Maps Hinglish phrases and synonyms to canonical DataFrame operations and column concepts.
"""
from __future__ import annotations
import re
# Hinglish to English operation mappings
HINGLISH_TO_ENGLISH_OP: dict[str, str] = {
"badhao": "increase", "badha do": "increase", "barha do": "increase", "barhao": "increase",
"zyada karo": "increase", "bada karo": "increase", "badha dijiye": "increase",
"ghatao": "decrease", "ghata do": "decrease", "kam karo": "decrease", "kam kar do": "decrease",
"chhota karo": "decrease", "reduce": "decrease", "cut": "decrease", "minus karo": "decrease", "ghata dijiye": "decrease",
"sirf": "filter", "dikhao": "filter", "show only": "filter", "show me": "filter",
"bas": "filter", "wale dikhao": "filter", "jitne": "filter", "laao": "filter",
"chota se bada": "sort_asc", "ascending": "sort_asc", "low to high": "sort_asc", "smallest first": "sort_asc",
"bada se chota": "sort_desc", "descending": "sort_desc", "high to low": "sort_desc", "largest first": "sort_desc",
"jod": "sum", "yog": "sum", "add up": "sum", "total batao": "sum", "kul": "sum", "jama": "sum",
"samanya": "average", "average nikalo": "average", "avg": "average",
"ginti": "count", "kitne": "count", "kitni rows": "count", "count karo": "count", "kitni": "count",
"sabse chhota": "min", "lowest": "min", "kam se kam": "min",
"sabse bada": "max", "highest": "max", "zyada se zyada": "max",
"badlo": "find_replace", "change": "find_replace", "dhundho": "find_replace", "replace karo": "find_replace",
"column hatao": "delete_column", "column delete karo": "delete_column", "remove column": "delete_column",
"column ka naam badlo": "rename_column", "column rename karo": "rename_column", "naam badlo": "rename_column",
"naya column banao": "add_column", "column add karo": "add_column", "new column": "add_column",
"duplicate hatao": "remove_duplicates", "duplicates remove karo": "remove_duplicates", "unique rakho": "remove_duplicates",
"type badlo": "cast_type", "data type change": "cast_type", "convert type": "cast_type",
}
# Common column name synonym mappings
COMMON_COLUMN_SYNONYMS: dict[str, str] = {
"vetan": "salary", "kamai": "salary", "paisa": "salary", "income": "salary",
"umar": "age", "umra": "age",
"naam": "name",
"mulya": "price", "daam": "price", "keemat": "price", "cost": "price",
"tareekh": "date", "din": "date",
"shahar": "city", "shehar": "city",
"desh": "country",
"phone": "mobile", "mobile number": "mobile", "contact": "mobile",
}
class SynonymMapper:
"""Handles mapping of synonym phrases and normalizes Hinglish queries."""
def __init__(self, custom_column_synonyms: dict[str, str] | None = None):
self.op_map = HINGLISH_TO_ENGLISH_OP
self.col_map = {**COMMON_COLUMN_SYNONYMS, **(custom_column_synonyms or {})}
def normalize_text(self, text: str) -> str:
"""Normalize general Hinglish operations and column names to canonical terms."""
text_lower = text.lower().strip()
# 1. Normalize operations (longer/more specific phrases first to prevent partial match issues)
sorted_ops = sorted(self.op_map.keys(), key=len, reverse=True)
for h_op in sorted_ops:
e_op = self.op_map[h_op]
# Replace complete word/phrase boundaries where possible
if h_op in text_lower:
text_lower = re.sub(rf'\b{re.escape(h_op)}\b', e_op, text_lower)
# 2. Normalize columns
sorted_cols = sorted(self.col_map.keys(), key=len, reverse=True)
for h_col in sorted_cols:
e_col = self.col_map[h_col]
if h_col in text_lower:
text_lower = re.sub(rf'\b{re.escape(h_col)}\b', e_col, text_lower)
return text_lower
|