Spaces:
Sleeping
Sleeping
| """ | |
| Synonyms and Hinglish normalization module. | |
| Maps Hinglish phrases and synonyms to canonical DataFrame operations and column concepts. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| # Hinglish to English operation mappings | |
| HINGLISH_TO_ENGLISH_OP: dict[str, str] = { | |
| "badhao": "increase", "badha do": "increase", "barha do": "increase", "barhao": "increase", | |
| "zyada karo": "increase", "bada karo": "increase", "badha dijiye": "increase", | |
| "ghatao": "decrease", "ghata do": "decrease", "kam karo": "decrease", "kam kar do": "decrease", | |
| "chhota karo": "decrease", "reduce": "decrease", "cut": "decrease", "minus karo": "decrease", "ghata dijiye": "decrease", | |
| "sirf": "filter", "dikhao": "filter", "show only": "filter", "show me": "filter", | |
| "bas": "filter", "wale dikhao": "filter", "jitne": "filter", "laao": "filter", | |
| "chota se bada": "sort_asc", "ascending": "sort_asc", "low to high": "sort_asc", "smallest first": "sort_asc", | |
| "bada se chota": "sort_desc", "descending": "sort_desc", "high to low": "sort_desc", "largest first": "sort_desc", | |
| "jod": "sum", "yog": "sum", "add up": "sum", "total batao": "sum", "kul": "sum", "jama": "sum", | |
| "samanya": "average", "average nikalo": "average", "avg": "average", | |
| "ginti": "count", "kitne": "count", "kitni rows": "count", "count karo": "count", "kitni": "count", | |
| "sabse chhota": "min", "lowest": "min", "kam se kam": "min", | |
| "sabse bada": "max", "highest": "max", "zyada se zyada": "max", | |
| "badlo": "find_replace", "change": "find_replace", "dhundho": "find_replace", "replace karo": "find_replace", | |
| "column hatao": "delete_column", "column delete karo": "delete_column", "remove column": "delete_column", | |
| "column ka naam badlo": "rename_column", "column rename karo": "rename_column", "naam badlo": "rename_column", | |
| "naya column banao": "add_column", "column add karo": "add_column", "new column": "add_column", | |
| "duplicate hatao": "remove_duplicates", "duplicates remove karo": "remove_duplicates", "unique rakho": "remove_duplicates", | |
| "type badlo": "cast_type", "data type change": "cast_type", "convert type": "cast_type", | |
| } | |
| # Common column name synonym mappings | |
| COMMON_COLUMN_SYNONYMS: dict[str, str] = { | |
| "vetan": "salary", "kamai": "salary", "paisa": "salary", "income": "salary", | |
| "umar": "age", "umra": "age", | |
| "naam": "name", | |
| "mulya": "price", "daam": "price", "keemat": "price", "cost": "price", | |
| "tareekh": "date", "din": "date", | |
| "shahar": "city", "shehar": "city", | |
| "desh": "country", | |
| "phone": "mobile", "mobile number": "mobile", "contact": "mobile", | |
| } | |
| class SynonymMapper: | |
| """Handles mapping of synonym phrases and normalizes Hinglish queries.""" | |
| def __init__(self, custom_column_synonyms: dict[str, str] | None = None): | |
| self.op_map = HINGLISH_TO_ENGLISH_OP | |
| self.col_map = {**COMMON_COLUMN_SYNONYMS, **(custom_column_synonyms or {})} | |
| def normalize_text(self, text: str) -> str: | |
| """Normalize general Hinglish operations and column names to canonical terms.""" | |
| text_lower = text.lower().strip() | |
| # 1. Normalize operations (longer/more specific phrases first to prevent partial match issues) | |
| sorted_ops = sorted(self.op_map.keys(), key=len, reverse=True) | |
| for h_op in sorted_ops: | |
| e_op = self.op_map[h_op] | |
| # Replace complete word/phrase boundaries where possible | |
| if h_op in text_lower: | |
| text_lower = re.sub(rf'\b{re.escape(h_op)}\b', e_op, text_lower) | |
| # 2. Normalize columns | |
| sorted_cols = sorted(self.col_map.keys(), key=len, reverse=True) | |
| for h_col in sorted_cols: | |
| e_col = self.col_map[h_col] | |
| if h_col in text_lower: | |
| text_lower = re.sub(rf'\b{re.escape(h_col)}\b', e_col, text_lower) | |
| return text_lower | |