import unicodedata import regex as re class SupernovaNepaliNormalizer: def __init__(self): self.zero_width_rules = {'\u200c': '', '\u200d': ''} self.punctuation_rules = {'—': '-', '–': '-', '“': '"', '”': '"', '‘': "'", '’': "'"} def normalize(self, text): if text is None: return "" text = str(text) text = unicodedata.normalize("NFC", text) for old, new in self.zero_width_rules.items(): text = text.replace(old, new) for old, new in self.punctuation_rules.items(): text = text.replace(old, new) text = text.replace("\\r\\n", "\\n") text = text.replace("\\r", "\\n") text = text.replace("\\t", " ") text = re.sub(r" {2,}", " ", text) text = re.sub(r" *\\n *", "\\n", text) text = re.sub(r"\\n{3,}", "\\n\\n", text) return unicodedata.normalize("NFC", text).strip()