| import unicodedata |
| import regex as re |
|
|
|
|
| class SupernovaNepaliNormalizer: |
|
|
| def __init__(self): |
| self.zero_width_rules = {'\u200c': '', '\u200d': ''} |
|
|
| self.punctuation_rules = {'—': '-', '–': '-', '“': '"', '”': '"', '‘': "'", '’': "'"} |
|
|
| def normalize(self, text): |
|
|
| if text is None: |
| return "" |
|
|
| text = str(text) |
|
|
| text = unicodedata.normalize("NFC", text) |
|
|
| for old, new in self.zero_width_rules.items(): |
| text = text.replace(old, new) |
|
|
| for old, new in self.punctuation_rules.items(): |
| text = text.replace(old, new) |
|
|
| text = text.replace("\\r\\n", "\\n") |
| text = text.replace("\\r", "\\n") |
| text = text.replace("\\t", " ") |
|
|
| text = re.sub(r" {2,}", " ", text) |
| text = re.sub(r" *\\n *", "\\n", text) |
| text = re.sub(r"\\n{3,}", "\\n\\n", text) |
|
|
| return unicodedata.normalize("NFC", text).strip() |
|
|
