File size: 948 Bytes
d16fb01 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | import unicodedata
import regex as re
class SupernovaNepaliNormalizer:
def __init__(self):
self.zero_width_rules = {'\u200c': '', '\u200d': ''}
self.punctuation_rules = {'—': '-', '–': '-', '“': '"', '”': '"', '‘': "'", '’': "'"}
def normalize(self, text):
if text is None:
return ""
text = str(text)
text = unicodedata.normalize("NFC", text)
for old, new in self.zero_width_rules.items():
text = text.replace(old, new)
for old, new in self.punctuation_rules.items():
text = text.replace(old, new)
text = text.replace("\\r\\n", "\\n")
text = text.replace("\\r", "\\n")
text = text.replace("\\t", " ")
text = re.sub(r" {2,}", " ", text)
text = re.sub(r" *\\n *", "\\n", text)
text = re.sub(r"\\n{3,}", "\\n\\n", text)
return unicodedata.normalize("NFC", text).strip()
|