HK-UTM-LLM / src /regx /normalize.py
GordonUK's picture
Create src/regx/normalize.py
550526b verified
Raw
History Blame Contribute Delete
369 Bytes
import re, unicodedata
WS_RE = re.compile(r"[ \t]+")
MULTI_NL_RE = re.compile(r"\n{3,}")
def normalize_text(s: str) -> str:
s = unicodedata.normalize("NFKC", s)
s = s.replace("\r\n", "\n").replace("\r", "\n")
s = WS_RE.sub(" ", s)
s = MULTI_NL_RE.sub("\n\n", s)
return s.strip()
def breadcrumb(path: list[str]) -> str:
return " > ".join(path)