rag/util_text.py

import re

_WS = re.compile(r"\s+")
def clean_text(x: str) -> str:
    if not x:
        return ""
    t = x.replace("\u3000", " ").strip()
    t = _WS.sub(" ", t)
    return t