code/normalization.py

import unicodedata


NORMALIZATION_VERSION = "nfkc-soft-separator-v1-20260612"
SOFT_SEPARATOR_CHARS = frozenset(
    "-_'`\\/|.,;:!?()[]{}<>\"\u2010\u2011\u2012\u2013\u2014\u2015\u2212"
)


def normalize_input(text: str) -> str:
    text = unicodedata.normalize("NFKC", text.strip()).lower()
    out = []
    for ch in text:
        if ch.isspace() or ch in SOFT_SEPARATOR_CHARS or unicodedata.category(ch) == "Cf":
            continue
        out.append(ch)
    return "".join(out)