"""Canonicalize text so several RAID attacks are defeated before the model sees it. Applied at BOTH train and inference (the invariant that makes it work): if every input is mapped to the same ASCII-ish canonical form, then RAID's homoglyph, zero-width-space, and whitespace attacks become no-ops — the perturbed and clean versions collapse to the same string, so the detector never has to spend capacity learning them. Word-level attacks (synonym, misspelling, paraphrase) are deliberately left intact; those are handled by training data, not here. Keep this module dependency-free and fast: it runs on every request. """ from __future__ import annotations import unicodedata # Zero-width and invisible formatting characters RAID (and real humanizers) # inject to break tokenization. Deleted outright. _ZERO_WIDTH_CHARS = ( "​" # zero-width space (RAID's zero_width_space attack) "‌" # zero-width non-joiner "‍" # zero-width joiner "⁠" # word joiner "" # zero-width no-break space / BOM "­" # soft hyphen ) _ZERO_WIDTH_TABLE = {ord(char): None for char in _ZERO_WIDTH_CHARS} # Common confusable code points mapped back to their ASCII twin. Covers the # Cyrillic and Greek look-alikes RAID's homoglyph attack uses plus the fancy # quotes/dashes that leak in from copy-paste. NFKC (below) handles most width/ # ligature variants; this table covers the cross-script look-alikes it does not. _HOMOGLYPH_MAP = { "а": "a", "е": "e", "о": "o", "р": "p", "с": "c", "у": "y", "х": "x", "і": "i", "ј": "j", "һ": "h", "А": "A", "В": "B", "Е": "E", "К": "K", "М": "M", "Н": "H", "О": "O", "Р": "P", "С": "C", "Т": "T", "Х": "X", "Α": "A", "Β": "B", "Ε": "E", "Ζ": "Z", "Η": "H", "Ι": "I", "Κ": "K", "Μ": "M", "Ν": "N", "Ο": "O", "Ρ": "P", "Τ": "T", "Υ": "Y", "Χ": "X", "ο": "o", "α": "a", "‘": "'", "’": "'", "“": '"', "”": '"', "–": "-", "—": "-", " ": " ", } _HOMOGLYPH_TABLE = {ord(source): target for source, target in _HOMOGLYPH_MAP.items()} def strip_zero_width(text: str) -> str: """Remove zero-width / invisible formatting characters.""" return text.translate(_ZERO_WIDTH_TABLE) def fold_homoglyphs(text: str) -> str: """Map cross-script look-alike characters back to their ASCII twin.""" return text.translate(_HOMOGLYPH_TABLE) def collapse_whitespace(text: str) -> str: """Collapse any run of whitespace to a single space and trim the ends.""" return " ".join(text.split()) def normalize_text(text: str) -> str: """Canonicalize one string: NFKC, strip invisibles, fold look-alikes, dewhite. Order matters. NFKC first folds width/ligature/compatibility variants; then invisible characters are deleted (before whitespace collapse, so a run they split is rejoined); then cross-script look-alikes are folded to ASCII; then whitespace is normalised last so paragraph-insert and whitespace attacks both reduce to single spaces. """ if not text: return "" folded = unicodedata.normalize("NFKC", text) folded = strip_zero_width(folded) folded = fold_homoglyphs(folded) return collapse_whitespace(folded)