File size: 3,220 Bytes
33baa69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | """Canonicalize text so several RAID attacks are defeated before the model sees it.
Applied at BOTH train and inference (the invariant that makes it work): if every
input is mapped to the same ASCII-ish canonical form, then RAID's homoglyph,
zero-width-space, and whitespace attacks become no-ops — the perturbed and clean
versions collapse to the same string, so the detector never has to spend
capacity learning them. Word-level attacks (synonym, misspelling, paraphrase)
are deliberately left intact; those are handled by training data, not here.
Keep this module dependency-free and fast: it runs on every request.
"""
from __future__ import annotations
import unicodedata
# Zero-width and invisible formatting characters RAID (and real humanizers)
# inject to break tokenization. Deleted outright.
_ZERO_WIDTH_CHARS = (
"" # zero-width space (RAID's zero_width_space attack)
"" # zero-width non-joiner
"" # zero-width joiner
"" # word joiner
"" # zero-width no-break space / BOM
"" # soft hyphen
)
_ZERO_WIDTH_TABLE = {ord(char): None for char in _ZERO_WIDTH_CHARS}
# Common confusable code points mapped back to their ASCII twin. Covers the
# Cyrillic and Greek look-alikes RAID's homoglyph attack uses plus the fancy
# quotes/dashes that leak in from copy-paste. NFKC (below) handles most width/
# ligature variants; this table covers the cross-script look-alikes it does not.
_HOMOGLYPH_MAP = {
"а": "a", "е": "e", "о": "o", "р": "p", "с": "c",
"у": "y", "х": "x", "і": "i", "ј": "j", "һ": "h",
"А": "A", "В": "B", "Е": "E", "К": "K", "М": "M",
"Н": "H", "О": "O", "Р": "P", "С": "C", "Т": "T",
"Х": "X", "Α": "A", "Β": "B", "Ε": "E", "Ζ": "Z",
"Η": "H", "Ι": "I", "Κ": "K", "Μ": "M", "Ν": "N",
"Ο": "O", "Ρ": "P", "Τ": "T", "Υ": "Y", "Χ": "X",
"ο": "o", "α": "a", "‘": "'", "’": "'", "“": '"',
"”": '"', "–": "-", "—": "-", " ": " ",
}
_HOMOGLYPH_TABLE = {ord(source): target for source, target in _HOMOGLYPH_MAP.items()}
def strip_zero_width(text: str) -> str:
"""Remove zero-width / invisible formatting characters."""
return text.translate(_ZERO_WIDTH_TABLE)
def fold_homoglyphs(text: str) -> str:
"""Map cross-script look-alike characters back to their ASCII twin."""
return text.translate(_HOMOGLYPH_TABLE)
def collapse_whitespace(text: str) -> str:
"""Collapse any run of whitespace to a single space and trim the ends."""
return " ".join(text.split())
def normalize_text(text: str) -> str:
"""Canonicalize one string: NFKC, strip invisibles, fold look-alikes, dewhite.
Order matters. NFKC first folds width/ligature/compatibility variants; then
invisible characters are deleted (before whitespace collapse, so a run they
split is rejoined); then cross-script look-alikes are folded to ASCII; then
whitespace is normalised last so paragraph-insert and whitespace attacks
both reduce to single spaces.
"""
if not text:
return ""
folded = unicodedata.normalize("NFKC", text)
folded = strip_zero_width(folded)
folded = fold_homoglyphs(folded)
return collapse_whitespace(folded)
|