File size: 3,220 Bytes
33baa69
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
"""Canonicalize text so several RAID attacks are defeated before the model sees it.

Applied at BOTH train and inference (the invariant that makes it work): if every
input is mapped to the same ASCII-ish canonical form, then RAID's homoglyph,
zero-width-space, and whitespace attacks become no-ops — the perturbed and clean
versions collapse to the same string, so the detector never has to spend
capacity learning them. Word-level attacks (synonym, misspelling, paraphrase)
are deliberately left intact; those are handled by training data, not here.

Keep this module dependency-free and fast: it runs on every request.
"""

from __future__ import annotations

import unicodedata

# Zero-width and invisible formatting characters RAID (and real humanizers)
# inject to break tokenization. Deleted outright.
_ZERO_WIDTH_CHARS = (
    "​"  # zero-width space (RAID's zero_width_space attack)
    "‌"  # zero-width non-joiner
    "‍"  # zero-width joiner
    "⁠"  # word joiner
    ""  # zero-width no-break space / BOM
    "­"  # soft hyphen
)
_ZERO_WIDTH_TABLE = {ord(char): None for char in _ZERO_WIDTH_CHARS}

# Common confusable code points mapped back to their ASCII twin. Covers the
# Cyrillic and Greek look-alikes RAID's homoglyph attack uses plus the fancy
# quotes/dashes that leak in from copy-paste. NFKC (below) handles most width/
# ligature variants; this table covers the cross-script look-alikes it does not.
_HOMOGLYPH_MAP = {
    "а": "a", "е": "e", "о": "o", "р": "p", "с": "c",
    "у": "y", "х": "x", "і": "i", "ј": "j", "һ": "h",
    "А": "A", "В": "B", "Е": "E", "К": "K", "М": "M",
    "Н": "H", "О": "O", "Р": "P", "С": "C", "Т": "T",
    "Х": "X", "Α": "A", "Β": "B", "Ε": "E", "Ζ": "Z",
    "Η": "H", "Ι": "I", "Κ": "K", "Μ": "M", "Ν": "N",
    "Ο": "O", "Ρ": "P", "Τ": "T", "Υ": "Y", "Χ": "X",
    "ο": "o", "α": "a", "‘": "'", "’": "'", "“": '"',
    "”": '"', "–": "-", "—": "-", " ": " ",
}
_HOMOGLYPH_TABLE = {ord(source): target for source, target in _HOMOGLYPH_MAP.items()}


def strip_zero_width(text: str) -> str:
    """Remove zero-width / invisible formatting characters."""
    return text.translate(_ZERO_WIDTH_TABLE)


def fold_homoglyphs(text: str) -> str:
    """Map cross-script look-alike characters back to their ASCII twin."""
    return text.translate(_HOMOGLYPH_TABLE)


def collapse_whitespace(text: str) -> str:
    """Collapse any run of whitespace to a single space and trim the ends."""
    return " ".join(text.split())


def normalize_text(text: str) -> str:
    """Canonicalize one string: NFKC, strip invisibles, fold look-alikes, dewhite.

    Order matters. NFKC first folds width/ligature/compatibility variants; then
    invisible characters are deleted (before whitespace collapse, so a run they
    split is rejoined); then cross-script look-alikes are folded to ASCII; then
    whitespace is normalised last so paragraph-insert and whitespace attacks
    both reduce to single spaces.
    """
    if not text:
        return ""
    folded = unicodedata.normalize("NFKC", text)
    folded = strip_zero_width(folded)
    folded = fold_homoglyphs(folded)
    return collapse_whitespace(folded)