argus-v5 / normalize.py
vineeth98's picture
Add Argus v5 weights, canonicalization code, and model card
33baa69 verified
Raw History Blame Contribute Delete
3.22 kB
"""Canonicalize text so several RAID attacks are defeated before the model sees it.
Applied at BOTH train and inference (the invariant that makes it work): if every
input is mapped to the same ASCII-ish canonical form, then RAID's homoglyph,
zero-width-space, and whitespace attacks become no-ops — the perturbed and clean
versions collapse to the same string, so the detector never has to spend
capacity learning them. Word-level attacks (synonym, misspelling, paraphrase)
are deliberately left intact; those are handled by training data, not here.
Keep this module dependency-free and fast: it runs on every request.
"""
from __future__ import annotations
import unicodedata
# Zero-width and invisible formatting characters RAID (and real humanizers)
# inject to break tokenization. Deleted outright.
_ZERO_WIDTH_CHARS = (
"​" # zero-width space (RAID's zero_width_space attack)
"‌" # zero-width non-joiner
"‍" # zero-width joiner
"⁠" # word joiner
"" # zero-width no-break space / BOM
"­" # soft hyphen
)
_ZERO_WIDTH_TABLE = {ord(char): None for char in _ZERO_WIDTH_CHARS}
# Common confusable code points mapped back to their ASCII twin. Covers the
# Cyrillic and Greek look-alikes RAID's homoglyph attack uses plus the fancy
# quotes/dashes that leak in from copy-paste. NFKC (below) handles most width/
# ligature variants; this table covers the cross-script look-alikes it does not.
_HOMOGLYPH_MAP = {
"а": "a", "е": "e", "о": "o", "р": "p", "с": "c",
"у": "y", "х": "x", "і": "i", "ј": "j", "һ": "h",
"А": "A", "В": "B", "Е": "E", "К": "K", "М": "M",
"Н": "H", "О": "O", "Р": "P", "С": "C", "Т": "T",
"Х": "X", "Α": "A", "Β": "B", "Ε": "E", "Ζ": "Z",
"Η": "H", "Ι": "I", "Κ": "K", "Μ": "M", "Ν": "N",
"Ο": "O", "Ρ": "P", "Τ": "T", "Υ": "Y", "Χ": "X",
"ο": "o", "α": "a", "‘": "'", "’": "'", "“": '"',
"”": '"', "–": "-", "—": "-", " ": " ",
}
_HOMOGLYPH_TABLE = {ord(source): target for source, target in _HOMOGLYPH_MAP.items()}
def strip_zero_width(text: str) -> str:
"""Remove zero-width / invisible formatting characters."""
return text.translate(_ZERO_WIDTH_TABLE)
def fold_homoglyphs(text: str) -> str:
"""Map cross-script look-alike characters back to their ASCII twin."""
return text.translate(_HOMOGLYPH_TABLE)
def collapse_whitespace(text: str) -> str:
"""Collapse any run of whitespace to a single space and trim the ends."""
return " ".join(text.split())
def normalize_text(text: str) -> str:
"""Canonicalize one string: NFKC, strip invisibles, fold look-alikes, dewhite.
Order matters. NFKC first folds width/ligature/compatibility variants; then
invisible characters are deleted (before whitespace collapse, so a run they
split is rejoined); then cross-script look-alikes are folded to ASCII; then
whitespace is normalised last so paragraph-insert and whitespace attacks
both reduce to single spaces.
"""
if not text:
return ""
folded = unicodedata.normalize("NFKC", text)
folded = strip_zero_width(folded)
folded = fold_homoglyphs(folded)
return collapse_whitespace(folded)