Spaces:
Running on Zero
Running on Zero
File size: 5,500 Bytes
0fe62e1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | """
ToxiGuard AI — Text Preprocessing Pipeline
============================================
Two distinct preprocessing pipelines optimised for different consumers:
1. preprocess_for_rules(text) → str
Aggressive normalisation for the keyword rule engine.
- Lowercases, strips URLs/emails, de-obfuscates leet-speak
- Strips punctuation and non-ASCII (ASCII-safe for pattern matching)
2. preprocess_for_model(text) → str
Light clean for DeBERTa tokenizer.
- Removes URLs and excessive whitespace
- Preserves unicode, emojis, and casing
- DeBERTa's tokenizer handles the rest
3. preprocess(text) → dict [backward-compatible wrapper]
Returns {"clean_text": ..., "rule_text": ..., "tokens": [...]}
"""
import re
import string
# ──────────────────────────────────────────────────────────────────────────────
# LEET-SPEAK DE-OBFUSCATION MAP (rule engine only)
# ──────────────────────────────────────────────────────────────────────────────
_OBFUSCATION_MAP: dict[str, str] = {
"@": "a",
"$": "s",
"0": "o",
"1": "i",
"3": "e",
"4": "a",
"5": "s",
"7": "t",
"*": "",
"!": "",
"_": "",
"-": " ",
}
_URL_RE = re.compile(r"https?://\S+|www\.\S+", re.IGNORECASE)
_EMAIL_RE = re.compile(r"\S+@\S+\.\S+")
_WHITESPACE_RE = re.compile(r"\s+")
# ──────────────────────────────────────────────────────────────────────────────
# PIPELINE 1 — Rule engine (aggressive, ASCII-safe)
# ──────────────────────────────────────────────────────────────────────────────
def preprocess_for_rules(text: str) -> str:
"""
Normalise text for the keyword rule engine.
Steps:
1. Lowercase + strip
2. Remove URLs and email addresses
3. De-obfuscate leet-speak characters (@ → a, $ → s, etc.)
4. Drop non-ASCII (covers emoji that can't map to leet-speak)
5. Remove punctuation
6. Collapse whitespace
"""
if not text:
return ""
text = text.lower().strip()
text = _URL_RE.sub(" ", text)
text = _EMAIL_RE.sub(" ", text)
for char, replacement in _OBFUSCATION_MAP.items():
text = text.replace(char, replacement)
# Drop non-ASCII (emoji, exotic unicode)
text = text.encode("ascii", "ignore").decode()
# Remove punctuation but keep spaces
text = text.translate(str.maketrans("", "", string.punctuation))
text = _WHITESPACE_RE.sub(" ", text).strip()
return text
# ──────────────────────────────────────────────────────────────────────────────
# PIPELINE 2 — DeBERTa tokenizer (light clean, unicode-safe)
# ──────────────────────────────────────────────────────────────────────────────
def preprocess_for_model(text: str) -> str:
"""
Light normalisation for DeBERTa.
DeBERTa's SentencePiece tokenizer handles:
- Casing (it's uncased internally)
- Punctuation
- Unicode and emojis
We only remove noise that adds no semantic value:
- URLs
- Email addresses
- Excessive whitespace
"""
if not text:
return ""
text = _URL_RE.sub(" [URL] ", text)
text = _EMAIL_RE.sub(" [EMAIL] ", text)
text = _WHITESPACE_RE.sub(" ", text).strip()
return text
# ──────────────────────────────────────────────────────────────────────────────
# BACKWARD-COMPAT WRAPPER
# ──────────────────────────────────────────────────────────────────────────────
def tokenize(text: str) -> list[str]:
"""Split rule-preprocessed text into tokens."""
if not text:
return []
return text.split()
def preprocess(text: str) -> dict:
"""
Full preprocessing pipeline — backward-compatible wrapper.
Returns:
{
"clean_text": str — rule-safe (aggressive, ASCII)
"model_text": str — model-safe (light, unicode)
"tokens": list — tokens from clean_text
}
"""
rule_text = preprocess_for_rules(text)
model_text = preprocess_for_model(text)
return {
"clean_text": rule_text, # kept for backward compat
"model_text": model_text,
"tokens": tokenize(rule_text),
}
__all__ = [
"preprocess",
"preprocess_for_rules",
"preprocess_for_model",
"tokenize",
]
|