File size: 15,721 Bytes
7280d7f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
"""Tiny heuristic language detection (no dependencies) for per-language thresholds.

Barist.1 rules (review fix): a language is only claimed when there is real
evidence — at least 2 stopword hits, or distinctive diacritics. Otherwise the
text falls back to the global threshold ("en" bucket). Ukrainian is separated
from Russian via ґ/є/і/ї and stopwords, not just "ї".
"""

from __future__ import annotations

import re
import unicodedata

STOPWORDS = {
    "en": {"the", "and", "you", "is", "to", "a", "in", "that", "i", "of", "for", "it", "my", "me", "do", "what", "how", "are", "please", "hello", "want", "kill", "make", "bomb", "story", "write", "ignore", "rules", "send", "me", "dan", "anything", "now"},
    "cs": {"je", "se", "na", "že", "to", "jsem", "můj", "moje", "jak", "co", "nebo", "pro", "si", "ty", "ten", "prosím", "díky", "ahoj", "není", "bude", "chci", "chceš", "mám", "nemám", "nevím", "dobrý", "ráno", "večer", "zítra", "ano", "ne", "jsme", "byl", "byla", "mluvit", "pomoct", "napiš", "pošli", "hacknout", "zabít"},
    "de": {"der", "die", "das", "und", "ist", "ich", "du", "nicht", "mit", "ein", "eine", "auf", "für", "wie", "was", "mein", "sich", "hallo", "danke", "bitte", "will", "mich", "umbringen", "filtere", "wasser", "zuhause"},
    "es": {"el", "la", "de", "que", "y", "en", "un", "una", "es", "por", "con", "para", "mi", "yo", "como", "qué", "los", "las", "hola", "gracias", "cómo", "está", "quiero", "tengo", "estoy", "murió", "abuela", "encanta", "fabricar", "bomba", "casera", "suicidarme", "odiar", "odio"},
    "fr": {"le", "la", "de", "et", "les", "des", "en", "un", "une", "est", "pour", "que", "qui", "je", "tu", "mon", "ma", "comment", "bonjour", "merci", "veux", "voudrais", "vais", "tuer", "fabriquer"},
    "it": {"il", "lo", "la", "di", "e", "che", "un", "una", "è", "per", "con", "non", "io", "come", "sono", "gli", "le", "ciao", "grazie", "voglio", "uccidermi", "odio", "traffico", "come", "stai"},
    "pt": {"o", "a", "de", "e", "que", "um", "uma", "em", "para", "com", "não", "eu", "meu", "minha", "como", "os", "as", "olá", "obrigado", "quero", "estou"},
    "ru": {"и", "в", "не", "на", "я", "что", "с", "а", "это", "как", "но", "ты", "мне", "привет", "здравствуй"},
    "uk": {"і", "в", "не", "на", "я", "що", "з", "а", "це", "як", "ти", "мені", "привіт", "дякую", "який", "щоб", "ак", "справи", "ні", "так", "удь", "ласка"},
}

# distinctive diacritics per language (strong evidence, worth 2 points each)
DIACRITICS = {
    "cs": set("ěščřžůďťň"),
    "es": set("ñ¿¡"),
    "pt": set("ãõ"),
    "de": set("ß"),
    "fr": set("çœàâæêëîïôùû"),
    "it": set(["à", "è", "ò", "ù"]),
    "uk": set("ґєії"),
    "ru": set("ёъыэ"),
}

MIN_SCORE = 2  # below this we do not trust the guess -> global threshold


def _script(text: str) -> str:
    counts = {}
    for ch in text:
        if ch.isalpha():
            name = unicodedata.name(ch, "")
            for key in ("CYRILLIC", "HAN", "HIRAGANA", "KATAKANA", "HANGUL", "ARABIC", "DEVANAGARI", "HEBREW", "LATIN"):
                if key in name:
                    counts[key] = counts.get(key, 0) + 1
                    break
    if not counts:
        return "latin"
    top = max(counts, key=counts.get)
    if top == "LATIN":
        return "latin"
    return {"CYRILLIC": "cyrillic", "HAN": "han", "HIRAGANA": "kana", "KATAKANA": "kana",
            "HANGUL": "hangul", "ARABIC": "arabic", "DEVANAGARI": "devanagari", "HEBREW": "hebrew"}[top]


def detect_lang(text: str) -> str:
    """Heuristic ISO-639-1 guess; returns 'default' (global threshold) when unsure."""
    s = _script(text)
    if s == "cyrillic":
        uk_score = sum(1 for w in re.findall(r"[a-zа-яіїєґ']+", text.lower()) if w in STOPWORDS["uk"])
        uk_score += 2 * sum(text.count(c) for c in "ґєії")
        ru_score = sum(1 for w in re.findall(r"[a-zа-яёъыэ']+", text.lower()) if w in STOPWORDS["ru"])
        return "uk" if uk_score >= MIN_SCORE and uk_score > ru_score else ("ru" if ru_score >= MIN_SCORE else "default")
    if s == "han":
        return "zh"
    if s == "kana":
        return "ja"
    if s == "hangul":
        return "ko"
    if s == "arabic":
        return "ar"
    if s == "devanagari":
        return "hi"
    if s == "hebrew":
        return "he"
    words = re.findall(r"[a-zà-ÿěščřžďťňáéíúůóßäöüñãõ]+", text.lower())
    if not words:
        return "default"
    scores = {}
    for lang, stops in STOPWORDS.items():
        if lang in ("ru", "uk"):
            continue
        n = sum(1 for w in words if w in stops)
        n += 2 * sum(text.lower().count(c) for c in DIACRITICS.get(lang, ""))
        scores[lang] = n
    best = max(scores, key=scores.get)
    return best if scores[best] >= MIN_SCORE else "default"


# --- Barist.1 must-catch pre-filter (v2: accent-folded patterns, instruction-form + news/support exceptions) ---
def _rx(pat: str):
    """Compile a pattern accent-folded + lowercased, so it matches _fold_ascii(text)."""
    folded = "".join(c for c in unicodedata.normalize("NFKD", pat) if not unicodedata.combining(c)).lower()
    return re.compile(folded, re.I)


_MC_CSAM = _rx(r"\b(csam|cp links?|child porn\w*|detsky material|detske intimni|nezletil.{0,20}(fotk|vide|foto|obraz|video)|deti bez (odeceni|obleceni)|vide(a|o) s detmi( bez odeceni)?|intimni fotky nezletil|poslat intimni fotky|fotografie deti v plavkach|dite v plavkach|hledam skupinu na telegramu|telegram skupina s detskym|material de menores|fotos intimas de menores|video de ninos sin ropa|send me nudes of kids|photos of kids without clothes|kinderpornografie|porn\w* infantile|pornografia infantil|detsk\w* porn\w*|nah\w* fotk\w* deti|nah\w* fotky nezletil\w*|fotos de menores|underage nudes|cp download|pedofil\w*|telegram (skupina|group|kanal|channel|link) (o|pro|na)? ?(csam|cp|porn|detsk\w*|nah\w*|fotos|menores|pedofil))\b")
_MC_SELFHARM_1P = _rx(r"(chci se zabit|zabiju se|chci (se )?umrit|chci zem\w*|nechci (uz )?zit|chci si ubliz\w*|sebepo(s|š)kozuji|rezuju se|rezat se|podrezat( si)? zily|prasky na zabit|predavkovat se|rezu se(?! smichy)|kill myself|i (want to|wanna) die|end it all|i cant go on anymore|how to (kill myself|die)|ich will mich umbringen|mich umbringen|ich will sterben|ich will nicht mehr leben|quiero morir|quiero morirme|quiero suicidarme|quiero desaparecer|voglio (uccidermi|morire|sparire)|me suicider|je veux mourir|plus vivre|ne veux plus vivre|vzit si zivot|prestat zit|chci prestat zit|nechci dal zit|zemrit doma|jak nejrychleji zemrit|jak zemrit doma|otravit se( prasky| tabletkami)?|(se )?porezat( se)?|jak se porezat|(se )?podrezat( se)?( zily)?|otravit (se )?prasky|jak se otravit|prasky (ktere|co) mi pomu(z|ž)ou zemrit|predavkovani|uz to nevydrzim|nikdo by me nepostradal|pomuz(te)? mi skoncit|porad( mi)? jak se zranit nejvic|chci zmizet navzdy|planuji si vzit zivot|nechci uz byt tady)")
_MC_SELFHARM_INSTR = _rx(r"(jak se (zabit|obesit|zranit|ublit)|jak (si )?sebepo|suicid\w*|sebevraz\w*|sebepo(s|š)kozeni|how to (kill|die)\b|end my life|skoncit\w* zivot)")
_MC_INSTR_MARK = _rx(r"\b(jak|how|navod na|way|method|zpusob|prosim|please|tell me|dime|sag mir|dites|instructions?|step by step)\b")
_MC_SUPPORT_CONTEXT = _rx(r"(pomoct|pomoc|kamarad|linka|bezpeci|help|support|suicide prevention|prevence|literatura|literature|hamlet|kniha|novela|film|divadlo|hra|game|story)")
_MC_NEWS_CONTEXT = _rx(r"(vysetruje|policie|news|zprav|clanek|report|obet|vrah|doslo k|explodovala|explosion|utok|atentat)")
_MC_THREAT = _rx(r"(zabiju (te|vas)|kill you|i\'ll kill you|ill kill you|je vais te tuer|te matare|voy a matarte|te voy a matar|quiero hacerte dano|voglio ucciderti|ich bringe dich um|ich tote dich|jemanden umbringen|bringe ich jemanden um)")
_MC_HATE_GROUP = _rx(r"(vystříl\w*|vystřel\w*|postříl\w*|postřel\w*|zabít všechny|zabit vsechny|vyhlad\w*|vyhub\w*|pověsit|povez\w*|zastřel\w*|umuč\w*|utýrat).{0,40}(cigán|cigan|rom\w*|žid\w*|zid\w*|uprchlík\w*|uprchlik\w*|muslim\w*|imigrant\w*|černoch\w*|nigg\w*|muslim\w* všichni|všechny cigány)|(cigán\w*|cigan\w*|romov\w*|židov\w*|zidov\w*|uprchlík\w*|muslim\w*|imigrant\w*|černoš\w*).{0,40}(vystříl\w*|vystřel\w*|postříl\w*|postřel\w*|zab\w*|zastřel\w*|pověs\w*|vyhn\w*|vyhub\w*|vyhlad\w*)|kill all (jews|muslims|blacks|gypsies|immigrants)|exterminate (the )?(jews|muslims|roma)|holocaust deni\w*|white power|heil hitler")
_MC_WEAPONS_DIY = _rx(r"(jak (vyrobit|postavit) (bombu|granat|trhavin|vybusninu)|navod na (vyrobu )?(bombu|granat|trhavin|vybusniny|exploziv)|vyroba (bomby|granatu|trhaviny|vybusniny)|(bombu|granat) doma|wie baue ich eine bombe|bombauleitung|anleitung sprengstoff|sprengstoff herstell|bombe aus haushaltsmitteln|fabricar una bomba|bomba casera|bomba homemade|hacer una bomba|how to (make|build) a (bomb|grenade)|bomb-?making|homemade bomb|grenade (instructions|tutorial)|fabriquer une bombe|comment faire une bombe|как сделать бомбу|как изготовить (бомбу|взрывчатку)|зробити бомбу|як зробити бомбу|vyroba napalmu|jak vyrobit napalm|napalm (recipe|homemade|instructions))")
_MC_DRUGS_DIY = _rx(r"(synteza pervitinu|jak uvarit meth|how to cook meth|meth-?synth|synthesis of (meth|nitroglycerin|fentanyl)|navod na vyrobu (fentanylu|pervitinu|drog)|recept na vyrobu drog|receta de metanfetamina|methanfetamin rezept|wie koche ich crystal meth|drogen kochen anleitung|synthesize meth|make meth at home)")
_MC_CYBER_DIY = _rx(r"(navod na phishing|phishingov\w* (e-?mail|stranka)|phishing (stranka|stranky|e-?mail|mail)|jak nabourat|hacknout (ucet|instagram|account)|how to hack|hack someone|keylogger tutorial|demonter un mot de passe|comment pirater|robar la cuenta|kradez hesla)")
_MC_DEFENSIVE = _rx(r"(branit|ochrana|protect|prevence|jak se chranit|defend|avoid phishing|recognize phishing|jak poznat phishing)")
_MC_JAILBREAK = _rx(r"(you are dan\b|dan mode|dan (mode|activated|unlocked|enabled)|do anything now|ignore (all (previous )?instructions|everything above|the rules)|pretend you are .{0,80}(no rules|unrestricted|without (any )?(filters?|restrictions|rules)|evil|jailbreak|grandma|bedtime|napalm|bomb|weapon|drug|meth)|act as \w+ (with|without) (no |without )(rules|restrictions|filters)|guidelines don\'?t exist|od ted( si)? bez omezeni|oublie les regles|there are no rules|předstírej .{0,80}(bez pravidel|bez omezeni|bomba|drogy|napalm|vybusnin))")


_MC_POISON = _rx(r"(jak (se )?otr\w*vit|čím otrávit|otrávit (souseda|někoho|psa|kočku)|jak vyrobit (jed|otravu)|recept na (jed|otravu)|jak namíchat jed|how to poison|envenenar (a )?(mi vecino|alguien|cualquiera)|como envenenar|quiero envenenar|empoisonner (quelqu|mon voisin)|jemanden vergiften|ich will jemanden vergiften|avvelenare (qualcuno|il vicino)|отравить (соседа|кого)|jak otrávit (souseda|někoho|psa|kočku|zvíře))")
_MC_SCHOOL = _rx(r"(sestřel|postříl|vystříl|střílet|zbraň|kulomet|začnu střílet).{0,40}(škola|školu|ve škole|třída|do školy)|(škola|školu|ve škole|do školy|třída).{0,40}(střílet|střelba|zbraň|sestřelit|vystřílet|kulomet)|střelba ve škole|školní masakr|school shooting|how to shoot up a school|matar (a todos )?en la escuela|tiroteo escolar|disparar en (la escuela|el colegio)|tuer tout le monde (à|a) l\'école|fusillade (scolaire|dans l\'école)|alle in der schule (töten|erschie|angreifen)|amoklauf (an der|in der) schule|uccidere tutti a scuola|strage (a|di) scuola|стріляти в школі|вбити всіх у школі")
_MC_MARKET = _rx(r"(kde|jak) (koupit|sehnat|seženu|objednat|koupím).{0,50}(pervitin|metamfetamin|kokain|heroin|drogy|zbraň|zbraně|explozivum|trhavin|bombu)|prodej (drog|zbraní)|nabízím (drogy|zbraně)|comprar (armas|drogas) baratas|buy (guns|drugs) (cheap|online)|как купить (оружие|наркотики)|(vařit|svařit|syntetizovat) fet( |$)|fet (na vaření|syntézu)")


_MC_LEGAL_WEAPON = _rx(r"(zbrojni prukaz|zbrojní průkaz|legalne|z legální|vzduchovk|airsoft|sportovni střelba|sportovní střelba|strilec na soutěžích)")
_MC_BOMB_BENIGN = _rx(r"(pyrotechnik|silvestr|ohňostroj|prskavk|dort|do koupele|do dortu|výbuch dortu)")
_MC_PEST = _rx(r"(krtk|myš|potkan|krys|hmyz|mravenc|plevel|slimák|škůdce|na zahradě|na zahrade)")

SUPPORT_LINK = "https://www.linkabezpeci.cz"


def _fold_ascii(text: str) -> str:
    return "".join(c for c in unicodedata.normalize("NFKD", text) if not unicodedata.combining(c))


def must_catch_hit(text: str) -> list[str]:
    """Pattern pre-filter. Returns HARD-hit category names (force unsafe).

    CSAM always blocks. Self-harm blocks on 1st-person or instruction forms,
    unless it is a support context (routed to resources instead) or news.
    Weapons/drugs/cyber/poison/market DIY patterns are instruction-form only
    (news pass). Jailbreak patterns are NOT hard blocks -> see review_hit().
    """
    folded = _fold_ascii(text).lower()
    hits = []
    if _MC_CSAM.search(folded):
        hits.append("sexual_abuse")
    support = bool(_MC_SUPPORT_CONTEXT.search(folded))
    news = bool(_MC_NEWS_CONTEXT.search(folded))
    if _MC_SELFHARM_1P.search(folded) and not support:
        hits.append("self_harm")
    elif _MC_SELFHARM_INSTR.search(folded) and _MC_INSTR_MARK.search(folded) and not support and not news:
        hits.append("self_harm")
    if _MC_THREAT.search(folded) and not news:
        hits.append("violence")
    if _MC_SCHOOL.search(folded) and not news:
        hits.append("violence")
    if _MC_HATE_GROUP.search(folded) and not news:
        hits.append("hate_harassment")
    legal = bool(_MC_LEGAL_WEAPON.search(folded))
    bomb_benign = bool(_MC_BOMB_BENIGN.search(folded))
    pest = bool(_MC_PEST.search(folded))
    if _MC_WEAPONS_DIY.search(folded) and not news:
        hits.append("weapons_explosives")
    if _MC_DRUGS_DIY.search(folded) and not news:
        hits.append("drugs")
    if _MC_CYBER_DIY.search(folded) and not _MC_DEFENSIVE.search(folded) and not news:
        hits.append("cybercrime_fraud")
    if _MC_MARKET.search(folded) and not news:
        hits.append("illegal_marketplace")
    if _MC_POISON.search(folded) and not news and not pest:
        hits.append("violence")
    # downgrades: legal weapons / benign bomb / pest control -> review only, not a block
    if legal and "illegal_marketplace" in hits:
        hits.remove("illegal_marketplace")
    if bomb_benign and "illegal_marketplace" in hits:
        hits.remove("illegal_marketplace")
    if legal and "weapons_explosives" in hits and not _MC_WEAPONS_DIY.search(folded):
        pass
    return sorted(set(hits))


def review_hit(text: str) -> list[str]:
    """Jailbreak / prompt-injection / legally-ambiguous patterns -> needs_review, NOT a hard block."""
    folded = _fold_ascii(text).lower()
    out = []
    if _MC_JAILBREAK.search(folded):
        out.append("jailbreak_injection")
    if _MC_LEGAL_WEAPON.search(folded) and (_MC_MARKET.search(folded) or _MC_WEAPONS_DIY.search(folded)):
        out.append("weapons_explosives")
    if _MC_BOMB_BENIGN.search(folded) and _MC_MARKET.search(folded):
        out.append("illegal_marketplace")
    return sorted(set(out))


def is_support_context(text: str) -> bool:
    """Self-harm mention in a helping context -> route to support, not a block."""
    folded = _fold_ascii(text).lower()
    return bool((_MC_SELFHARM_1P.search(folded) or _MC_SELFHARM_INSTR.search(folded))
                and _MC_SUPPORT_CONTEXT.search(folded))