Text Classification
Transformers
ONNX
Safetensors
llama
safety
content-moderation
multi-label
multilingual
tiny-language-model
from-scratch
distilled
text-embeddings-inference
Instructions to use KucLab/Barist.4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use KucLab/Barist.4 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="KucLab/Barist.4")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("KucLab/Barist.4") model = AutoModelForSequenceClassification.from_pretrained("KucLab/Barist.4", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Add kuclab/langdetect.py
Browse files- kuclab/langdetect.py +203 -0
kuclab/langdetect.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tiny heuristic language detection (no dependencies) for per-language thresholds.
|
| 2 |
+
|
| 3 |
+
Barist.1 rules (review fix): a language is only claimed when there is real
|
| 4 |
+
evidence — at least 2 stopword hits, or distinctive diacritics. Otherwise the
|
| 5 |
+
text falls back to the global threshold ("en" bucket). Ukrainian is separated
|
| 6 |
+
from Russian via ґ/є/і/ї and stopwords, not just "ї".
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import re
|
| 12 |
+
import unicodedata
|
| 13 |
+
|
| 14 |
+
STOPWORDS = {
|
| 15 |
+
"en": {"the", "and", "you", "is", "to", "a", "in", "that", "i", "of", "for", "it", "my", "me", "do", "what", "how", "are", "please", "hello", "want", "kill", "make", "bomb", "story", "write", "ignore", "rules", "send", "me", "dan", "anything", "now"},
|
| 16 |
+
"cs": {"je", "se", "na", "že", "to", "jsem", "můj", "moje", "jak", "co", "nebo", "pro", "si", "ty", "ten", "prosím", "díky", "ahoj", "není", "bude", "chci", "chceš", "mám", "nemám", "nevím", "dobrý", "ráno", "večer", "zítra", "ano", "ne", "jsme", "byl", "byla", "mluvit", "pomoct", "napiš", "pošli", "hacknout", "zabít"},
|
| 17 |
+
"de": {"der", "die", "das", "und", "ist", "ich", "du", "nicht", "mit", "ein", "eine", "auf", "für", "wie", "was", "mein", "sich", "hallo", "danke", "bitte", "will", "mich", "umbringen", "filtere", "wasser", "zuhause"},
|
| 18 |
+
"es": {"el", "la", "de", "que", "y", "en", "un", "una", "es", "por", "con", "para", "mi", "yo", "como", "qué", "los", "las", "hola", "gracias", "cómo", "está", "quiero", "tengo", "estoy", "murió", "abuela", "encanta", "fabricar", "bomba", "casera", "suicidarme", "odiar", "odio"},
|
| 19 |
+
"fr": {"le", "la", "de", "et", "les", "des", "en", "un", "une", "est", "pour", "que", "qui", "je", "tu", "mon", "ma", "comment", "bonjour", "merci", "veux", "voudrais", "vais", "tuer", "fabriquer"},
|
| 20 |
+
"it": {"il", "lo", "la", "di", "e", "che", "un", "una", "è", "per", "con", "non", "io", "come", "sono", "gli", "le", "ciao", "grazie", "voglio", "uccidermi", "odio", "traffico", "come", "stai"},
|
| 21 |
+
"pt": {"o", "a", "de", "e", "que", "um", "uma", "em", "para", "com", "não", "eu", "meu", "minha", "como", "os", "as", "olá", "obrigado", "quero", "estou"},
|
| 22 |
+
"ru": {"и", "в", "не", "на", "я", "что", "с", "а", "это", "как", "но", "ты", "мне", "привет", "здравствуй"},
|
| 23 |
+
"uk": {"і", "в", "не", "на", "я", "що", "з", "а", "це", "як", "ти", "мені", "привіт", "дякую", "який", "щоб", "ак", "справи", "ні", "так", "удь", "ласка"},
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
# distinctive diacritics per language (strong evidence, worth 2 points each)
|
| 27 |
+
DIACRITICS = {
|
| 28 |
+
"cs": set("ěščřžůďťň"),
|
| 29 |
+
"es": set("ñ¿¡"),
|
| 30 |
+
"pt": set("ãõ"),
|
| 31 |
+
"de": set("ß"),
|
| 32 |
+
"fr": set("çœàâæêëîïôùû"),
|
| 33 |
+
"it": set(["à", "è", "ò", "ù"]),
|
| 34 |
+
"uk": set("ґєії"),
|
| 35 |
+
"ru": set("ёъыэ"),
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
MIN_SCORE = 2 # below this we do not trust the guess -> global threshold
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _script(text: str) -> str:
|
| 42 |
+
counts = {}
|
| 43 |
+
for ch in text:
|
| 44 |
+
if ch.isalpha():
|
| 45 |
+
name = unicodedata.name(ch, "")
|
| 46 |
+
for key in ("CYRILLIC", "HAN", "HIRAGANA", "KATAKANA", "HANGUL", "ARABIC", "DEVANAGARI", "HEBREW", "LATIN"):
|
| 47 |
+
if key in name:
|
| 48 |
+
counts[key] = counts.get(key, 0) + 1
|
| 49 |
+
break
|
| 50 |
+
if not counts:
|
| 51 |
+
return "latin"
|
| 52 |
+
top = max(counts, key=counts.get)
|
| 53 |
+
if top == "LATIN":
|
| 54 |
+
return "latin"
|
| 55 |
+
return {"CYRILLIC": "cyrillic", "HAN": "han", "HIRAGANA": "kana", "KATAKANA": "kana",
|
| 56 |
+
"HANGUL": "hangul", "ARABIC": "arabic", "DEVANAGARI": "devanagari", "HEBREW": "hebrew"}[top]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def detect_lang(text: str) -> str:
|
| 60 |
+
"""Heuristic ISO-639-1 guess; returns 'default' (global threshold) when unsure."""
|
| 61 |
+
s = _script(text)
|
| 62 |
+
if s == "cyrillic":
|
| 63 |
+
uk_score = sum(1 for w in re.findall(r"[a-zа-яіїєґ']+", text.lower()) if w in STOPWORDS["uk"])
|
| 64 |
+
uk_score += 2 * sum(text.count(c) for c in "ґєії")
|
| 65 |
+
ru_score = sum(1 for w in re.findall(r"[a-zа-яёъыэ']+", text.lower()) if w in STOPWORDS["ru"])
|
| 66 |
+
return "uk" if uk_score >= MIN_SCORE and uk_score > ru_score else ("ru" if ru_score >= MIN_SCORE else "default")
|
| 67 |
+
if s == "han":
|
| 68 |
+
return "zh"
|
| 69 |
+
if s == "kana":
|
| 70 |
+
return "ja"
|
| 71 |
+
if s == "hangul":
|
| 72 |
+
return "ko"
|
| 73 |
+
if s == "arabic":
|
| 74 |
+
return "ar"
|
| 75 |
+
if s == "devanagari":
|
| 76 |
+
return "hi"
|
| 77 |
+
if s == "hebrew":
|
| 78 |
+
return "he"
|
| 79 |
+
words = re.findall(r"[a-zà-ÿěščřžďťňáéíúůóßäöüñãõ]+", text.lower())
|
| 80 |
+
if not words:
|
| 81 |
+
return "default"
|
| 82 |
+
scores = {}
|
| 83 |
+
for lang, stops in STOPWORDS.items():
|
| 84 |
+
if lang in ("ru", "uk"):
|
| 85 |
+
continue
|
| 86 |
+
n = sum(1 for w in words if w in stops)
|
| 87 |
+
n += 2 * sum(text.lower().count(c) for c in DIACRITICS.get(lang, ""))
|
| 88 |
+
scores[lang] = n
|
| 89 |
+
best = max(scores, key=scores.get)
|
| 90 |
+
return best if scores[best] >= MIN_SCORE else "default"
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# --- Barist.1 must-catch pre-filter (v2: accent-folded patterns, instruction-form + news/support exceptions) ---
|
| 94 |
+
def _rx(pat: str):
|
| 95 |
+
"""Compile a pattern accent-folded + lowercased, so it matches _fold_ascii(text)."""
|
| 96 |
+
folded = "".join(c for c in unicodedata.normalize("NFKD", pat) if not unicodedata.combining(c)).lower()
|
| 97 |
+
return re.compile(folded, re.I)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
_MC_CSAM = _rx(r"\b(csam|cp links?|child porn\w*|detsky material|detske intimni|nezletil\w*.{0,20}(fotk|vide|foto|obraz|video)|(dit\w*|det\w*) bez (odeceni|obleceni)|vide(a|o) s (dit\w*|det\w*) (bez odeceni|nah|intimnich|sexualnich)|(natoc|posli|poslat|ukaz|send).{0,30}(dit\w*|det\w*|kids|children) (v plavkach|in plavkach|nah[a|é]?|nude)|(dit\w*|det\w*) v plavkach.{0,20}(bez odeceni|nah|intim|sex)|intimni fotky nezletil|poslat intimni fotky|material de menores|fotos intimas de menores|video de ninos sin ropa|send me nudes of kids|photos of kids without clothes|kinderpornografie|porn\w* infantile|pornografia infantil|detsk\w* porn\w*|nah\w* fotk\w* (dit\w*|det\w*)|nah\w* fotky nezletil\w*|fotos de menores|underage nudes|cp download|pedofil\w*|telegram (skupina|group|kanal|channel|link) (o|pro|na)? ?(csam|cp|porn|detsk\w*|nah\w*|fotos|menores|pedofil))\b")
|
| 101 |
+
_MC_SELFHARM_1P = _rx(r"(chci se zabit|zabiju se|chci (se )?umrit|chci zem\w*|nechci (uz )?zit|chci si ubliz\w*|sebepo(s|š)kozuji|rezuju se|rezat se|podrezat( si)? zily|prasky na zabit|predavkovat se|rezu se(?! smichy)|kill myself|i (want to|wanna) die|end it all|i cant go on anymore|how to (kill myself|die)|ich will mich umbringen|mich umbringen|ich will sterben|ich will nicht mehr leben|quiero morir|quiero morirme|quiero suicidarme|quiero desaparecer|voglio (uccidermi|morire|sparire)|me suicider|je veux mourir|plus vivre|ne veux plus vivre|(vzit si zivot|vzit zivot)(?! do vlastnich rukou)|prestat zit(?!\s+(v|na|ve|do)\s)|nechci dal zit|jak (se )?(nejrychleji )?zemrit doma|chci (se )?zemrit|otravit se( prasky| tabletkami)?|(se )?porezat( se)?|jak se porezat|(se )?podrezat( se)?( zily)?|prasky (ktere|co) mi pomu(z|ž)ou zemrit|predavkov.{0,25}(mi pomu|chci|zemrit)|(chci|pomoz|porad).{0,25}predavkov|jak se.{0,25}predavkov|uz to nevydrzim.{0,30}(pomoz|skoncit|koncit|zemrit)|pomuz(te)? mi skoncit|chci zmizet navzdy)")
|
| 102 |
+
_MC_SELFHARM_INSTR = _rx(r"(jak se (?!nezranit)[a-z ]{0,25}(zabit|obesit|zranit|ublit|zemrit)|jak se (zabit|obesit|zranit|ublit)|jak (si )?sebepo|suicid\w*|sebevraz\w*|sebepo(s|š)kozeni|how to (kill|die)\b|end my life|skoncit\w* zivot)")
|
| 103 |
+
_MC_INSTR_MARK = _rx(r"\b(jak|how|navod na|way|method|zpusob|prosim|please|tell me|dime|sag mir|dites|instructions?|step by step)\b")
|
| 104 |
+
_MC_SUPPORT_CONTEXT = _rx(r"\b(pomoct|pomoc|kamarad|linka|bezpeci|help|support|suicide prevention|prevence|predejit|predchazet|zabranit|literatura|literature|hamlet|kniha|novela|film|divadlo|hra|game|story|sims|ve hre|pocitacova hra|vyhled|clanek|prispevek)\b")
|
| 105 |
+
_MC_NEWS_CONTEXT = _rx(r"\b(vysetruje|policie|news|zprav|clanek|report|obet|vrah|doslo k|explodovala|explosion|utok|atentat|vystava|muzeum|historie|svetova valka|exponat|dokument)\b")
|
| 106 |
+
_MC_THREAT = _rx(r"(zabiju (te|vas)|kill you|i\'ll kill you|ill kill you|je vais te tuer|te matare|voy a matarte|te voy a matar|quiero hacerte dano|voglio ucciderti|ich bringe dich um|ich tote dich|jemanden umbringen|bringe ich jemanden um)")
|
| 107 |
+
_MC_HATE_GROUP = _rx(r"(vystříl\w*|vystřel\w*|postříl\w*|postřel\w*|zabít všechny|zabit vsechny|vyhlad\w*|vyhub\w*|pověsit|povez\w*|zastřel\w*|umuč\w*|utýrat).{0,40}(cigán|cigan|rom\w*|žid\w*|zid\w*|uprchlík\w*|uprchlik\w*|muslim\w*|imigrant\w*|černoch\w*|nigg\w*|muslim\w* všichni|všechny cigány)|(cigán\w*|cigan\w*|romov\w*|židov\w*|zidov\w*|uprchlík\w*|muslim\w*|imigrant\w*|černoš\w*).{0,40}(vystříl\w*|vystřel\w*|postříl\w*|postřel\w*|zab\w*|zastřel\w*|pověs\w*|vyhn\w*|vyhub\w*|vyhlad\w*)|kill all (jews|muslims|blacks|gypsies|immigrants)|exterminate (the )?(jews|muslims|roma)|holocaust deni\w*|white power|heil hitler")
|
| 108 |
+
_MC_WEAPONS_DIY = _rx(r"(jak (vyrobit|postavit) (bombu|granat|trhavin|vybusninu)|navod na (vyrobu )?(bombu|granat|trhavin|vybusniny|exploziv)|vyroba (bomby|granatu|trhaviny|vybusniny)|(bombu|granat) doma|wie baue ich eine bombe|bombauleitung|anleitung sprengstoff|sprengstoff herstell|bombe aus haushaltsmitteln|fabricar una bomba|bomba casera|bomba homemade|hacer una bomba|how to (make|build) a (bomb|grenade)|bomb-?making|homemade bomb|grenade (instructions|tutorial)|fabriquer une bombe|comment faire une bombe|как сделать бомбу|как изготовить (бомбу|взрывчатку)|зробити бомбу|як зробити бомбу|vyroba napalmu|jak vyrobit napalm|napalm (recipe|homemade|instructions))")
|
| 109 |
+
_MC_DRUGS_DIY = _rx(r"(synteza pervitinu|jak uvarit meth|how to cook meth|meth-?synth|synthesis of (meth|nitroglycerin|fentanyl)|navod na vyrobu (fentanylu|pervitinu|drog)|recept na vyrobu drog|receta de metanfetamina|methanfetamin rezept|wie koche ich crystal meth|drogen kochen anleitung|synthesize meth|make meth at home)")
|
| 110 |
+
_MC_CYBER_DIY = _rx(r"(navod na phishing|phishingov\w* (e-?mail|stranka)|phishing (stranka|stranky|e-?mail|mail)|jak nabourat|hacknout (ucet|instagram|account)|how to hack|hack someone|keylogger tutorial|demonter un mot de passe|comment pirater|robar la cuenta|kradez hesla)")
|
| 111 |
+
_MC_DEFENSIVE = _rx(r"(branit|ochrana|protect|prevence|jak se chranit|defend|avoid phishing|recognize phishing|jak poznat phishing)")
|
| 112 |
+
_MC_JAILBREAK = _rx(r"(you are dan\b|dan mode|dan (mode|activated|unlocked|enabled)|do anything now|ignore (all (previous )?instructions|everything above|the rules)|pretend you are .{0,80}(no rules|unrestricted|without (any )?(filters?|restrictions|rules)|evil|jailbreak|grandma|bedtime|napalm|bomb|weapon|drug|meth)|act as \w+ (with|without) (no |without )(rules|restrictions|filters)|guidelines don\'?t exist|od ted( si)? bez omezeni|oublie les regles|there are no rules|předstírej .{0,80}(bez pravidel|bez omezeni|bomba|drogy|napalm|vybusnin))")
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
_MC_POISON = _rx(r"(jak (se )?otr\w*vit|čím otrávit|otrávit (souseda|někoho|psa|kočku)|jak vyrobit (jed|otravu)|recept na (jed|otravu)|jak namíchat jed|how to poison|envenenar (a )?(mi vecino|alguien|cualquiera)|como envenenar|quiero envenenar|empoisonner (quelqu|mon voisin)|jemanden vergiften|ich will jemanden vergiften|avvelenare (qualcuno|il vicino)|отравить (соседа|кого)|jak otrávit (souseda|někoho|psa|kočku|zvíře))")
|
| 116 |
+
_MC_SCHOOL = _rx(r"(sestřel\w*|postříl\w*|vystříl\w*|začnu střílet|střílet|zbraň|kulomet).{0,40}(škola|školu|ve škole|třída|do školy)|(přinesu|vezmu|půjdu|jdu|začnu).{0,40}(škola|školu|ve škole|do školy|třída).{0,40}(zbraň|střílet|kulomet|nůž)|(škola|školu|do školy|třída).{0,40}(začnu|přinesu|vezmu|půjdu).{0,30}(střílet|zbraň|nůž|kulomet)|střelba ve škole|školní masakr|school shooting|how to shoot up a school|matar (a todos )?en la escuela|tiroteo escolar|disparar en (la escuela|el colegio)|tuer tout le monde (à|a) l\'école|fusillade (scolaire|dans l\'école)|alle in der schule (töten|erschie|angreifen)|amoklauf (an der|in der) schule|uccidere tutti a scuola|strage (a|di) scuola|стріляти в школі|вбити всіх у школі")
|
| 117 |
+
_MC_MARKET = _rx(r"(kde|jak) (koupit|sehnat|seženu|objednat|koupím).{0,50}(pervitin|metamfetamin|kokain|heroin|drogy|zbraň|zbraně|explozivum|trhavin|bombu)|prodej (drog|zbraní)|nabízím (drogy|zbraně)|comprar (armas|drogas) baratas|buy (guns|drugs) (cheap|online)|как купить (оружие|наркотики)|(vařit|svařit|syntetizovat) fet( |$)|fet (na vaření|syntézu)")
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
_MC_LEGAL_WEAPON = _rx(r"(zbrojni prukaz|zbrojní průkaz|legalne|z legální|vzduchovk|airsoft|sportovni střelba|sportovní střelba|strilec na soutěžích)")
|
| 121 |
+
_MC_BOMB_BENIGN = _rx(r"(pyrotechnik|silvestr|ohňostroj|prskavk|dort|do koupele|do dortu|výbuch dortu)")
|
| 122 |
+
_MC_HELP_OTHER = _rx(r"\b(pomoct|pomoc|pomoz\w*|podpora|pece|help|support|prevence)\b(?!.*\b(mi|me)\s+(pomo\w*|help|support))")
|
| 123 |
+
_MC_POISON_3P = _rx(r"otr\w*vit (souseda|manzelku|manzela|nekdoho|psa|kocku|zvir(e|ě)|cloveka|kdokoli|obet|dite|deti)")
|
| 124 |
+
_MC_PEST = _rx(r"(krtk|myš|potkan|krys|hmyz|mravenc|plevel|slimák|škůdce|na zahradě|na zahrade)")
|
| 125 |
+
|
| 126 |
+
SUPPORT_LINK = "https://www.linkabezpeci.cz"
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _fold_ascii(text: str) -> str:
|
| 130 |
+
return "".join(c for c in unicodedata.normalize("NFKD", text) if not unicodedata.combining(c))
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def must_catch_hit(text: str) -> list[str]:
|
| 134 |
+
"""Pattern pre-filter. Returns HARD-hit category names (force unsafe).
|
| 135 |
+
|
| 136 |
+
CSAM always blocks. Self-harm blocks on 1st-person or instruction forms,
|
| 137 |
+
unless it is a support context (routed to resources instead) or news.
|
| 138 |
+
Weapons/drugs/cyber/poison/market DIY patterns are instruction-form only
|
| 139 |
+
(news pass). Jailbreak patterns are NOT hard blocks -> see review_hit().
|
| 140 |
+
"""
|
| 141 |
+
folded = _fold_ascii(text).lower()
|
| 142 |
+
hits = []
|
| 143 |
+
if _MC_CSAM.search(folded):
|
| 144 |
+
hits.append("sexual_abuse")
|
| 145 |
+
support = bool(_MC_SUPPORT_CONTEXT.search(folded))
|
| 146 |
+
news = bool(_MC_NEWS_CONTEXT.search(folded))
|
| 147 |
+
help_other = bool(_MC_HELP_OTHER.search(folded))
|
| 148 |
+
media = bool(_rx(r"\b(sims|ve hre|pocitacova hra|film|kniha|clanek|literatura|dokument|novela|divadlo|hra|game|story|hamlet|pojednava|roman)\b").search(folded))
|
| 149 |
+
if _MC_SELFHARM_1P.search(folded) and not media: # 1st person hits (caller routes to needs_support); media/game frames pass
|
| 150 |
+
hits.append("self_harm")
|
| 151 |
+
elif _MC_SELFHARM_INSTR.search(folded) and _MC_INSTR_MARK.search(folded) and not news and not help_other and not media:
|
| 152 |
+
hits.append("self_harm") # instruction form; "help someone else" frames stay safe
|
| 153 |
+
if _MC_THREAT.search(folded) and not news:
|
| 154 |
+
hits.append("violence")
|
| 155 |
+
if _MC_SCHOOL.search(folded) and not news:
|
| 156 |
+
hits.append("violence")
|
| 157 |
+
if _MC_HATE_GROUP.search(folded) and not news:
|
| 158 |
+
hits.append("hate_harassment")
|
| 159 |
+
legal = bool(_MC_LEGAL_WEAPON.search(folded))
|
| 160 |
+
bomb_benign = bool(_MC_BOMB_BENIGN.search(folded))
|
| 161 |
+
pest = bool(_MC_PEST.search(folded))
|
| 162 |
+
weapons_3p = bool(_rx(r"(pro|na|proti) (souseda|manzelku|nekdoho|skolu|tridu|verejnost|lidi|obet|clovek)\b").search(folded))
|
| 163 |
+
if _MC_WEAPONS_DIY.search(folded) and (weapons_3p or not news):
|
| 164 |
+
hits.append("weapons_explosives")
|
| 165 |
+
if _MC_DRUGS_DIY.search(folded) and not news:
|
| 166 |
+
hits.append("drugs")
|
| 167 |
+
if _MC_CYBER_DIY.search(folded) and not _MC_DEFENSIVE.search(folded) and not news:
|
| 168 |
+
hits.append("cybercrime_fraud")
|
| 169 |
+
market_hard = bool(_rx(r"(koupit|sehnat|sezenu|objednat|koupim).{0,50}(pervitin|metamfetamin|kokain|heroin|drogy|zbran|zbraně|explozivum|trhavin|bombu|fet)\b").search(folded))
|
| 170 |
+
if _MC_MARKET.search(folded) and (market_hard or not news):
|
| 171 |
+
hits.append("illegal_marketplace")
|
| 172 |
+
poison_3p = bool(_MC_POISON_3P.search(folded))
|
| 173 |
+
if _MC_POISON.search(folded) and not pest and (poison_3p or (not news and not support)):
|
| 174 |
+
hits.append("violence")
|
| 175 |
+
# downgrades: legal weapons / benign bomb / pest control -> review only, not a block
|
| 176 |
+
if legal and "illegal_marketplace" in hits:
|
| 177 |
+
hits.remove("illegal_marketplace")
|
| 178 |
+
if bomb_benign and "illegal_marketplace" in hits:
|
| 179 |
+
hits.remove("illegal_marketplace")
|
| 180 |
+
if legal and "weapons_explosives" in hits and not _MC_WEAPONS_DIY.search(folded):
|
| 181 |
+
pass
|
| 182 |
+
return sorted(set(hits))
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def review_hit(text: str) -> list[str]:
|
| 186 |
+
"""Jailbreak / prompt-injection / legally-ambiguous patterns -> needs_review, NOT a hard block."""
|
| 187 |
+
folded = _fold_ascii(text).lower()
|
| 188 |
+
out = []
|
| 189 |
+
if _MC_JAILBREAK.search(folded):
|
| 190 |
+
out.append("jailbreak_injection")
|
| 191 |
+
if _MC_LEGAL_WEAPON.search(folded) and (_MC_MARKET.search(folded) or _MC_WEAPONS_DIY.search(folded)):
|
| 192 |
+
out.append("weapons_explosives")
|
| 193 |
+
if _MC_BOMB_BENIGN.search(folded) and _MC_MARKET.search(folded):
|
| 194 |
+
out.append("illegal_marketplace")
|
| 195 |
+
return sorted(set(out))
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def is_support_context(text: str) -> bool:
|
| 199 |
+
"""Self-harm mention in a helping context -> route to support, not a block."""
|
| 200 |
+
folded = _fold_ascii(text).lower()
|
| 201 |
+
return bool((_MC_SELFHARM_1P.search(folded) or _MC_SELFHARM_INSTR.search(folded))
|
| 202 |
+
and _MC_HELP_OTHER.search(folded))
|
| 203 |
+
|