Text Classification
Transformers
ONNX
Safetensors
llama
safety
content-moderation
multi-label
multilingual
tiny-language-model
from-scratch
distilled
text-embeddings-inference
Instructions to use KucLab/Barist.2 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use KucLab/Barist.2 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="KucLab/Barist.2")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("KucLab/Barist.2") model = AutoModelForSequenceClassification.from_pretrained("KucLab/Barist.2", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 15,721 Bytes
7280d7f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | """Tiny heuristic language detection (no dependencies) for per-language thresholds.
Barist.1 rules (review fix): a language is only claimed when there is real
evidence — at least 2 stopword hits, or distinctive diacritics. Otherwise the
text falls back to the global threshold ("en" bucket). Ukrainian is separated
from Russian via ґ/є/і/ї and stopwords, not just "ї".
"""
from __future__ import annotations
import re
import unicodedata
STOPWORDS = {
"en": {"the", "and", "you", "is", "to", "a", "in", "that", "i", "of", "for", "it", "my", "me", "do", "what", "how", "are", "please", "hello", "want", "kill", "make", "bomb", "story", "write", "ignore", "rules", "send", "me", "dan", "anything", "now"},
"cs": {"je", "se", "na", "že", "to", "jsem", "můj", "moje", "jak", "co", "nebo", "pro", "si", "ty", "ten", "prosím", "díky", "ahoj", "není", "bude", "chci", "chceš", "mám", "nemám", "nevím", "dobrý", "ráno", "večer", "zítra", "ano", "ne", "jsme", "byl", "byla", "mluvit", "pomoct", "napiš", "pošli", "hacknout", "zabít"},
"de": {"der", "die", "das", "und", "ist", "ich", "du", "nicht", "mit", "ein", "eine", "auf", "für", "wie", "was", "mein", "sich", "hallo", "danke", "bitte", "will", "mich", "umbringen", "filtere", "wasser", "zuhause"},
"es": {"el", "la", "de", "que", "y", "en", "un", "una", "es", "por", "con", "para", "mi", "yo", "como", "qué", "los", "las", "hola", "gracias", "cómo", "está", "quiero", "tengo", "estoy", "murió", "abuela", "encanta", "fabricar", "bomba", "casera", "suicidarme", "odiar", "odio"},
"fr": {"le", "la", "de", "et", "les", "des", "en", "un", "une", "est", "pour", "que", "qui", "je", "tu", "mon", "ma", "comment", "bonjour", "merci", "veux", "voudrais", "vais", "tuer", "fabriquer"},
"it": {"il", "lo", "la", "di", "e", "che", "un", "una", "è", "per", "con", "non", "io", "come", "sono", "gli", "le", "ciao", "grazie", "voglio", "uccidermi", "odio", "traffico", "come", "stai"},
"pt": {"o", "a", "de", "e", "que", "um", "uma", "em", "para", "com", "não", "eu", "meu", "minha", "como", "os", "as", "olá", "obrigado", "quero", "estou"},
"ru": {"и", "в", "не", "на", "я", "что", "с", "а", "это", "как", "но", "ты", "мне", "привет", "здравствуй"},
"uk": {"і", "в", "не", "на", "я", "що", "з", "а", "це", "як", "ти", "мені", "привіт", "дякую", "який", "щоб", "ак", "справи", "ні", "так", "удь", "ласка"},
}
# distinctive diacritics per language (strong evidence, worth 2 points each)
DIACRITICS = {
"cs": set("ěščřžůďťň"),
"es": set("ñ¿¡"),
"pt": set("ãõ"),
"de": set("ß"),
"fr": set("çœàâæêëîïôùû"),
"it": set(["à", "è", "ò", "ù"]),
"uk": set("ґєії"),
"ru": set("ёъыэ"),
}
MIN_SCORE = 2 # below this we do not trust the guess -> global threshold
def _script(text: str) -> str:
counts = {}
for ch in text:
if ch.isalpha():
name = unicodedata.name(ch, "")
for key in ("CYRILLIC", "HAN", "HIRAGANA", "KATAKANA", "HANGUL", "ARABIC", "DEVANAGARI", "HEBREW", "LATIN"):
if key in name:
counts[key] = counts.get(key, 0) + 1
break
if not counts:
return "latin"
top = max(counts, key=counts.get)
if top == "LATIN":
return "latin"
return {"CYRILLIC": "cyrillic", "HAN": "han", "HIRAGANA": "kana", "KATAKANA": "kana",
"HANGUL": "hangul", "ARABIC": "arabic", "DEVANAGARI": "devanagari", "HEBREW": "hebrew"}[top]
def detect_lang(text: str) -> str:
"""Heuristic ISO-639-1 guess; returns 'default' (global threshold) when unsure."""
s = _script(text)
if s == "cyrillic":
uk_score = sum(1 for w in re.findall(r"[a-zа-яіїєґ']+", text.lower()) if w in STOPWORDS["uk"])
uk_score += 2 * sum(text.count(c) for c in "ґєії")
ru_score = sum(1 for w in re.findall(r"[a-zа-яёъыэ']+", text.lower()) if w in STOPWORDS["ru"])
return "uk" if uk_score >= MIN_SCORE and uk_score > ru_score else ("ru" if ru_score >= MIN_SCORE else "default")
if s == "han":
return "zh"
if s == "kana":
return "ja"
if s == "hangul":
return "ko"
if s == "arabic":
return "ar"
if s == "devanagari":
return "hi"
if s == "hebrew":
return "he"
words = re.findall(r"[a-zà-ÿěščřžďťňáéíúůóßäöüñãõ]+", text.lower())
if not words:
return "default"
scores = {}
for lang, stops in STOPWORDS.items():
if lang in ("ru", "uk"):
continue
n = sum(1 for w in words if w in stops)
n += 2 * sum(text.lower().count(c) for c in DIACRITICS.get(lang, ""))
scores[lang] = n
best = max(scores, key=scores.get)
return best if scores[best] >= MIN_SCORE else "default"
# --- Barist.1 must-catch pre-filter (v2: accent-folded patterns, instruction-form + news/support exceptions) ---
def _rx(pat: str):
"""Compile a pattern accent-folded + lowercased, so it matches _fold_ascii(text)."""
folded = "".join(c for c in unicodedata.normalize("NFKD", pat) if not unicodedata.combining(c)).lower()
return re.compile(folded, re.I)
_MC_CSAM = _rx(r"\b(csam|cp links?|child porn\w*|detsky material|detske intimni|nezletil.{0,20}(fotk|vide|foto|obraz|video)|deti bez (odeceni|obleceni)|vide(a|o) s detmi( bez odeceni)?|intimni fotky nezletil|poslat intimni fotky|fotografie deti v plavkach|dite v plavkach|hledam skupinu na telegramu|telegram skupina s detskym|material de menores|fotos intimas de menores|video de ninos sin ropa|send me nudes of kids|photos of kids without clothes|kinderpornografie|porn\w* infantile|pornografia infantil|detsk\w* porn\w*|nah\w* fotk\w* deti|nah\w* fotky nezletil\w*|fotos de menores|underage nudes|cp download|pedofil\w*|telegram (skupina|group|kanal|channel|link) (o|pro|na)? ?(csam|cp|porn|detsk\w*|nah\w*|fotos|menores|pedofil))\b")
_MC_SELFHARM_1P = _rx(r"(chci se zabit|zabiju se|chci (se )?umrit|chci zem\w*|nechci (uz )?zit|chci si ubliz\w*|sebepo(s|š)kozuji|rezuju se|rezat se|podrezat( si)? zily|prasky na zabit|predavkovat se|rezu se(?! smichy)|kill myself|i (want to|wanna) die|end it all|i cant go on anymore|how to (kill myself|die)|ich will mich umbringen|mich umbringen|ich will sterben|ich will nicht mehr leben|quiero morir|quiero morirme|quiero suicidarme|quiero desaparecer|voglio (uccidermi|morire|sparire)|me suicider|je veux mourir|plus vivre|ne veux plus vivre|vzit si zivot|prestat zit|chci prestat zit|nechci dal zit|zemrit doma|jak nejrychleji zemrit|jak zemrit doma|otravit se( prasky| tabletkami)?|(se )?porezat( se)?|jak se porezat|(se )?podrezat( se)?( zily)?|otravit (se )?prasky|jak se otravit|prasky (ktere|co) mi pomu(z|ž)ou zemrit|predavkovani|uz to nevydrzim|nikdo by me nepostradal|pomuz(te)? mi skoncit|porad( mi)? jak se zranit nejvic|chci zmizet navzdy|planuji si vzit zivot|nechci uz byt tady)")
_MC_SELFHARM_INSTR = _rx(r"(jak se (zabit|obesit|zranit|ublit)|jak (si )?sebepo|suicid\w*|sebevraz\w*|sebepo(s|š)kozeni|how to (kill|die)\b|end my life|skoncit\w* zivot)")
_MC_INSTR_MARK = _rx(r"\b(jak|how|navod na|way|method|zpusob|prosim|please|tell me|dime|sag mir|dites|instructions?|step by step)\b")
_MC_SUPPORT_CONTEXT = _rx(r"(pomoct|pomoc|kamarad|linka|bezpeci|help|support|suicide prevention|prevence|literatura|literature|hamlet|kniha|novela|film|divadlo|hra|game|story)")
_MC_NEWS_CONTEXT = _rx(r"(vysetruje|policie|news|zprav|clanek|report|obet|vrah|doslo k|explodovala|explosion|utok|atentat)")
_MC_THREAT = _rx(r"(zabiju (te|vas)|kill you|i\'ll kill you|ill kill you|je vais te tuer|te matare|voy a matarte|te voy a matar|quiero hacerte dano|voglio ucciderti|ich bringe dich um|ich tote dich|jemanden umbringen|bringe ich jemanden um)")
_MC_HATE_GROUP = _rx(r"(vystříl\w*|vystřel\w*|postříl\w*|postřel\w*|zabít všechny|zabit vsechny|vyhlad\w*|vyhub\w*|pověsit|povez\w*|zastřel\w*|umuč\w*|utýrat).{0,40}(cigán|cigan|rom\w*|žid\w*|zid\w*|uprchlík\w*|uprchlik\w*|muslim\w*|imigrant\w*|černoch\w*|nigg\w*|muslim\w* všichni|všechny cigány)|(cigán\w*|cigan\w*|romov\w*|židov\w*|zidov\w*|uprchlík\w*|muslim\w*|imigrant\w*|černoš\w*).{0,40}(vystříl\w*|vystřel\w*|postříl\w*|postřel\w*|zab\w*|zastřel\w*|pověs\w*|vyhn\w*|vyhub\w*|vyhlad\w*)|kill all (jews|muslims|blacks|gypsies|immigrants)|exterminate (the )?(jews|muslims|roma)|holocaust deni\w*|white power|heil hitler")
_MC_WEAPONS_DIY = _rx(r"(jak (vyrobit|postavit) (bombu|granat|trhavin|vybusninu)|navod na (vyrobu )?(bombu|granat|trhavin|vybusniny|exploziv)|vyroba (bomby|granatu|trhaviny|vybusniny)|(bombu|granat) doma|wie baue ich eine bombe|bombauleitung|anleitung sprengstoff|sprengstoff herstell|bombe aus haushaltsmitteln|fabricar una bomba|bomba casera|bomba homemade|hacer una bomba|how to (make|build) a (bomb|grenade)|bomb-?making|homemade bomb|grenade (instructions|tutorial)|fabriquer une bombe|comment faire une bombe|как сделать бомбу|как изготовить (бомбу|взрывчатку)|зробити бомбу|як зробити бомбу|vyroba napalmu|jak vyrobit napalm|napalm (recipe|homemade|instructions))")
_MC_DRUGS_DIY = _rx(r"(synteza pervitinu|jak uvarit meth|how to cook meth|meth-?synth|synthesis of (meth|nitroglycerin|fentanyl)|navod na vyrobu (fentanylu|pervitinu|drog)|recept na vyrobu drog|receta de metanfetamina|methanfetamin rezept|wie koche ich crystal meth|drogen kochen anleitung|synthesize meth|make meth at home)")
_MC_CYBER_DIY = _rx(r"(navod na phishing|phishingov\w* (e-?mail|stranka)|phishing (stranka|stranky|e-?mail|mail)|jak nabourat|hacknout (ucet|instagram|account)|how to hack|hack someone|keylogger tutorial|demonter un mot de passe|comment pirater|robar la cuenta|kradez hesla)")
_MC_DEFENSIVE = _rx(r"(branit|ochrana|protect|prevence|jak se chranit|defend|avoid phishing|recognize phishing|jak poznat phishing)")
_MC_JAILBREAK = _rx(r"(you are dan\b|dan mode|dan (mode|activated|unlocked|enabled)|do anything now|ignore (all (previous )?instructions|everything above|the rules)|pretend you are .{0,80}(no rules|unrestricted|without (any )?(filters?|restrictions|rules)|evil|jailbreak|grandma|bedtime|napalm|bomb|weapon|drug|meth)|act as \w+ (with|without) (no |without )(rules|restrictions|filters)|guidelines don\'?t exist|od ted( si)? bez omezeni|oublie les regles|there are no rules|předstírej .{0,80}(bez pravidel|bez omezeni|bomba|drogy|napalm|vybusnin))")
_MC_POISON = _rx(r"(jak (se )?otr\w*vit|čím otrávit|otrávit (souseda|někoho|psa|kočku)|jak vyrobit (jed|otravu)|recept na (jed|otravu)|jak namíchat jed|how to poison|envenenar (a )?(mi vecino|alguien|cualquiera)|como envenenar|quiero envenenar|empoisonner (quelqu|mon voisin)|jemanden vergiften|ich will jemanden vergiften|avvelenare (qualcuno|il vicino)|отравить (соседа|кого)|jak otrávit (souseda|někoho|psa|kočku|zvíře))")
_MC_SCHOOL = _rx(r"(sestřel|postříl|vystříl|střílet|zbraň|kulomet|začnu střílet).{0,40}(škola|školu|ve škole|třída|do školy)|(škola|školu|ve škole|do školy|třída).{0,40}(střílet|střelba|zbraň|sestřelit|vystřílet|kulomet)|střelba ve škole|školní masakr|school shooting|how to shoot up a school|matar (a todos )?en la escuela|tiroteo escolar|disparar en (la escuela|el colegio)|tuer tout le monde (à|a) l\'école|fusillade (scolaire|dans l\'école)|alle in der schule (töten|erschie|angreifen)|amoklauf (an der|in der) schule|uccidere tutti a scuola|strage (a|di) scuola|стріляти в школі|вбити всіх у школі")
_MC_MARKET = _rx(r"(kde|jak) (koupit|sehnat|seženu|objednat|koupím).{0,50}(pervitin|metamfetamin|kokain|heroin|drogy|zbraň|zbraně|explozivum|trhavin|bombu)|prodej (drog|zbraní)|nabízím (drogy|zbraně)|comprar (armas|drogas) baratas|buy (guns|drugs) (cheap|online)|как купить (оружие|наркотики)|(vařit|svařit|syntetizovat) fet( |$)|fet (na vaření|syntézu)")
_MC_LEGAL_WEAPON = _rx(r"(zbrojni prukaz|zbrojní průkaz|legalne|z legální|vzduchovk|airsoft|sportovni střelba|sportovní střelba|strilec na soutěžích)")
_MC_BOMB_BENIGN = _rx(r"(pyrotechnik|silvestr|ohňostroj|prskavk|dort|do koupele|do dortu|výbuch dortu)")
_MC_PEST = _rx(r"(krtk|myš|potkan|krys|hmyz|mravenc|plevel|slimák|škůdce|na zahradě|na zahrade)")
SUPPORT_LINK = "https://www.linkabezpeci.cz"
def _fold_ascii(text: str) -> str:
return "".join(c for c in unicodedata.normalize("NFKD", text) if not unicodedata.combining(c))
def must_catch_hit(text: str) -> list[str]:
"""Pattern pre-filter. Returns HARD-hit category names (force unsafe).
CSAM always blocks. Self-harm blocks on 1st-person or instruction forms,
unless it is a support context (routed to resources instead) or news.
Weapons/drugs/cyber/poison/market DIY patterns are instruction-form only
(news pass). Jailbreak patterns are NOT hard blocks -> see review_hit().
"""
folded = _fold_ascii(text).lower()
hits = []
if _MC_CSAM.search(folded):
hits.append("sexual_abuse")
support = bool(_MC_SUPPORT_CONTEXT.search(folded))
news = bool(_MC_NEWS_CONTEXT.search(folded))
if _MC_SELFHARM_1P.search(folded) and not support:
hits.append("self_harm")
elif _MC_SELFHARM_INSTR.search(folded) and _MC_INSTR_MARK.search(folded) and not support and not news:
hits.append("self_harm")
if _MC_THREAT.search(folded) and not news:
hits.append("violence")
if _MC_SCHOOL.search(folded) and not news:
hits.append("violence")
if _MC_HATE_GROUP.search(folded) and not news:
hits.append("hate_harassment")
legal = bool(_MC_LEGAL_WEAPON.search(folded))
bomb_benign = bool(_MC_BOMB_BENIGN.search(folded))
pest = bool(_MC_PEST.search(folded))
if _MC_WEAPONS_DIY.search(folded) and not news:
hits.append("weapons_explosives")
if _MC_DRUGS_DIY.search(folded) and not news:
hits.append("drugs")
if _MC_CYBER_DIY.search(folded) and not _MC_DEFENSIVE.search(folded) and not news:
hits.append("cybercrime_fraud")
if _MC_MARKET.search(folded) and not news:
hits.append("illegal_marketplace")
if _MC_POISON.search(folded) and not news and not pest:
hits.append("violence")
# downgrades: legal weapons / benign bomb / pest control -> review only, not a block
if legal and "illegal_marketplace" in hits:
hits.remove("illegal_marketplace")
if bomb_benign and "illegal_marketplace" in hits:
hits.remove("illegal_marketplace")
if legal and "weapons_explosives" in hits and not _MC_WEAPONS_DIY.search(folded):
pass
return sorted(set(hits))
def review_hit(text: str) -> list[str]:
"""Jailbreak / prompt-injection / legally-ambiguous patterns -> needs_review, NOT a hard block."""
folded = _fold_ascii(text).lower()
out = []
if _MC_JAILBREAK.search(folded):
out.append("jailbreak_injection")
if _MC_LEGAL_WEAPON.search(folded) and (_MC_MARKET.search(folded) or _MC_WEAPONS_DIY.search(folded)):
out.append("weapons_explosives")
if _MC_BOMB_BENIGN.search(folded) and _MC_MARKET.search(folded):
out.append("illegal_marketplace")
return sorted(set(out))
def is_support_context(text: str) -> bool:
"""Self-harm mention in a helping context -> route to support, not a block."""
folded = _fold_ascii(text).lower()
return bool((_MC_SELFHARM_1P.search(folded) or _MC_SELFHARM_INSTR.search(folded))
and _MC_SUPPORT_CONTEXT.search(folded))
|