File size: 5,336 Bytes
c1e5dc4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Native (not translated) texts for a multilingual hate-speech classifier, Qwen3.8-27B via vLLM. Per prompt: one language, one
platform and one protected-characteristic category; 10 texts: 4 hateful toward a group of that category (explicit and implicit/coded
forms), 3 non-hateful texts that mention such groups (neutral or positive statements, news reporting, counter-speech or quoting hate
to criticise it), 3 offensive but not hate speech (insults or profanity aimed at an individual, or attacks on non-protected targets
such as politicians, sports fans or companies). Labels come later from the teacher (hate/teacher_hate.py); the requested type is kept
only for statistics. No benchmark texts or templates are shown. N=... SHARD=i OUT=dir python hate/gen_hate.py -> $OUT/hate_<shard>.jsonl"""
import json, os, random, sys
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "sentiment"))
from gen_sent import parse_json
import importlib.util
_sp = importlib.util.spec_from_file_location("sc", os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "sentiment", "common.py"))
_m = importlib.util.module_from_spec(_sp); _sp.loader.exec_module(_m); hard_exit = _m.hard_exit
LANGS = ["English", "Spanish", "French", "German", "Italian", "Portuguese (Brazil)", "Portuguese (Portugal)", "Dutch", "Polish", "Czech",
"Slovak", "Romanian", "Hungarian", "Greek", "Bulgarian", "Serbian", "Croatian", "Ukrainian", "Russian", "Swedish", "Danish",
"Norwegian", "Finnish", "Turkish", "Arabic (Modern Standard)", "Arabic (Egyptian)", "Arabic (Gulf)", "Arabic (Levantine)",
"Arabic (Maghrebi)", "Hebrew", "Persian", "Urdu", "Hindi", "Hinglish (Hindi in Latin letters mixed with English)", "Bengali",
"Tamil", "Telugu", "Marathi", "Chinese (Simplified)", "Chinese (Traditional)", "Japanese", "Korean", "Vietnamese", "Thai",
"Indonesian", "Malay", "Tagalog", "Swahili", "Amharic", "Hausa", "Yoruba", "Afrikaans"]
CATS = ["race or ethnicity", "religion", "nationality, national origin or migration status (immigrants, refugees)",
"gender (women, men, transgender or non-binary people)", "sexual orientation (gay, lesbian, bisexual people)",
"disability (physical, mental or cognitive)", "age (old people, young people)"]
PLATFORMS = ["comments under a news article", "a YouTube comment section", "an online game chat", "a football fan forum", "replies on a social network",
"a political discussion group", "a messaging group", "a local community forum", "a meme page", "an anonymous imageboard"]
def prompt(lang, cat, platform):
return f"""You are creating training data for a content-moderation classifier that must recognise hate speech written in {lang}.
Write 10 different realistic texts from {platform}, in {lang}, as native speakers really write them (slang, abbreviations,
dialect, emojis, typos where natural; do not translate from English). Focus on groups defined by {cat} that are actually targeted
in {lang}-speaking societies:
- 4 HATEFUL texts against such a group, of different kinds: slurs, dehumanising comparisons, harmful stereotypes, calls for exclusion,
discrimination or violence, and implicit or coded hate (dog whistles, "jokes", "just asking questions").
- 3 NON-HATEFUL texts that mention such groups: neutral or positive statements, a news-style report, counter-speech that rejects hate,
or a quote of a hateful remark in order to criticise it.
- 3 OFFENSIVE BUT NOT HATE SPEECH texts: insults or profanity aimed at one person without referring to the group, or harsh attacks on
non-protected targets (politicians, a sports team's fans, a company, a government's policy).
Return only JSON: {{"texts": [{{"text": "...", "type": "hateful|non-hateful|offensive"}}, ...]}}"""
if __name__ == "__main__":
from vllm import LLM, SamplingParams
n, shard, out = int(os.environ.get("N", 3000)), int(os.environ.get("SHARD", 0)), os.environ["OUT"]
rng = random.Random(9800 + shard)
jobs = [(rng.choice(LANGS), rng.choice(CATS), rng.choice(PLATFORMS)) for _ in range(n)]
llm = LLM(os.environ.get("MODEL", "Qwen/Qwen3.8-27B-FP8"), max_model_len=4096, gpu_memory_utilization=0.92, max_num_seqs=512,
limit_mm_per_prompt={"image": 0, "video": 0})
outs = llm.chat([[{"role": "user", "content": prompt(*j)}] for j in jobs], SamplingParams(temperature=0.95, top_p=0.95, max_tokens=2500, seed=800 + shard),
chat_template_kwargs={"enable_thinking": False})
os.makedirs(out, exist_ok=True); k = 0; refused = 0
with open(f"{out}/hate_{shard}.jsonl", "w") as f:
for (l, c, p), o in zip(jobs, outs):
j = parse_json(o.outputs[0].text)
items = (j or {}).get("texts", []) if isinstance(j, dict) else []
refused += not items
for it in items:
if isinstance(it, dict) and isinstance(it.get("text"), str) and 3 <= len(it["text"]) <= 1000:
t = it["text"].encode("utf-8", "ignore").decode().strip()
f.write(json.dumps(dict(text=t, lang=l, category=c.split(" (")[0], platform=p, intended=str(it.get("type", ""))), ensure_ascii=False) + "\n"); k += 1
print("texts", k, "from", len(jobs), "prompts; empty/refused", refused, flush=True); hard_exit(0)
|