| """Hate-speech training data: teacher-labelled Civil pool (hate/build_pool.py) + teacher-labelled generated texts (hate/gen_hate.py). |
| v2 adds teacher-labelled real social-media texts (hate/build_pool2.py) with --real SHARDS..., repeated REAL_MULT times (default 2) in train. |
| Val = 5% held out: Civil by source comment (a comment and its translations stay together), generated and real texts at random. |
| python hate/build_data.py OUT_DIR TOX_V0_TRAIN.parquet --civil SHARDS... --gen SHARDS... [--real SHARDS...]""" |
| import glob, json, os, re, sys |
| import numpy as np, pandas as pd |
| L = ["hate", "target_race", "target_religion", "target_origin", "target_gender", "target_sexuality", "target_age", "target_disability"] |
| norm = lambda s: re.sub(r"\W+", " ", s.lower()).strip() |
| ev = {norm(t) for p in glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), "evals", "*.parquet")) for t in pd.read_parquet(p).text} |
| out, v0 = sys.argv[1:3]; a = sys.argv[3:]; ci, gi = a.index("--civil"), a.index("--gen"); ri = a.index("--real") if "--real" in a else len(a) |
| civ = pd.concat([pd.read_parquet(p) for p in a[ci + 1:gi]], ignore_index=True) |
| gen = pd.concat([pd.read_parquet(p) for p in a[gi + 1:ri]], ignore_index=True).assign(source="gen") |
| real = pd.concat([pd.read_parquet(p) for p in a[ri + 1:]], ignore_index=True) if ri < len(a) else pd.DataFrame(columns=["text", "lang", "source"] + L) |
| src = pd.read_parquet(v0, columns=["text", "src_idx"]).drop_duplicates("text").set_index("text").src_idx |
| civ["grp"] = civ.text.map(src).fillna(-1).astype(int) |
| rng = np.random.RandomState(0) |
| vg = set(rng.choice(civ.grp.unique(), int(0.05 * civ.grp.nunique()), replace=False)) |
| civ["val"] = civ.grp.isin(vg); gen["val"] = rng.rand(len(gen)) < 0.05; real["val"] = rng.rand(len(real)) < 0.05 |
| d = pd.concat([civ, gen, real], ignore_index=True) |
| k = d.text.map(norm); n_ev = int(k.isin(ev).sum()); d = d[~k.isin(ev) & (k.str.len() >= 3)].drop_duplicates("text") |
| cols = ["text", "lang", "source"] + L |
| os.makedirs(out, exist_ok=True) |
| tr, va = d[~d.val][cols], d[d.val][cols] |
| real_tr = tr[tr.source.isin(["mhs", "hatexplain"])] |
| tr = pd.concat([tr] + [real_tr] * (int(os.environ.get("REAL_MULT", 2)) - 1)).sample(frac=1, random_state=0) |
| tr.to_parquet(f"{out}/train.parquet"); va.to_parquet(f"{out}/val.parquet") |
| st = dict(train=len(tr), val=len(va), real_mult=int(os.environ.get("REAL_MULT", 2)) if len(real) else 0, removed_eval_overlap=n_ev, by_source=d.source.value_counts().to_dict(), |
| hate_ge_05={s: round(float((g.hate >= 0.5).mean()), 3) for s, g in d.groupby("source")}, |
| target_ge_05_among_hate={l: round(float((d[d.hate >= 0.5][l] >= 0.5).mean()), 3) for l in L[1:]}, |
| gen_hate_by_intended={k: round(float(v), 3) for k, v in d[d.source == "gen"].groupby("intended").hate.mean().items() if k in ("hateful", "non-hateful", "offensive")} if "intended" in d else {}, |
| n_lang=int(d.lang.nunique())) |
| json.dump(st, open(f"{out}/stats.json", "w"), indent=1); print(json.dumps(st, indent=1)) |
|
|
