File size: 975 Bytes
c1e5dc4 | 1 2 3 4 5 6 7 8 9 10 11 12 | """Real-comment pool for hate-speech training: Civil Comments (CC0) and their machine translations from the toxicity v1.0 data
(tox/build_tox.py): 60k rows with identity_attack >= 0.2 + 60k other random rows; eval overlap removed. Labels come from the
teacher (hate/teacher_hate.py label). python hate/build_pool.py TOX_V0_TRAIN.parquet OUT.parquet"""
import glob, os, re, sys
import pandas as pd
norm = lambda s: re.sub(r"\W+", " ", s.lower()).strip()
ev = {norm(t) for p in glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), "evals", "*.parquet")) for t in pd.read_parquet(p).text}
v = pd.read_parquet(sys.argv[1]); v = v[~v.text.map(norm).isin(ev)]
ia = v.identity_attack >= 0.2
d = pd.concat([v[ia].sample(60000, random_state=0), v[~ia].sample(60000, random_state=0)])[["text", "lang"]].assign(source="civil")
d = d.sample(frac=1, random_state=0).reset_index(drop=True); d.to_parquet(sys.argv[2]); print(len(d), d.lang.value_counts().head(5).to_dict())
|