File size: 1,592 Bytes
c1e5dc4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | """Real social-media texts for hate v2 (teacher-labelled later; the datasets' own labels are NOT used):
- Measuring Hate Speech (CC-BY-4.0) comments that are not in the 6,000-comment eval sample (hate/evals/mhs.parquet);
- HateXplain (CC-BY-4.0) train + validation posts (the test split stays the eval set).
Eval overlap (all hate/evals sets) removed by normalised exact match. python hate/build_pool2.py OUT.parquet"""
import glob, json, os, re, sys, urllib.request
import pandas as pd
from huggingface_hub import hf_hub_download
norm = lambda s: re.sub(r"\W+", " ", s.lower()).strip()
ev = {norm(t) for p in glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), "evals", "*.parquet")) for t in pd.read_parquet(p).text}
m = pd.read_parquet(hf_hub_download("ucberkeley-dlab/measuring-hate-speech", "measuring-hate-speech.parquet", repo_type="dataset"), columns=["comment_id", "text"])
m = m.drop_duplicates("comment_id")[["text"]].assign(source="mhs", lang="English")
B = "https://raw.githubusercontent.com/hate-alert/HateXplain/master/Data/"
ds = json.load(urllib.request.urlopen(B + "dataset.json")); sp = json.load(urllib.request.urlopen(B + "post_id_divisions.json"))
h = pd.DataFrame([dict(text=" ".join(ds[p]["post_tokens"]), source="hatexplain", lang="English") for p in sp["train"] + sp["val"]])
d = pd.concat([m, h], ignore_index=True); k = d.text.map(norm)
d = d[~k.isin(ev) & (k.str.len() >= 3)].drop_duplicates("text").reset_index(drop=True)
d.to_parquet(sys.argv[1]); print(len(d), d.source.value_counts().to_dict(), "eval overlap removed", int(k.isin(ev).sum()))
|