#!/usr/bin/env python3 """构建优化训练集：teacher 清洗数据 + 之前 25k 数据合并，广告过采样 + 稀有 kind/severity 补齐。 - teacher 数据优先（同 text），早前数据补充；排除外部测试集文本，避免泄漏 - --gmeow 接入 gmeow 封禁广告语料（ads_final.jsonl，text/label/kind/scope/severity）， --gmeow_val_texts 指定留出 val 文本清单：留出行只进 val（并从 teacher/earlier 中剔除同文本防泄漏） - normal 的 scope/severity 置 -100（交给二分类 gate），scope/severity 头只学广告 - train 广告过采样到 20%；kind 每类最少 1000；severity 0/1 最少 500/1000 - 重复样本用 marker_augment 生成标记形态变体，降低死记 - 输出 data/optimized_4h/{train,dev,val,holdout827}.jsonl + teacher_holdout/earlier_val """ import argparse import json import random from collections import Counter from pathlib import Path import numpy as np from sklearn.model_selection import train_test_split from marker_augment import augment_markers KINDS = ["crypto", "gambling", "porn", "porn_bait", "promo", "scam", "spam_flood"] def load(p): return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()] def key(text: str) -> str: return "".join(text.split()).lower() def from_gmeow(r): sev = r.get("severity") return { "text": r["text"], "label_bin": 0, "label_kind": KINDS.index(r["kind"]) if r.get("kind") in KINDS else -100, "label_scope": (0 if r.get("scope") == "account" else 1) if r.get("scope") in ("account", "message") else -100, "label_sev": int(round(sev)) if isinstance(sev, (int, float)) and 0 <= sev <= 3 else -100, "source": "gmeow", } def from_teacher(r): is_ad = r.get("label") == "ad" kind = r.get("ad_kind") return { "text": r["text"], "label_bin": 0 if is_ad else 1, "label_kind": KINDS.index(kind) if (is_ad and kind in KINDS) else -100, "label_scope": (0 if r.get("scope") == "account" else 1) if is_ad else -100, "label_sev": (int(r.get("severity_bucket", -100))) if is_ad else -100, "source": "teacher", } def from_earlier(r): is_ad = r["label_bin"] == 0 return { "text": r["text"], "label_bin": r["label_bin"], "label_kind": r["label_kind"] if is_ad else -100, "label_scope": r["label_scope"] if is_ad else -100, "label_sev": r["label_sev"] if is_ad else -100, "source": "earlier", } def main(): ap = argparse.ArgumentParser() ap.add_argument("--teacher", default="data/cleaned/archive_latest_full_clean.jsonl") ap.add_argument("--earlier_train", default="data/processed_4h/train.jsonl") ap.add_argument("--external_teacher", default="data/processed_teacher/holdout827.jsonl") ap.add_argument("--external_earlier", default="data/processed_4h/val.jsonl") ap.add_argument("--out", default="data/optimized_4h") ap.add_argument("--seed", type=int, default=42) ap.add_argument("--ad_ratio", type=float, default=0.20) ap.add_argument("--kind_min", type=int, default=1000) ap.add_argument("--sev0_min", type=int, default=500) ap.add_argument("--sev1_min", type=int, default=1000) ap.add_argument("--gmeow", default=None, help="gmeow ads_final.jsonl（text/label/kind/scope/severity）") ap.add_argument("--gmeow_val_texts", default=None, help="jsonl({text})：gmeow 留出 val 的文本清单") args = ap.parse_args() teacher = load(args.teacher) earlier = load(args.earlier_train) gmeow_hold, gmeow_in = [], [] if args.gmeow: hold_keys = set() if args.gmeow_val_texts: hold_keys = {key(r["text"]) for r in load(args.gmeow_val_texts)} gmeow = [from_gmeow(r) for r in load(args.gmeow) if r.get("label") == "ad" and r.get("kind") in KINDS] gmeow_hold = [m for m in gmeow if key(m["text"]) in hold_keys] gmeow_in = [m for m in gmeow if key(m["text"]) not in hold_keys] print(f"gmeow ads={len(gmeow)} train_pool={len(gmeow_in)} val_hold={len(gmeow_hold)}") ext_t = [from_earlier(r) if "label_bin" in r else from_teacher(r) for r in load(args.external_teacher)] ext_e = [from_earlier(r) if "label_bin" in r else from_teacher(r) for r in load(args.external_earlier)] ext_texts = {r["text"] for r in ext_t + ext_e} hold_keys = {key(m["text"]) for m in gmeow_hold} combined = {} for m in gmeow_in: if m["text"] not in ext_texts: combined[m["text"]] = m for r in earlier: m = from_earlier(r) if m["text"] not in ext_texts and key(m["text"]) not in hold_keys: combined[m["text"]] = m for r in teacher: m = from_teacher(r) if m["text"] not in ext_texts and key(m["text"]) not in hold_keys: combined[m["text"]] = m rows = list(combined.values()) print("combined unique", len(rows), "ads", sum(1 for r in rows if r["label_bin"] == 0), "kind", Counter(r["label_kind"] for r in rows if r["label_kind"] >= 0), "sev", Counter(r["label_sev"] for r in rows if r["label_sev"] >= 0)) y = np.array([r["label_bin"] for r in rows]) idx = np.arange(len(rows)) tr, rest = train_test_split(idx, test_size=0.10, stratify=y, random_state=args.seed) dev, rest2 = train_test_split(rest, test_size=0.50, stratify=y[rest], random_state=args.seed) val, hold = train_test_split(rest2, test_size=0.50, stratify=y[rest2], random_state=args.seed) splits = {"train": [rows[i] for i in tr], "dev": [rows[i] for i in dev], "val": [rows[i] for i in val], "holdout827": [rows[i] for i in hold]} splits["val"] = splits["val"] + gmeow_hold if gmeow_hold: print("gmeow val holdout", len(gmeow_hold), "kind", Counter(KINDS[m["label_kind"]] for m in gmeow_hold if m["label_kind"] >= 0)) train = splits["train"] rng = random.Random(args.seed) normals = sum(1 for r in train if r["label_bin"] == 1) ads = [r for r in train if r["label_bin"] == 0] target_ads = int(args.ad_ratio / (1 - args.ad_ratio) * normals) extra = max(0, target_ads - len(ads)) def dup(src, n): out = [] for i in range(n): r = dict(rng.choice(src)) r["text"] = augment_markers(r["text"], random.Random(args.seed * 1000 + i), p=0.5) r["source"] = "oversample" out.append(r) return out train += dup(ads, extra) for k in range(len(KINDS)): pool = [r for r in train if r["label_kind"] == k] if pool and len(pool) < args.kind_min: train += dup(pool, args.kind_min - len(pool)) for sev, minimum in ((0, args.sev0_min), (1, args.sev1_min)): pool = [r for r in train if r["label_sev"] == sev] if pool and len(pool) < minimum: train += dup(pool, minimum - len(pool)) splits["train"] = train out = Path(args.out) out.mkdir(parents=True, exist_ok=True) for name, data in splits.items(): with open(out / f"{name}.jsonl", "w", encoding="utf-8") as f: for x in data: f.write(json.dumps(x, ensure_ascii=False) + "\n") print(name, len(data), "bin", Counter(x["label_bin"] for x in data), "kind", Counter(x["label_kind"] for x in data if x["label_kind"] >= 0), "scope", Counter(x["label_scope"] for x in data if x["label_scope"] >= 0), "sev", Counter(x["label_sev"] for x in data if x["label_sev"] >= 0)) for name, data in (("teacher_holdout", ext_t), ("earlier_val", ext_e)): with open(out / f"{name}.jsonl", "w", encoding="utf-8") as f: for x in data: f.write(json.dumps(x, ensure_ascii=False) + "\n") print(name, len(data), "bin", Counter(x["label_bin"] for x in data), "kind", Counter(x["label_kind"] for x in data if x["label_kind"] >= 0)) print("out", out.resolve()) if __name__ == "__main__": main()