|
|
| """构建优化训练集：teacher 清洗数据 + 之前 25k 数据合并，广告过采样 + 稀有 kind/severity 补齐。
|
|
|
| - teacher 数据优先（同 text），早前数据补充；排除外部测试集文本，避免泄漏
|
| - --gmeow 接入 gmeow 封禁广告语料（ads_final.jsonl，text/label/kind/scope/severity），
|
| --gmeow_val_texts 指定留出 val 文本清单：留出行只进 val（并从 teacher/earlier 中剔除同文本防泄漏）
|
| - normal 的 scope/severity 置 -100（交给二分类 gate），scope/severity 头只学广告
|
| - train 广告过采样到 20%；kind 每类最少 1000；severity 0/1 最少 500/1000
|
| - 重复样本用 marker_augment 生成标记形态变体，降低死记
|
| - 输出 data/optimized_4h/{train,dev,val,holdout827}.jsonl + teacher_holdout/earlier_val
|
| """
|
| import argparse
|
| import json
|
| import random
|
| from collections import Counter
|
| from pathlib import Path
|
|
|
| import numpy as np
|
| from sklearn.model_selection import train_test_split
|
|
|
| from marker_augment import augment_markers
|
|
|
| KINDS = ["crypto", "gambling", "porn", "porn_bait", "promo", "scam", "spam_flood"]
|
|
|
|
|
| def load(p):
|
| return [json.loads(l) for l in open(p, encoding="utf-8") if l.strip()]
|
|
|
|
|
| def key(text: str) -> str:
|
| return "".join(text.split()).lower()
|
|
|
|
|
| def from_gmeow(r):
|
| sev = r.get("severity")
|
| return {
|
| "text": r["text"],
|
| "label_bin": 0,
|
| "label_kind": KINDS.index(r["kind"]) if r.get("kind") in KINDS else -100,
|
| "label_scope": (0 if r.get("scope") == "account" else 1) if r.get("scope") in ("account", "message") else -100,
|
| "label_sev": int(round(sev)) if isinstance(sev, (int, float)) and 0 <= sev <= 3 else -100,
|
| "source": "gmeow",
|
| }
|
|
|
|
|
| def from_teacher(r):
|
| is_ad = r.get("label") == "ad"
|
| kind = r.get("ad_kind")
|
| return {
|
| "text": r["text"],
|
| "label_bin": 0 if is_ad else 1,
|
| "label_kind": KINDS.index(kind) if (is_ad and kind in KINDS) else -100,
|
| "label_scope": (0 if r.get("scope") == "account" else 1) if is_ad else -100,
|
| "label_sev": (int(r.get("severity_bucket", -100))) if is_ad else -100,
|
| "source": "teacher",
|
| }
|
|
|
|
|
| def from_earlier(r):
|
| is_ad = r["label_bin"] == 0
|
| return {
|
| "text": r["text"],
|
| "label_bin": r["label_bin"],
|
| "label_kind": r["label_kind"] if is_ad else -100,
|
| "label_scope": r["label_scope"] if is_ad else -100,
|
| "label_sev": r["label_sev"] if is_ad else -100,
|
| "source": "earlier",
|
| }
|
|
|
|
|
| def main():
|
| ap = argparse.ArgumentParser()
|
| ap.add_argument("--teacher", default="data/cleaned/archive_latest_full_clean.jsonl")
|
| ap.add_argument("--earlier_train", default="data/processed_4h/train.jsonl")
|
| ap.add_argument("--external_teacher", default="data/processed_teacher/holdout827.jsonl")
|
| ap.add_argument("--external_earlier", default="data/processed_4h/val.jsonl")
|
| ap.add_argument("--out", default="data/optimized_4h")
|
| ap.add_argument("--seed", type=int, default=42)
|
| ap.add_argument("--ad_ratio", type=float, default=0.20)
|
| ap.add_argument("--kind_min", type=int, default=1000)
|
| ap.add_argument("--sev0_min", type=int, default=500)
|
| ap.add_argument("--sev1_min", type=int, default=1000)
|
| ap.add_argument("--gmeow", default=None,
|
| help="gmeow ads_final.jsonl（text/label/kind/scope/severity）")
|
| ap.add_argument("--gmeow_val_texts", default=None,
|
| help="jsonl({text})：gmeow 留出 val 的文本清单")
|
| args = ap.parse_args()
|
|
|
| teacher = load(args.teacher)
|
| earlier = load(args.earlier_train)
|
|
|
| gmeow_hold, gmeow_in = [], []
|
| if args.gmeow:
|
| hold_keys = set()
|
| if args.gmeow_val_texts:
|
| hold_keys = {key(r["text"]) for r in load(args.gmeow_val_texts)}
|
| gmeow = [from_gmeow(r) for r in load(args.gmeow)
|
| if r.get("label") == "ad" and r.get("kind") in KINDS]
|
| gmeow_hold = [m for m in gmeow if key(m["text"]) in hold_keys]
|
| gmeow_in = [m for m in gmeow if key(m["text"]) not in hold_keys]
|
| print(f"gmeow ads={len(gmeow)} train_pool={len(gmeow_in)} val_hold={len(gmeow_hold)}")
|
|
|
| ext_t = [from_earlier(r) if "label_bin" in r else from_teacher(r) for r in load(args.external_teacher)]
|
| ext_e = [from_earlier(r) if "label_bin" in r else from_teacher(r) for r in load(args.external_earlier)]
|
| ext_texts = {r["text"] for r in ext_t + ext_e}
|
| hold_keys = {key(m["text"]) for m in gmeow_hold}
|
|
|
| combined = {}
|
| for m in gmeow_in:
|
| if m["text"] not in ext_texts:
|
| combined[m["text"]] = m
|
| for r in earlier:
|
| m = from_earlier(r)
|
| if m["text"] not in ext_texts and key(m["text"]) not in hold_keys:
|
| combined[m["text"]] = m
|
| for r in teacher:
|
| m = from_teacher(r)
|
| if m["text"] not in ext_texts and key(m["text"]) not in hold_keys:
|
| combined[m["text"]] = m
|
| rows = list(combined.values())
|
| print("combined unique", len(rows), "ads", sum(1 for r in rows if r["label_bin"] == 0),
|
| "kind", Counter(r["label_kind"] for r in rows if r["label_kind"] >= 0),
|
| "sev", Counter(r["label_sev"] for r in rows if r["label_sev"] >= 0))
|
|
|
| y = np.array([r["label_bin"] for r in rows])
|
| idx = np.arange(len(rows))
|
| tr, rest = train_test_split(idx, test_size=0.10, stratify=y, random_state=args.seed)
|
| dev, rest2 = train_test_split(rest, test_size=0.50, stratify=y[rest], random_state=args.seed)
|
| val, hold = train_test_split(rest2, test_size=0.50, stratify=y[rest2], random_state=args.seed)
|
| splits = {"train": [rows[i] for i in tr], "dev": [rows[i] for i in dev],
|
| "val": [rows[i] for i in val], "holdout827": [rows[i] for i in hold]}
|
| splits["val"] = splits["val"] + gmeow_hold
|
| if gmeow_hold:
|
| print("gmeow val holdout", len(gmeow_hold),
|
| "kind", Counter(KINDS[m["label_kind"]] for m in gmeow_hold if m["label_kind"] >= 0))
|
|
|
| train = splits["train"]
|
| rng = random.Random(args.seed)
|
| normals = sum(1 for r in train if r["label_bin"] == 1)
|
| ads = [r for r in train if r["label_bin"] == 0]
|
| target_ads = int(args.ad_ratio / (1 - args.ad_ratio) * normals)
|
| extra = max(0, target_ads - len(ads))
|
|
|
| def dup(src, n):
|
| out = []
|
| for i in range(n):
|
| r = dict(rng.choice(src))
|
| r["text"] = augment_markers(r["text"], random.Random(args.seed * 1000 + i), p=0.5)
|
| r["source"] = "oversample"
|
| out.append(r)
|
| return out
|
|
|
| train += dup(ads, extra)
|
| for k in range(len(KINDS)):
|
| pool = [r for r in train if r["label_kind"] == k]
|
| if pool and len(pool) < args.kind_min:
|
| train += dup(pool, args.kind_min - len(pool))
|
| for sev, minimum in ((0, args.sev0_min), (1, args.sev1_min)):
|
| pool = [r for r in train if r["label_sev"] == sev]
|
| if pool and len(pool) < minimum:
|
| train += dup(pool, minimum - len(pool))
|
| splits["train"] = train
|
|
|
| out = Path(args.out)
|
| out.mkdir(parents=True, exist_ok=True)
|
| for name, data in splits.items():
|
| with open(out / f"{name}.jsonl", "w", encoding="utf-8") as f:
|
| for x in data:
|
| f.write(json.dumps(x, ensure_ascii=False) + "\n")
|
| print(name, len(data), "bin", Counter(x["label_bin"] for x in data),
|
| "kind", Counter(x["label_kind"] for x in data if x["label_kind"] >= 0),
|
| "scope", Counter(x["label_scope"] for x in data if x["label_scope"] >= 0),
|
| "sev", Counter(x["label_sev"] for x in data if x["label_sev"] >= 0))
|
| for name, data in (("teacher_holdout", ext_t), ("earlier_val", ext_e)):
|
| with open(out / f"{name}.jsonl", "w", encoding="utf-8") as f:
|
| for x in data:
|
| f.write(json.dumps(x, ensure_ascii=False) + "\n")
|
| print(name, len(data), "bin", Counter(x["label_bin"] for x in data),
|
| "kind", Counter(x["label_kind"] for x in data if x["label_kind"] >= 0))
|
| print("out", out.resolve())
|
|
|
|
|
| if __name__ == "__main__":
|
| main()
|
|
|
