| |
| """在 earlier val 2,927 条上用 jev-1.13 重判，和 v1 标签 / 本地模型同口径对比。 |
| |
| 注意：earlier val 的标签来自 v1，不是人工真值；这组是"与 v1 一致率"，不是绝对正确率。 |
| earlier val 没有 sender/bio/recent context，state 只带 message.text；教师按文本判。 |
| """ |
| import json |
| import sys |
| from concurrent.futures import ThreadPoolExecutor, as_completed |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parent)) |
| from clean_with_teacher import load_env, request_teacher |
| from sklearn.metrics import accuracy_score, precision_recall_fscore_support |
|
|
| KINDS = ["crypto", "gambling", "porn", "porn_bait", "promo", "scam", "spam_flood"] |
|
|
|
|
| def make_state(text): |
| return { |
| "chat": {"id": 0, "title": "production group"}, |
| "message": {"text": text, "has_link": any(x in text for x in ("http", "t.me/", "www.")), |
| "has_mention": "@" in text, "has_media": False, "is_forwarded": False, |
| "length": len(text)}, |
| "sender": {"user_id": 0, "username": "", "first_name": "", "last_name": "", |
| "bio": "", "bio_links": [], "age_hours": 0, "age_known": False, |
| "msgs_in_group": 0, "prior_ad_hits": 0}, |
| "recent_context": [], "known_ad_patterns": "", "known_false_positives": "", |
| "matched_rules": [], |
| } |
|
|
|
|
| def main(): |
| cfg = load_env() |
| template = json.load(open(".work/teacher/jev_request_template.json", encoding="utf-8")) |
| q = template["questions"] |
| rows = [json.loads(l) for l in open("data/optimized_4h/earlier_val.jsonl", encoding="utf-8")] |
| out_path = Path("data/cleaned/earlier_val_jev.jsonl") |
| out_path.parent.mkdir(parents=True, exist_ok=True) |
| if out_path.exists(): |
| out_path.unlink() |
| results = [] |
|
|
| def work(r): |
| resp = request_teacher(cfg["TEACHER_URL"], cfg["TEACHER_KEY"], |
| cfg.get("TEACHER_MODEL", "jev-1.13"), q, make_state(r["text"])) |
| a = resp.get("answers") or {} |
| ia = (a.get("is_ad") or {}).get("choice") |
| kd = (a.get("ad_kind") or {}).get("choice") |
| sc = (a.get("ad_scope") or {}).get("choice") |
| sv = (a.get("severity") or {}).get("score") |
| return {"text": r["text"], "v1_label_bin": r["label_bin"], "v1_kind": r["label_kind"], |
| "v1_scope": r.get("label_scope"), "v1_sev": r.get("label_sev"), |
| "jev_label": "ad" if ia == "ad" else "normal", |
| "jev_kind": kd or "none", "jev_scope": sc or "message", "jev_sev": sv, |
| "usage": resp.get("usage")} |
|
|
| with ThreadPoolExecutor(max_workers=256) as ex: |
| futs = [ex.submit(work, r) for r in rows] |
| for i, f in enumerate(as_completed(futs), 1): |
| results.append(f.result()) |
| if i % 200 == 0: |
| print(f"progress {i}/{len(rows)}", flush=True) |
| with open(out_path, "w", encoding="utf-8") as f: |
| for r in results: |
| f.write(json.dumps(r, ensure_ascii=False) + "\n") |
|
|
| y = [r["v1_label_bin"] for r in results] |
| p = [0 if r["jev_label"] == "ad" else 1 for r in results] |
| prec, rec, f1, _ = precision_recall_fscore_support(y, p, labels=[0], average=None, zero_division=0) |
| print(f"jev vs v1: acc={accuracy_score(y,p):.4f} adP={prec[0]:.4f} adR={rec[0]:.4f} adF1={f1[0]:.4f}") |
| kidx = [i for i, r in enumerate(results) if r["v1_kind"] >= 0] |
| if kidx: |
| yk = [results[i]["v1_kind"] for i in kidx] |
| pk = [] |
| for i in kidx: |
| k = results[i]["jev_kind"] |
| pk.append(KINDS.index(k) if k in KINDS else -1) |
| valid = [(a, b) for a, b in zip(yk, pk) if b >= 0] |
| if valid: |
| ya = [a for a, _ in valid]; pa = [b for _, b in valid] |
| print(f"jev kind ns={len(valid)} acc={accuracy_score(ya,pa):.4f}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
