File size: 3,942 Bytes
e2492ec | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | #!/usr/bin/env python3
"""在 earlier val 2,927 条上用 jev-1.13 重判，和 v1 标签 / 本地模型同口径对比。
注意：earlier val 的标签来自 v1，不是人工真值；这组是"与 v1 一致率"，不是绝对正确率。
earlier val 没有 sender/bio/recent context，state 只带 message.text；教师按文本判。
"""
import json
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from clean_with_teacher import load_env, request_teacher # noqa: E402
from sklearn.metrics import accuracy_score, precision_recall_fscore_support # noqa: E402
KINDS = ["crypto", "gambling", "porn", "porn_bait", "promo", "scam", "spam_flood"]
def make_state(text):
return {
"chat": {"id": 0, "title": "production group"},
"message": {"text": text, "has_link": any(x in text for x in ("http", "t.me/", "www.")),
"has_mention": "@" in text, "has_media": False, "is_forwarded": False,
"length": len(text)},
"sender": {"user_id": 0, "username": "", "first_name": "", "last_name": "",
"bio": "", "bio_links": [], "age_hours": 0, "age_known": False,
"msgs_in_group": 0, "prior_ad_hits": 0},
"recent_context": [], "known_ad_patterns": "", "known_false_positives": "",
"matched_rules": [],
}
def main():
cfg = load_env()
template = json.load(open(".work/teacher/jev_request_template.json", encoding="utf-8"))
q = template["questions"]
rows = [json.loads(l) for l in open("data/optimized_4h/earlier_val.jsonl", encoding="utf-8")]
out_path = Path("data/cleaned/earlier_val_jev.jsonl")
out_path.parent.mkdir(parents=True, exist_ok=True)
if out_path.exists():
out_path.unlink()
results = []
def work(r):
resp = request_teacher(cfg["TEACHER_URL"], cfg["TEACHER_KEY"],
cfg.get("TEACHER_MODEL", "jev-1.13"), q, make_state(r["text"]))
a = resp.get("answers") or {}
ia = (a.get("is_ad") or {}).get("choice")
kd = (a.get("ad_kind") or {}).get("choice")
sc = (a.get("ad_scope") or {}).get("choice")
sv = (a.get("severity") or {}).get("score")
return {"text": r["text"], "v1_label_bin": r["label_bin"], "v1_kind": r["label_kind"],
"v1_scope": r.get("label_scope"), "v1_sev": r.get("label_sev"),
"jev_label": "ad" if ia == "ad" else "normal",
"jev_kind": kd or "none", "jev_scope": sc or "message", "jev_sev": sv,
"usage": resp.get("usage")}
with ThreadPoolExecutor(max_workers=256) as ex:
futs = [ex.submit(work, r) for r in rows]
for i, f in enumerate(as_completed(futs), 1):
results.append(f.result())
if i % 200 == 0:
print(f"progress {i}/{len(rows)}", flush=True)
with open(out_path, "w", encoding="utf-8") as f:
for r in results:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
y = [r["v1_label_bin"] for r in results]
p = [0 if r["jev_label"] == "ad" else 1 for r in results]
prec, rec, f1, _ = precision_recall_fscore_support(y, p, labels=[0], average=None, zero_division=0)
print(f"jev vs v1: acc={accuracy_score(y,p):.4f} adP={prec[0]:.4f} adR={rec[0]:.4f} adF1={f1[0]:.4f}")
kidx = [i for i, r in enumerate(results) if r["v1_kind"] >= 0]
if kidx:
yk = [results[i]["v1_kind"] for i in kidx]
pk = []
for i in kidx:
k = results[i]["jev_kind"]
pk.append(KINDS.index(k) if k in KINDS else -1)
valid = [(a, b) for a, b in zip(yk, pk) if b >= 0]
if valid:
ya = [a for a, _ in valid]; pa = [b for _, b in valid]
print(f"jev kind ns={len(valid)} acc={accuracy_score(ya,pa):.4f}")
if __name__ == "__main__":
main()
|