File size: 2,531 Bytes
f8d2531 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | #!/usr/bin/env python3
"""把 teacher 清洗输出转换成训练用 jsonl（teacher 标签为准）。
输入: clean_with_teacher.py 的 */latest_full.jsonl（含 teacher + v1）
输出: text / label(ad|normal) / ad_kind / scope / severity / severity_bucket / 教师置信度 / v1 对照
"""
import argparse
import json
from collections import Counter
from pathlib import Path
def bucket(score):
if score is None:
return -1
if score < 0.5:
return 0
if score < 1.5:
return 1
if score < 2.5:
return 2
return 3
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--input", default="data/cleaned/archive_latest_full.jsonl")
ap.add_argument("--out", default="data/cleaned/archive_latest_full_clean.jsonl")
args = ap.parse_args()
n = 0
counts = Counter()
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
with open(args.input, encoding="utf-8") as f, open(out, "w", encoding="utf-8") as w:
for line in f:
if not line.strip():
continue
r = json.loads(line)
t = r.get("teacher") or {}
is_ad = (t.get("is_ad") or {}).get("choice") == "ad"
kind = (t.get("ad_kind") or {}).get("choice") or "none"
scope = (t.get("ad_scope") or {}).get("choice") or "message"
sev = (t.get("severity") or {}).get("score")
row = {
"text": r.get("text"),
"label": "ad" if is_ad else "normal",
"ad_kind": kind if is_ad else "normal",
"scope": scope if is_ad else "message",
"severity": sev if is_ad else 0.0,
"severity_bucket": bucket(sev) if is_ad else 0,
"confidence": (t.get("is_ad") or {}).get("confidence"),
"teacher": "jev-1.13",
"v1_label": "ad" if r.get("v1_label") == "ad" else "normal",
"v1_kind": r.get("v1_kind"),
"v1_decider": r.get("v1_decider"),
"v1_scope": r.get("v1_scope"),
"v1_severity": r.get("v1_severity"),
"chat_id": r.get("chat_id"),
"message_id": r.get("message_id"),
}
w.write(json.dumps(row, ensure_ascii=False) + "\n")
n += 1
counts[(row["label"], row["ad_kind"])] += 1
print("wrote", n, "->", out.resolve())
print("label/kind:", dict(counts.most_common(20)))
if __name__ == "__main__":
main()
|