r12 assets: pondernet/gen/r12/build_mix_r12.py

pondernet/gen/r12/build_mix_r12.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""build_mix_r12.py — r12 混合语料组装
+配比: chat_v3 x2 (2760, 54%) + zh_cot 1049 (不动, CE 锚) + alpaca_zh 500 + alpaca_en 400
+     + agent 400 (保留原 system)  = 5,109 行
+"""
+import json, random
+
+random.seed(12)
+
+v3 = [json.loads(l) for l in open("/home/z/my-project/download/chat_synth_v3.jsonl")]
+mix11 = [json.loads(l) for l in open("/tmp/mix_r11.jsonl")]
+
+def cat_of(r):
+    tid = r.get("task_id", "") or ""
+    if tid.startswith("cs_"):
+        return "chat_v2"
+    return r.get("cat", "") or "zh_cot"
+
+groups = {}
+for r in mix11:
+    groups.setdefault(cat_of(r), []).append(r)
+print("mix11 组成:", {k: len(v) for k, v in groups.items()})
+
+def reply_len(r):
+    return len(r["messages"][-1]["content"])
+
+# alpaca 长度过滤 (反截断: 掐掉超长列表式回复)
+zh_pool = [r for r in groups["alpaca_zh"] if reply_len(r) <= 450]
+en_pool = [r for r in groups["alpaca_en"] if reply_len(r) <= 900]
+random.shuffle(zh_pool)
+random.shuffle(en_pool)
+alpaca_zh = zh_pool[:500]
+alpaca_en = en_pool[:400]
+
+agent = groups["file"] + groups["text"] + groups["math"] + groups["data"] + \
+    groups["git"] + groups["sysadmin"] + groups["script"]
+zh_cot = groups["zh_cot"]
+
+mix = []
+mix += v3 * 2                       # chat 主攻桶 x2 upsample
+mix += zh_cot                       # CE 锚, 原样保留
+mix += alpaca_zh
+mix += alpaca_en
+mix += agent
+
+random.shuffle(mix)
+out = "/home/z/my-project/download/mix_r12.jsonl"
+with open(out, "w", encoding="utf-8") as f:
+    for r in mix:
+        f.write(json.dumps(r, ensure_ascii=False) + "\n")
+
+from collections import Counter
+c = Counter(cat_of(r) if cat_of(r) != "chat_v2" else "chat_v3" for r in mix)
+print("mix_r12 总数:", len(mix))
+print("组成:", dict(c))
+sys_n = sum(1 for r in mix if r["messages"][0]["role"] == "system")
+print("含 system:", sys_n)
+print("WRITTEN", out)