code/kod/faz6_prep_v15b.py

"""
Faz 6 v1.5b — sohbet-ağırlıklı bilingual SFT verisi üretici (çok-turlu {messages}).

v1 (faz6_prep_data.py = tek-tur Magpie+Quardo) KORUNUR; bu YENİ üretici sohbet için.
Karar: "tutarlıysa uzun/sohbet-ağırlıklı" (faz5 kapısına bağlı). İNSAN-öncelikli, distilasyon-kaçınma (public).

Kaynaklar (hepsi HF şema+lisans DOĞRULANDI 2026-07-02):
  oasst2  OpenAssistant/oasst2         Apache  ağaç(role/lang/rank)  EN+TR  İNSAN çok-turlu  ⭐
  tc_sft  lumees/turkish-corpus-100b   Apache  messages[]            TR     çok-turlu chat   ⭐  (data_files=sft/train.parquet)
  aya     CohereForAI/aya_dataset      Apache  {inputs,targets,lang} TR     İNSAN tek-tur    ⭐  (language=='Turkish')
  magpie  Magpie-Reasoning-V2-250K...  Llama   {instruction,response} EN    reasoning CoT       (Llama-gen: GPT/Claude'dan az riskli)
  quardo  Quardo/Turkish-Alpaca-GPT-4O (VARSAYILAN KAPALI: GPT-4o türevi → public distilasyon riski)

Çıktı: JSONL {"messages":[{"role","content"},...], "lang", "src"}. Tek-tur = 1 user+1 assistant (superset).
Şablon (SP özel-token YOK → düz metin): "### Sistem/### Kullanıcı/### Asistan". faz6_sft.py çok-tur maskeleme yolu GEREKİR (yanıt span'leri).

Çalıştırma (Colab/yerel; datasets+sentencepiece kurulu, HF login):
  HF_TOKEN=hf_xxx python faz6_prep_v15b.py --out sft_v15b.jsonl --max_len 2048 --n_en 20000 --n_tr 20000
"""
import os, sys, json, re, random, argparse

THINK_RE = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
SYS_PFX, USER_PFX, ASST_PFX = "### Sistem:\n", "### Kullanıcı:\n", "### Asistan:\n"
_PFX = {"system": SYS_PFX, "user": USER_PFX, "assistant": ASST_PFX}


# ───────────── saf-mantık (yerelde test edilebilir) ─────────────
def render(messages):
    """çok-turlu {messages} → düz metin (faz6_sft aynısını kullanmalı)."""
    return "\n\n".join(_PFX[m["role"]] + (m["content"] or "").strip() for m in messages)


def valid(messages):
    """en az 1 user+1 assistant, son mesaj assistant, boş içerik yok, rol sırası mantıklı."""
    if len(messages) < 2 or messages[-1]["role"] != "assistant":
        return False
    if any(not (m.get("content") or "").strip() for m in messages):
        return False
    non_sys = [m["role"] for m in messages if m["role"] != "system"]
    return non_sys[0] == "user"   # ilk (system dışı) mesaj user olmalı


def trim_trailing_user(msgs):
    while msgs and msgs[-1]["role"] == "user":
        msgs.pop()
    return msgs


# ───────────── tokenizer ─────────────
def load_tok(token):
    import sentencepiece as spm
    from huggingface_hub import hf_hub_download
    p = hf_hub_download("kdirgul/smartcore-v1", "tokenizer/tokenizer.model", repo_type="model", token=token)
    return spm.SentencePieceProcessor(model_file=p)


def tok_len(sp, messages):
    return len(sp.encode(render(messages), out_type=int)) + 1   # +1 eos


# ───────────── kaynak yükleyiciler → [{messages, lang, src}] ─────────────
def load_oasst(cap, strip_think, max_turns):
    """OASST2 ağacından en-iyi-sıralı (rank) konuşma yolunu çıkar. lang∈{en,tr}, silinmemiş, sentetik değil."""
    from datasets import load_dataset
    rows = list(load_dataset("OpenAssistant/oasst2", split="train"))   # ~135K mesaj, küçük
    children = {}
    for r in rows:
        children.setdefault(r["parent_id"], []).append(r)

    def best_assistant(node):
        kids = [k for k in children.get(node["message_id"], [])
                if k["role"] == "assistant" and not k["deleted"] and not k.get("synthetic", False)]
        if not kids:
            return None
        return min(kids, key=lambda k: (k["rank"] if k.get("rank") is not None else 999))

    out = []
    roots = [r for r in rows if r["parent_id"] is None and r["role"] == "prompter"
             and r["lang"] in ("en", "tr") and not r["deleted"]]
    for root in roots:
        msgs = [{"role": "user", "content": root["text"]}]
        node = root
        for _ in range(max_turns):
            a = best_assistant(node)
            if a is None:
                break
            txt = THINK_RE.sub("", a["text"]).strip() if strip_think else a["text"]
            msgs.append({"role": "assistant", "content": txt})
            pk = [k for k in children.get(a["message_id"], [])
                  if k["role"] == "prompter" and not k["deleted"]]
            if not pk:
                break
            node = pk[0]
            msgs.append({"role": "user", "content": node["text"]})
        msgs = trim_trailing_user(msgs)
        if valid(msgs):
            out.append({"messages": msgs, "lang": root["lang"], "src": "oasst2"})
        if cap and len(out) >= cap:
            break
    return out


def load_tc_sft(cap, strip_think, max_turns):
    from datasets import load_dataset
    ds = load_dataset("lumees/turkish-corpus-100b", data_files="sft/train.parquet", split="train", streaming=True)
    out = []
    for ex in ds:
        msgs = [{"role": m["role"], "content": (m.get("content") or "")} for m in (ex.get("messages") or [])]
        msgs = msgs[:max_turns * 2 + 1]          # sistem + N tur
        if strip_think:
            for m in msgs:
                if m["role"] == "assistant":
                    m["content"] = THINK_RE.sub("", m["content"]).strip()
        msgs = trim_trailing_user(msgs)
        if valid(msgs):
            out.append({"messages": msgs, "lang": "tr", "src": "tc_sft"})
        if cap and len(out) >= cap:
            break
    return out


def load_aya(cap):
    from datasets import load_dataset
    ds = load_dataset("CohereForAI/aya_dataset", split="train")   # insan, çok-dilli
    out = []
    for ex in ds:
        if ex.get("language") != "Turkish":
            continue
        msgs = [{"role": "user", "content": ex.get("inputs") or ""},
                {"role": "assistant", "content": ex.get("targets") or ""}]
        if valid(msgs):
            out.append({"messages": msgs, "lang": "tr", "src": "aya"})
        if cap and len(out) >= cap:
            break
    return out


def load_magpie(cap, strip_think, quals, diffs, max_len_chars):
    from datasets import load_dataset
    ds = load_dataset("Magpie-Align/Magpie-Reasoning-V2-250K-CoT-Deepseek-R1-Llama-70B",
                      split="train", streaming=True)
    out = []
    for ex in ds:
        if (ex.get("language") or "EN").upper() != "EN":
            continue
        if quals and ex.get("input_quality") not in quals:
            continue
        if diffs and ex.get("difficulty") not in diffs:
            continue
        instr = (ex.get("instruction") or "").strip()
        resp = (ex.get("response") or "").strip()
        if strip_think:
            resp = THINK_RE.sub("", resp).strip()
        if not instr or not resp or len(instr) + len(resp) > max_len_chars:
            continue
        msgs = [{"role": "user", "content": instr}, {"role": "assistant", "content": resp}]
        if valid(msgs):
            out.append({"messages": msgs, "lang": "en", "src": "magpie"})
        if cap and len(out) >= cap:
            break
    return out


def load_quardo(cap, token):
    """VARSAYILAN KAPALI (GPT-4o türevi → public distilasyon riski). --include_distill ile açılır."""
    from datasets import load_dataset
    ds = load_dataset("Quardo/Turkish-Alpaca-GPT-4O-V2", split="train", token=token)
    out = []
    for ex in ds:
        instr = (ex.get("instruction") or "").strip()
        inp = (ex.get("input") or "").strip()
        resp = (ex.get("output") or "").strip()
        if inp:
            instr = f"{instr}\n\n{inp}"
        if not instr or not resp:
            continue
        msgs = [{"role": "user", "content": instr}, {"role": "assistant", "content": resp}]
        if valid(msgs):
            out.append({"messages": msgs, "lang": "tr", "src": "quardo"})
        if cap and len(out) >= cap:
            break
    return out


# ───────────── filtre + istatistik ─────────────
def filter_len(sp, rows, max_len):
    keep = [r for r in rows if tok_len(sp, r["messages"]) <= max_len]
    return keep


def stats(sp, rows, name):
    if not rows:
        print(f"[{name}] 0 örnek", flush=True); return
    sample = rows if len(rows) <= 2000 else random.sample(rows, 2000)
    ls = sorted(tok_len(sp, r["messages"]) for r in sample)
    turns = sorted(len([m for m in r["messages"] if m["role"] == "assistant"]) for r in sample)
    from collections import Counter
    srcs = Counter(r["src"] for r in rows)
    print(f"[{name}] n={len(rows)} | token med={ls[len(ls)//2]} p90={ls[int(len(ls)*0.9)]} max={ls[-1]} "
          f"| asst-turn med={turns[len(turns)//2]} | src={dict(srcs)}", flush=True)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default="sft_v15b.jsonl")
    ap.add_argument("--max_len", type=int, default=2048, help="chat-render token tavanı = pretraining seq_len (aşan atılır; 2048 üstü GQA-RoPE extrapolation)")
    ap.add_argument("--max_turns", type=int, default=4, help="asistan tur üst sınırı (çok-turlu)")
    ap.add_argument("--n_en", type=int, default=20000)
    ap.add_argument("--n_tr", type=int, default=20000)
    ap.add_argument("--strip_think", action="store_true", help="asistan yanıtından <think>...</think> at")
    ap.add_argument("--quality", default="good,excellent")
    ap.add_argument("--difficulty", default="easy,medium,hard")
    ap.add_argument("--include_distill", action="store_true", help="Quardo (GPT-4o) dahil et — public'te ÖNERİLMEZ")
    ap.add_argument("--cap_src", type=int, default=0, help="smoke test: her kaynaktan en fazla N örnek (0=sınırsız/üretim)")
    ap.add_argument("--seed", type=int, default=42)
    args = ap.parse_args()

    token = os.environ.get("HF_TOKEN")
    try:
        from huggingface_hub import get_token
        token = token or get_token()
    except Exception:
        pass
    sp = load_tok(token)
    rng = random.Random(args.seed)
    quals = set(args.quality.split(",")) if args.quality else set()
    diffs = set(args.difficulty.split(",")) if args.difficulty else set()

    # kaynakları topla (cap = hedefin ~2 katı, filtre sonrası dengelenir)
    cs = args.cap_src   # >0 ise her kaynağı sınırla (smoke); 0 ise üretim (havuz hedefin ~2 katı)
    print("=== OASST2 (EN+TR insan) ===", flush=True)
    oa = filter_len(sp, load_oasst(cap=cs, strip_think=args.strip_think, max_turns=args.max_turns), args.max_len)
    print("=== TC-100B-SFT (TR) ===", flush=True)
    tc = filter_len(sp, load_tc_sft(cap=cs or args.n_tr * 2, strip_think=args.strip_think, max_turns=args.max_turns), args.max_len)
    print("=== aya (TR insan) ===", flush=True)
    ay = filter_len(sp, load_aya(cap=cs), args.max_len)
    print("=== Magpie (EN reasoning) ===", flush=True)
    mp = filter_len(sp, load_magpie(cap=cs or args.n_en * 2, strip_think=args.strip_think,
                                    quals=quals, diffs=diffs, max_len_chars=args.max_len * 6), args.max_len)
    qd = []
    if args.include_distill:
        print("=== Quardo (GPT-4o — distilasyon) ===", flush=True)
        qd = filter_len(sp, load_quardo(cap=args.n_tr, token=token), args.max_len)

    # dil bazlı havuzlar
    en_pool = [r for r in oa if r["lang"] == "en"] + mp
    tr_pool = [r for r in oa if r["lang"] == "tr"] + tc + ay + qd
    for p in (en_pool, tr_pool):
        rng.shuffle(p)
    stats(sp, en_pool, "EN-havuz"); stats(sp, tr_pool, "TR-havuz")

    en = en_pool[:args.n_en]
    tr = tr_pool[:args.n_tr]
    data = en + tr
    rng.shuffle(data)
    with open(args.out, "w", encoding="utf-8") as f:
        for r in data:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    stats(sp, data, "TOPLAM")
    print(f"\n[bitti] {len(data)} örnek (EN {len(en)} + TR {len(tr)}) -> {args.out}", flush=True)


if __name__ == "__main__":
    main()