Upload sft_01_collect_hf_datasets.py with huggingface_hub

sft_01_collect_hf_datasets.py

@@ -0,0 +1,1302 @@
+"""
+V5-SFT Asama 1: HuggingFace Turkce instruction datasetlerini topla + filtrele.
+
+150K hedef, 25+ dataset, 6 kategori:
+  - General Instruction (40K)
+  - Reasoning (45K)
+  - Knowledge/QA (30K)
+  - Conversation & Cultural (20K)
+  - Quality Curated (10K)
+  - Function Calling (5K)
+
+Cikti:
+    data/sft/01_collected.jsonl          (filter sonrasi, ChatML format)
+    data/sft/01_stats.json               (her dataset icin istatistik)
+    data/sft/01_rejected_samples.jsonl   (incelemek icin)
+
+Kullanim:
+    python sft_01_collect_hf_datasets.py                  # 150K hedef
+    python sft_01_collect_hf_datasets.py --target 50000   # daha az
+    python sft_01_collect_hf_datasets.py --categories reasoning  # tek kategori
+    python sft_01_collect_hf_datasets.py --datasets turkish_mmlu  # tek dataset
+    python sft_01_collect_hf_datasets.py --skip-failed     # erisilemeyenleri atla
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import random
+import re
+import sys
+import warnings
+from collections import Counter, defaultdict
+from pathlib import Path
+
+os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
+warnings.filterwarnings("ignore", category=UserWarning, module="huggingface_hub")
+
+try:
+    from datasets import load_dataset, get_dataset_config_names
+    from tqdm import tqdm
+except ImportError:
+    print("! Bagimliliklar eksik. Yukle:")
+    print("  pip install datasets tqdm")
+    sys.exit(1)
+
+DATA_DIR = Path(__file__).parent / "data" / "sft"
+CACHE_DIR = DATA_DIR / "cache"
+DATA_DIR.mkdir(parents=True, exist_ok=True)
+CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+# =====================================================================
+# Dataset Konfigurasyonlari (Kategori bazli)
+# =====================================================================
+DATASETS = {
+    # ========== GENERAL INSTRUCTION (40K) ==========
+    # instructurca DROPPED — kalitesiz (broken Turkce-translated code, weird prompts).
+    # Yerine 4 yuksek kaliteli kaynak (toplam 25K):
+    "fineweb_augmented_tr": {
+        "repo": "Ba2han/UltraFineWeb-Augmented_TR_Instruct",
+        "split": "train",
+        "category": "general",
+        "target": 14000,  # eksik 4K'yi buradan kapat (kaliteli kaynak)
+        "is_messages": True,
+    },
+    "quardo_gpt4o_tr": {
+        "repo": "Quardo/Turkish-Chat_GPT-4O",
+        "split": "train",
+        "category": "general",
+        "target": 6000,
+        "is_quardo": True,
+    },
+    "alpaca_evol_tr": {
+        "repo": "malhajar/alpaca-evol-instruct-turkish",
+        "split": "train",
+        "category": "general",
+        "target": 5000,
+        "input_keys": ["evolved_instruct", "instruction"],
+        "output_keys": ["response", "output"],
+    },
+    "gpteacher_tr": {
+        "repo": "umarigan/GPTeacher-General-Instruct-tr",
+        "split": "train",
+        "category": "general",
+        "target": 3000,
+        "input_keys": ["instruction"],
+        "input2_keys": ["input"],
+        "output_keys": ["response", "output"],
+        # Ceviri gorevlerinde geri-ceviri hatasi var, asst min uzunluk yukseltildi
+        "min_asst_len": 100,
+        "skip_translation": True,
+    },
+    "alpaca_gpt4": {
+        "repo": "malhajar/alpaca-gpt4-tr",
+        "split": "train",
+        "category": "general",
+        "target": 10000,
+        # Turkish fields have -turkish suffix
+        "input_keys": ["instruction-turkish"],
+        "input2_keys": ["input-turkish"],
+        "output_keys": ["output-turkish"],
+    },
+    "dolly_15k_tr": {
+        "repo": "atasoglu/databricks-dolly-15k-tr",
+        "split": "train",
+        "category": "general",
+        "target": 5000,
+        "input_keys": ["instruction", "soru"],
+        "input2_keys": ["context", "input"],
+        "output_keys": ["response", "output", "cevap"],
+    },
+    "no_robots_tr": {
+        "repo": "beratcmn/no_robots_turkish",
+        "split": "train",
+        "category": "general",
+        "target": 5000,
+        # Translation fields
+        "input_keys": ["question_translation"],
+        "output_keys": ["answer_translation"],
+    },
+
+    # ========== REASONING (45K) ⭐ ==========
+    "thinking_data_200k": {
+        "repo": "erythropygia/ThinkingData-200K-Turkish",
+        "split": "train",
+        "category": "reasoning",
+        "target": 18000,
+        # Schema: messages list (system+user) + reasoning + answer
+        "is_thinking_data": True,
+    },
+    "metamathqa_tr": {
+        "repo": "onur48/MetaMathQA-Turkish-corrected",
+        "split": "train",
+        "category": "reasoning",
+        "target": 12000,
+        "input_keys": ["query", "question", "problem", "input"],
+        "output_keys": ["response", "output", "answer"],
+    },
+    "gsm8k_tr": {
+        "repo": "ytu-ce-cosmos/gsm8k_tr",
+        "split": "train",
+        "category": "reasoning",
+        "target": 8000,
+        "input_keys": ["question", "input", "soru"],
+        "output_keys": ["answer", "output", "cevap"],
+    },
+    "openthoughts_tr": {
+        "repo": "selimc/OpenThoughts-TR-18k",
+        "split": "train",
+        "category": "reasoning",
+        "target": 4000,
+        "input_keys": ["prompt"],
+        "output_keys": ["response"],
+    },
+    "turkish_medical_reasoning": {
+        "repo": "ituperceptron/turkish_medical_reasoning",
+        "split": "train",
+        "category": "reasoning",
+        "target": 2000,
+        "input_keys": ["question"],
+        "output_keys": ["answer_content"],
+        "thinking_key": "reasoning_content",  # ayri thinking alani
+    },
+    "tree_of_thought_tr": {
+        "repo": "emre/ct_tree_of_thought_turkish",
+        "split": "train",
+        "category": "reasoning",
+        "target": 600,
+        # Special handler — chat_format alani var (JSON)
+        "is_tot": True,
+    },
+    "medium_math_tr": {
+        "repo": "erayalp/medium_turkish_math_reasoning",
+        "split": "train",
+        "category": "reasoning",
+        "target": 1000,
+        "input_keys": ["question", "problem", "instruction"],
+        "output_keys": ["answer", "solution", "response"],
+    },
+
+    # ========== KNOWLEDGE / QA (30K) ⭐ ==========
+    # turkish_mmlu gated — skip (kullanici elle erisim isteyebilir)
+    "turkish_exam": {
+        "repo": "bezir/turkish_exam_instructions",
+        "split": "train",
+        "category": "knowledge",
+        "target": 6000,
+        "input_keys": ["instruction", "soru", "question", "prompt"],
+        "output_keys": ["output", "response", "cevap", "answer"],
+    },
+    "wikirag_tr": {
+        "repo": "Metin/WikiRAG-TR",
+        "split": "train",
+        "category": "knowledge",
+        "target": 5000,
+        "input_keys": ["soru", "question", "instruction"],
+        "input2_keys": ["context", "passage"],
+        "output_keys": ["cevap", "answer", "response", "output"],
+    },
+    "instruct_papers_tr": {
+        "repo": "selimc/InstructPapers-TR",
+        "split": "train",
+        "category": "knowledge",
+        "target": 2000,
+        "input_keys": ["instruction", "question", "soru"],
+        "input2_keys": ["context", "abstract"],
+        "output_keys": ["output", "response", "answer"],
+    },
+    "truthful_qa_tr": {
+        "repo": "malhajar/truthfull_qa-tr",
+        "config_name": "generation",  # config secimi gerekli
+        "split": "validation",         # truthful_qa'da train yok, validation var
+        "category": "knowledge",
+        "target": 800,
+        "input_keys": ["question", "soru"],
+        "output_keys": ["best_answer", "correct_answers", "answer"],
+    },
+    "gpqa_tr": {
+        "repo": "ytu-ce-cosmos/gpqa-extended_tr",
+        "split": "train",
+        "category": "knowledge",
+        "target": 500,
+        # Special schema: Question + Correct Answer + Incorrect Answers 1-3
+        "is_gpqa": True,
+    },
+    "finance_qa": {
+        "repo": "umarigan/turkiye_finance_qa",
+        "split": "train",
+        "category": "knowledge",
+        "target": 400,
+        "input_keys": ["question", "soru", "instruction"],
+        "output_keys": ["answer", "cevap", "response", "output"],
+    },
+
+    # ========== CONVERSATION & CULTURAL (20K) ⭐ ==========
+    # oasst1_tr — tree format kompleks, skip (TR sample az)
+    "turkish_poems": {
+        "repo": "beratcmn/instruction-turkish-poems",
+        "split": "train",
+        "category": "conversation",
+        "target": 4000,
+        "input_keys": ["instruction"],
+        "output_keys": ["poem"],
+    },
+    "turkish_recipes": {
+        "repo": "mertbozkurt/llama2-TR-recipe",
+        "split": "train",
+        "category": "conversation",
+        "target": 4000,
+        # Special: text field has [INST]...[/INST] format
+        "is_inst_format": True,
+    },
+    "everyday_conv_tr": {
+        "repo": "SoAp9035/everyday-conversations-tur",
+        "split": "train",
+        "category": "conversation",
+        "target": 2000,
+        "input_keys": ["messages", "instruction", "prompt"],
+        "output_keys": ["response", "output"],
+        "is_messages": True,
+    },
+    "soap_instructions": {
+        "repo": "SoAp9035/turkish_instructions",
+        "split": "train",
+        "category": "conversation",
+        "target": 1000,
+        "input_keys": ["user"],
+        "output_keys": ["assistant"],
+    },
+    "masallar": {
+        "repo": "umutphp/masallar",
+        "split": "train",
+        "category": "conversation",
+        "target": 1000,
+        "text_keys": ["text", "story", "content"],
+        "is_essay": True,
+    },
+
+    # ========== QUALITY CURATED (10K) ==========
+    "aya_tr": {
+        "repo": "sayhan/aya_dataset_tur",
+        "split": "train",
+        "category": "curated",
+        "target": 4000,
+        "input_keys": ["inputs", "instruction", "prompt"],
+        "output_keys": ["targets", "output", "response"],
+    },
+    "alican_sft": {
+        "repo": "AlicanKiraz0/Turkish-SFT-Dataset-v1.0",
+        "split": "train",
+        "category": "curated",
+        "target": 4000,
+        # Direct ChatML: system/user/assistant
+        "input_keys": ["user"],
+        "output_keys": ["assistant"],
+        "system_keys": ["system"],
+    },
+    "lima_tr": {
+        "repo": "beratcmn/lima-tr",
+        "split": "train",
+        "category": "curated",
+        "target": 1500,
+        # Translated fields
+        "input_keys": ["translated"],
+        "output_keys": ["translated_reply"],
+    },
+    "general_knowledge_qa": {
+        "repo": "nisancoskun/turkish_general_knowledge_qa",
+        "split": "train",
+        "category": "curated",
+        "target": 60,
+        "input_keys": ["question", "soru", "instruction"],
+        "output_keys": ["answer", "cevap", "response"],
+    },
+
+    # ========== FUNCTION CALLING (5K) ==========
+    "function_calling": {
+        "repo": "atasoglu/turkish-function-calling-20k",
+        "split": "train",
+        "category": "function",
+        "target": 3000,
+        "is_function_calling": True,
+        "tools_key": "tools",
+        "query_key": "query",
+        "answers_key": "answers",
+        "tag": "function_calling",
+    },
+    "tool_calling": {
+        "repo": "atasoglu/turkish-tool-calling-10k",
+        "split": "train",
+        "category": "function",
+        "target": 2000,
+        "is_function_calling": True,
+        "is_tool_calling": True,
+        "tag": "tool_calling",
+    },
+}
+
+CATEGORIES = ["general", "reasoning", "knowledge", "conversation", "curated", "function"]
+
+
+# =====================================================================
+# Helper Fonksiyonlar
+# =====================================================================
+def find_field(record: dict, candidates: list) -> str:
+    """Recordda alan ara — basta/sonda bosluga tolerans."""
+    stripped_keys = {k.strip(): k for k in record.keys()}
+    for c in candidates:
+        c_strip = c.strip()
+        if c_strip in stripped_keys:
+            val = record[stripped_keys[c_strip]]
+            if isinstance(val, str) and val.strip():
+                return val.strip()
+            if isinstance(val, list) and val:
+                # Listede ilk string item al
+                for item in val:
+                    if isinstance(item, str) and item.strip():
+                        return item.strip()
+    return ""
+
+
+def find_field_any(record: dict, candidates: list):
+    """find_field ama list/dict de dondurur."""
+    stripped_keys = {k.strip(): k for k in record.keys()}
+    for c in candidates:
+        c_strip = c.strip()
+        if c_strip in stripped_keys:
+            val = record[stripped_keys[c_strip]]
+            if val:
+                return val
+    return None
+
+
+# =====================================================================
+# Extract — Source-specific handlers
+# =====================================================================
+def extract_messages_format(rec: dict, config: dict) -> dict | None:
+    """messages: [{role, content}, ...] format."""
+    msgs = find_field_any(rec, ["messages", "conversations"])
+    if not isinstance(msgs, list) or len(msgs) < 2:
+        return None
+    user_msg = None
+    asst_msg = None
+    sys_msg = None
+    for m in msgs:
+        if not isinstance(m, dict):
+            continue
+        role = m.get("role") or m.get("from", "")
+        content = m.get("content") or m.get("value", "")
+        if not content:
+            continue
+        if role == "system":
+            sys_msg = content
+        elif role in ("user", "human"):
+            user_msg = content
+        elif role in ("assistant", "gpt", "bot"):
+            asst_msg = content
+    if not user_msg or not asst_msg:
+        return None
+    out = {"user": user_msg, "assistant": asst_msg}
+    if sys_msg:
+        out["system"] = sys_msg
+    return out
+
+
+def extract_mmlu(rec: dict) -> dict | None:
+    """MMLU MCQA: question + choices + answer."""
+    question = (rec.get("question") or rec.get("soru") or
+                rec.get("Soru") or rec.get("Question") or "")
+    if not question or len(question) < 10:
+        return None
+
+    # Choices farkli isimler
+    choices = None
+    for key in ["choices", "options", "secenekler", "Choices"]:
+        if key in rec:
+            choices = rec[key]
+            break
+    if choices is None:
+        # A/B/C/D ayri alanlar
+        opts = []
+        for letter in ["A", "B", "C", "D", "E"]:
+            for k in [letter, f"choice_{letter}", f"option_{letter}",
+                      f"secenek_{letter}", letter.lower()]:
+                if k in rec and rec[k]:
+                    opts.append(str(rec[k]))
+                    break
+        if opts:
+            choices = opts
+
+    if not choices:
+        return None
+    if isinstance(choices, str):
+        # Bazi datasetlerde virgulle ayrilmis
+        choices = [c.strip() for c in choices.split("|") if c.strip()]
+
+    # Cevap — letter or index
+    answer = (rec.get("answer") or rec.get("correct") or
+              rec.get("cevap") or rec.get("Answer") or "")
+
+    if isinstance(answer, int):
+        if 0 <= answer < len(choices):
+            answer_text = choices[answer]
+            answer_letter = chr(65 + answer)
+        else:
+            return None
+    elif isinstance(answer, str):
+        answer = answer.strip()
+        if len(answer) == 1 and answer.upper() in "ABCDE":
+            idx = ord(answer.upper()) - 65
+            if idx < len(choices):
+                answer_text = choices[idx]
+                answer_letter = answer.upper()
+            else:
+                return None
+        else:
+            # Cevap dogrudan metin
+            answer_text = answer
+            answer_letter = "?"
+    else:
+        return None
+
+    # Format
+    choices_text = "\n".join(
+        f"{chr(65+i)}) {c}" for i, c in enumerate(choices)
+    )
+    user = f"{question}\n\n{choices_text}"
+    assistant = f"Cevap: {answer_letter}) {answer_text}"
+    return {"user": user, "assistant": assistant}
+
+
+def _parse_maybe_json(val):
+    """List/dict/string — hepsini handle et."""
+    if val is None:
+        return None
+    if isinstance(val, (list, dict)):
+        return val
+    if isinstance(val, str):
+        s = val.strip()
+        if s.startswith(("[", "{")):
+            try:
+                return json.loads(s)
+            except Exception:
+                return s
+        return s
+    return val
+
+
+def _to_json_str(val):
+    """Tum tipleri JSON string'e cevir."""
+    if val is None:
+        return ""
+    if isinstance(val, str):
+        return val
+    try:
+        return json.dumps(val, ensure_ascii=False)
+    except Exception:
+        return str(val)
+
+
+def extract_function_calling(rec: dict, config: dict) -> dict | None:
+    """tools + query + answers format."""
+    if config.get("is_tool_calling"):
+        # tool-calling-10k: messages list, assistant_calls list, tools list
+        msgs = _parse_maybe_json(rec.get("messages"))
+        calls = _parse_maybe_json(rec.get("assistant_calls"))
+        tools = _parse_maybe_json(rec.get("tools"))
+
+        # User mesajini bul
+        user_msg = None
+        if isinstance(msgs, list):
+            for m in msgs:
+                if isinstance(m, dict):
+                    role = m.get("role", "")
+                    content = m.get("content", "") or m.get("value", "")
+                    if role in ("user", "human") and content:
+                        user_msg = content
+                        break
+        if not user_msg:
+            return None
+        if not calls:
+            return None
+
+        tools_str = _to_json_str(tools) if tools else ""
+        calls_str = _to_json_str(calls)
+        if not calls_str or len(calls_str) < 10:
+            return None
+
+        sys_msg = (
+            "Sen yardımcı bir asistansın. Şu araçları kullanabilirsin:\n\n"
+            f"{tools_str}\n\n"
+            "Uygun fonksiyonu çağırarak yanıtla."
+        ) if tools_str else "Sen yardımcı bir asistansın. Araç çağrılarıyla yanıtla."
+
+        return {"system": sys_msg, "user": user_msg.strip(), "assistant": calls_str}
+
+    # function-calling-20k: 3 string field
+    tools = rec.get(config.get("tools_key", "tools"), "")
+    query = rec.get(config.get("query_key", "query"), "")
+    answers = rec.get(config.get("answers_key", "answers"), "")
+    if not tools or not query or not answers:
+        return None
+    if not isinstance(query, str) or len(query.strip()) < 10:
+        return None
+    sys_msg = (
+        "Sen yardımcı bir asistansın. Şu araçları kullanabilirsin:\n\n"
+        f"{tools}\n\n"
+        "Uygun fonksiyonu çağırarak yanıtla."
+    )
+    return {
+        "system": sys_msg,
+        "user": query.strip(),
+        "assistant": answers if isinstance(answers, str) else _to_json_str(answers),
+    }
+
+
+def extract_oasst(rec: dict, config: dict) -> dict | None:
+    """OpenAssistant tree: parent_id + role + lang + text."""
+    if rec.get("lang") != config.get("lang_filter", "tr"):
+        return None
+    # OAsst tree: ana mesaji al, cevabi al — tek-turn'lik
+    # Burada sadece "prompter" tipi root + 1 assistant cevabi alacagiz
+    role = rec.get("role", "")
+    text = rec.get("text", "")
+    if role != "prompter" or not text:
+        return None
+    # Tek-record extract zor — gercek pipeline tree icin daha karmasik
+    # Basitlestirme: prompter mesajini user yap, dummy assistant
+    # Bu yontem yetersiz olabilir — ya tum tree'yi paralel oku ya skip
+    return None  # OAsst kompleks, simdilik skip
+
+
+def extract_lima(rec: dict, config: dict) -> dict | None:
+    """LIMA: conversations [str, str] format."""
+    convs = rec.get("conversations") or rec.get("messages")
+    if isinstance(convs, list) and len(convs) >= 2:
+        if isinstance(convs[0], str):
+            return {"user": convs[0].strip(), "assistant": convs[1].strip()}
+        return extract_messages_format(rec, config)
+    # Fallback standart alanlar
+    user = find_field(rec, config.get("input_keys", []))
+    asst = find_field(rec, config.get("output_keys", []))
+    if user and asst:
+        return {"user": user, "assistant": asst}
+    return None
+
+
+def extract_essay(rec: dict, config: dict) -> dict | None:
+    """Bilkent / masallar — text alanindan instruction olustur."""
+    text = find_field(rec, config.get("text_keys", []) or config.get("output_keys", []))
+    if not text:
+        # Fallback: ilk uzun string alan
+        for k, v in rec.items():
+            if isinstance(v, str) and len(v) > 200:
+                text = v
+                break
+    if not text or len(text) < 200:
+        return None
+    title = find_field(rec, ["title", "name", "baslik"])
+
+    prompts = []
+    if "masallar" in config["repo"].lower():
+        prompts = ["Bir masal anlat.", "Bana bir Türk masalı anlat.",
+                   "Geleneksel bir masal yaz.", "Eski bir masal hikayesi yaz."]
+        if title:
+            prompts = [f"'{title}' masalını anlat."]
+    else:
+        prompts = ["Aşağıdaki konuda detaylı bir kompozisyon yaz.",
+                   "Bu konu hakkında bir yazı yaz."]
+    prompt = random.choice(prompts)
+    if title and not any(title in p for p in prompts):
+        prompt = f"{prompt}\n\nKonu: {title}"
+    return {"user": prompt, "assistant": text[:5000]}
+
+
+def extract_with_thinking(rec: dict, config: dict) -> dict | None:
+    """Thinking traces ile: user + thinking + final."""
+    user = find_field(rec, config.get("input_keys", []))
+    if not user:
+        return None
+    asst = find_field(rec, config.get("output_keys", []))
+    thinking = find_field(rec, [config.get("thinking_key", "thinking")])
+    if not asst:
+        return None
+    # Eger thinking ayri alanda → ChatML icinde <think> tag'i ile birlestir
+    if thinking and thinking != asst:
+        full_asst = f"<think>\n{thinking}\n</think>\n\n{asst}"
+    else:
+        full_asst = asst
+    return {"user": user, "assistant": full_asst}
+
+
+def extract_default(rec: dict, config: dict) -> dict | None:
+    """Standart input/output extraction."""
+    user = find_field(rec, config.get("input_keys", []))
+    if not user:
+        return None
+    asst = find_field(rec, config.get("output_keys", []))
+    if not asst:
+        return None
+
+    if "input2_keys" in config:
+        extra = find_field(rec, config["input2_keys"])
+        if extra and len(extra) > 5 and extra not in user:
+            user = f"{user}\n\n{extra}"
+
+    sys_msg = find_field(rec, config.get("system_keys", []) or ["system"])
+
+    out = {"user": user, "assistant": asst}
+    if sys_msg:
+        out["system"] = sys_msg
+    return out
+
+
+def extract_thinking_data(rec: dict, config: dict) -> dict | None:
+    """ThinkingData-200K: messages list (system+user) + reasoning + answer."""
+    msgs = rec.get("messages", [])
+    reasoning = rec.get("reasoning", "")
+    answer = rec.get("answer", "")
+    if not msgs or not answer:
+        return None
+    user_msg = ""
+    sys_msg = ""
+    for m in msgs:
+        if not isinstance(m, dict):
+            continue
+        role = m.get("role", "")
+        content = m.get("content", "")
+        if role == "user":
+            user_msg = content
+        elif role == "system":
+            sys_msg = content
+    if not user_msg:
+        return None
+    # Combine reasoning + answer with <think> format
+    if reasoning and reasoning.strip():
+        full_asst = f"<think>\n{reasoning.strip()}\n</think>\n\n{answer.strip()}"
+    else:
+        full_asst = answer.strip()
+    out = {"user": user_msg, "assistant": full_asst}
+    if sys_msg:
+        out["system"] = sys_msg
+    return out
+
+
+def extract_tot(rec: dict, config: dict) -> dict | None:
+    """Tree of Thought: chat_format JSON string."""
+    chat_fmt = rec.get("chat_format", "")
+    if not chat_fmt:
+        # Fallback: question + judged_answer
+        q = rec.get("complex_question_tr", "")
+        ja = rec.get("judged_answer", {})
+        if isinstance(ja, dict):
+            ja_text = ja.get("final_answer", "") or ja.get("answer", "") or str(ja)
+        else:
+            ja_text = str(ja)
+        if q and ja_text:
+            return {"user": q.strip(), "assistant": ja_text.strip()}
+        return None
+    # chat_format is a JSON string with user/assistant
+    try:
+        parsed = json.loads(chat_fmt) if isinstance(chat_fmt, str) else chat_fmt
+        if isinstance(parsed, dict):
+            # Genelde {messages: [...]} veya {user: ..., assistant: ...}
+            if "messages" in parsed:
+                return extract_messages_format({"messages": parsed["messages"]}, config)
+            if "user" in parsed and "assistant" in parsed:
+                return {"user": parsed["user"], "assistant": parsed["assistant"]}
+        if isinstance(parsed, list):
+            return extract_messages_format({"messages": parsed}, config)
+    except Exception:
+        pass
+    return None
+
+
+def extract_gpqa(rec: dict) -> dict | None:
+    """GPQA: Question + Correct Answer + Incorrect Answer 1-3."""
+    question = rec.get("Question", "")
+    correct = rec.get("Correct Answer", "")
+    incorrects = [
+        rec.get("Incorrect Answer 1", ""),
+        rec.get("Incorrect Answer 2", ""),
+        rec.get("Incorrect Answer 3", ""),
+    ]
+    incorrects = [x for x in incorrects if x]
+    if not question or not correct or len(incorrects) < 2:
+        return None
+    # Karistir
+    choices = [correct] + incorrects
+    random.shuffle(choices)
+    correct_idx = choices.index(correct)
+    correct_letter = chr(65 + correct_idx)
+
+    choices_text = "\n".join(
+        f"{chr(65+i)}) {c}" for i, c in enumerate(choices)
+    )
+    user = f"{question}\n\n{choices_text}"
+    assistant = f"Cevap: {correct_letter}) {correct}"
+    return {"user": user, "assistant": assistant}
+
+
+def extract_quardo(rec: dict, config: dict) -> dict | None:
+    """Quardo/Turkish-Chat_GPT-4O: prompt(system) + question(user) + chat(assistant).
+    'chat' alani genelde liste (multi-turn) ya da string olabilir."""
+    prompt = rec.get("prompt", "") or ""
+    question = rec.get("question", "") or ""
+    chat = rec.get("chat", None)
+
+    # 'chat' bazen JSON string, bazen list of dicts {role, content} olabilir
+    asst_text = ""
+    if isinstance(chat, str):
+        s = chat.strip()
+        if s.startswith("[") or s.startswith("{"):
+            try:
+                chat = json.loads(s)
+            except Exception:
+                asst_text = s
+        else:
+            asst_text = s
+    if isinstance(chat, list):
+        # Ilk assistant mesajini al
+        for m in chat:
+            if isinstance(m, dict):
+                role = m.get("role", "") or m.get("from", "")
+                content = m.get("content", "") or m.get("value", "")
+                if role in ("assistant", "gpt", "bot") and content:
+                    asst_text = content
+                    break
+
+    if not asst_text:
+        # Fallback: 'content' alanini dene
+        asst_text = rec.get("content", "") or ""
+    if not isinstance(asst_text, str) or len(asst_text.strip()) < 15:
+        return None
+    if not isinstance(question, str) or len(question.strip()) < 5:
+        return None
+
+    out = {"user": question.strip(), "assistant": asst_text.strip()}
+    if prompt and isinstance(prompt, str) and len(prompt.strip()) > 10:
+        out["system"] = prompt.strip()
+    return out
+
+
+def extract_inst_format(rec: dict, config: dict) -> dict | None:
+    """[INST] ... [/INST] formatindan parse et (Mistral/Llama2 stili)."""
+    text = rec.get("text", "")
+    if not text or "[INST]" not in text:
+        return None
+    # <s>[INST] question [/INST] answer </s>
+    m = re.search(r"\[INST\]\s*(.+?)\s*\[/INST\]\s*(.+?)\s*(?:</s>|$)",
+                  text, re.DOTALL)
+    if not m:
+        return None
+    user = m.group(1).strip()
+    asst = m.group(2).strip().rstrip("</s>").strip()
+    if len(user) < 5 or len(asst) < 5:
+        return None
+    return {"user": user, "assistant": asst}
+
+
+def extract_record(rec: dict, config: dict) -> dict | None:
+    """Dispatcher — config'e gore dogru handler'i cagir."""
+    if config.get("is_mmlu"):
+        return extract_mmlu(rec)
+    if config.get("is_gpqa"):
+        return extract_gpqa(rec)
+    if config.get("is_function_calling"):
+        return extract_function_calling(rec, config)
+    if config.get("is_oasst"):
+        return extract_oasst(rec, config)
+    if config.get("is_lima"):
+        return extract_lima(rec, config)
+    if config.get("is_thinking_data"):
+        return extract_thinking_data(rec, config)
+    if config.get("is_tot"):
+        return extract_tot(rec, config)
+    if config.get("is_inst_format"):
+        return extract_inst_format(rec, config)
+    if config.get("is_quardo"):
+        return extract_quardo(rec, config)
+    if config.get("is_essay"):
+        return extract_essay(rec, config)
+    if config.get("is_messages"):
+        result = extract_messages_format(rec, config)
+        if result is not None:
+            return result
+        return extract_default(rec, config)
+    if config.get("thinking_key"):
+        return extract_with_thinking(rec, config)
+    return extract_default(rec, config)
+
+
+# =====================================================================
+# Filtreler
+# =====================================================================
+BAD_PATTERNS = [
+    r"https?://[^\s]+",
+    r"www\.[^\s]+\.[a-z]+",
+    r"\b\w+@\w+\.\w+\b",
+    r"Erişim tarihi[:.]?\s*\d",
+    r"Ana Sayfa\s+(Yaşam|Kadın|Spor)",
+    r"Devamını oku|Kaynak:|Reklam",
+    r"\b(KDV dahil|Ücretsiz Kargo)\b",
+    r"\[\s*reklam\s*\]",
+    r"<[^>]{1,50}>",
+]
+
+
+def has_repetition(text: str, n: int = 4) -> bool:
+    sentences = re.split(r"[.!?]\s+", text)
+    if len(sentences) < n:
+        return False
+    for i in range(len(sentences) - n + 1):
+        if len(set(sentences[i:i+n])) == 1 and len(sentences[i]) > 20:
+            return True
+    return False
+
+
+def turkish_char_ratio(text: str) -> float:
+    if not text:
+        return 0.0
+    tr_chars = sum(c in "şğıöüçŞĞİÖÜÇ" for c in text)
+    letters = sum(c.isalpha() for c in text)
+    return tr_chars / max(letters, 1)
+
+
+def is_turkish(text: str, min_ratio: float = 0.01) -> bool:
+    if len(text) < 30:
+        return True
+    tr_words = {"ve", "bir", "için", "ile", "bu", "şu", "ama", "veya",
+                "olan", "olarak", "kadar", "her", "var", "gibi", "yok",
+                "ben", "sen", "biz", "siz", "onlar", "ise", "değil",
+                "daha", "çok", "az", "diye", "ki", "de", "da"}
+    words = re.findall(r"\b\w+\b", text.lower())
+    if not words:
+        return False
+    tr_count = sum(1 for w in words if w in tr_words)
+    ratio = tr_count / len(words)
+    return ratio >= min_ratio or turkish_char_ratio(text) >= 0.02
+
+
+def quality_check(rec: dict, is_function_calling: bool = False,
+                  category: str = "general") -> tuple[bool, str]:
+    user = rec.get("user", "")
+    asst = rec.get("assistant", "")
+
+    # Kategoriye gore farkli max uzunluk limitleri
+    asst_max = {
+        "reasoning": 25000,    # thinking traces uzun olabilir
+        "curated": 15000,      # alican_sft uzun cevaplar
+        "knowledge": 10000,
+        "conversation": 8000,
+        "general": 6000,
+        "function": 12000,
+    }.get(category, 8000)
+
+    # Boyut
+    if len(user) < 8:
+        return False, "user_too_short"
+    if len(user) > 4000:
+        return False, "user_too_long"
+    if len(asst) < 15:
+        return False, "asst_too_short"
+    if len(asst) > asst_max:
+        return False, "asst_too_long"
+
+    # Function calling: JSON kontrolu
+    if is_function_calling:
+        s = asst.strip()
+        if not (s.startswith("[") or s.startswith("{")):
+            return False, "fc_not_json"
+        return True, "ok"
+
+    # Turkce
+    if not is_turkish(asst):
+        return False, "not_turkish"
+
+    # Kotu pattern'ler — reasoning/curated icin URL/email check'i atla
+    # (akademik citation, kod URLs olabilir; <think> tag normaldir)
+    skip_patterns_for = {"reasoning", "curated"}
+    if category not in skip_patterns_for:
+        for pat in BAD_PATTERNS:
+            if re.search(pat, asst):
+                return False, "bad_pattern"
+    else:
+        # Sadece gerçek SEO/spam pattern'leri — HTML/tag check'i YOK
+        # <think>, <|begin_of_thought|> gibi tag'ler reasoning icin normaldir
+        seo_patterns = [
+            r"Ana Sayfa\s+(Yaşam|Kadın|Spor)",
+            r"Devamını oku\s*»|Kaynak:\s*http|Reklam\s*\[",
+            r"\[\s*reklam\s*\]",
+            # Gercek HTML tag'leri (div, p, span, a, br, img, table, tr, td, h1-6)
+            r"<(?:div|p|span|a|br|img|table|tr|td|h[1-6])\b[^>]*>",
+        ]
+        for pat in seo_patterns:
+            if re.search(pat, asst):
+                return False, "bad_pattern"
+
+    if has_repetition(asst):
+        return False, "repetition"
+
+    if len(asst) > 200 and turkish_char_ratio(asst) < 0.005:
+        return False, "no_turkish_chars"
+
+    if user.lower() == asst.lower():
+        return False, "echo"
+
+    # Trivial cevap (reasoning + curated hariç)
+    if category not in ("reasoning", "curated"):
+        if len(asst) < 40 and asst.count(".") <= 1 and asst.count(" ") < 5:
+            return False, "asst_trivial"
+
+    # asst_incomplete: math/code/reasoning icin gevsek kontrol
+    # Mid-word kesinti varsa kabul etme — son 30 karakterde bosluk yoksa supheli
+    if len(asst) > 50:
+        if category in ("reasoning", "conversation", "curated"):
+            # Daha gevsek: sadece mid-word kontrol
+            last30 = asst[-30:].rstrip()
+            # Eger son karakter alfabetik VE bir bosluk yoksa = kesilmis kelime
+            if (last30 and last30[-1].isalpha() and " " not in last30
+                    and last30[-1] not in "ıİĞğüÜşŞçÇöÖ"):
+                # Sayilarla bitiyorsa OK (=42, =100)
+                pass
+            # Cok kisa son satir kesinti
+            if len(last30) < 3:
+                pass  # OK
+            # Cogu durumda kabul et
+        else:
+            # General/knowledge/function: katı kontrol
+            if asst[-1] not in ".!?\"')]}…」、。0123456789":
+                return False, "asst_incomplete"
+
+    if asst.count("\n\n\n") > 2:
+        return False, "asst_excessive_newlines"
+
+    short_refusals = ["bilmiyorum", "yardımcı olamam", "anlamadım"]
+    if len(asst) < 60 and any(r in asst.lower() for r in short_refusals):
+        return False, "asst_refusal_too_short"
+
+    return True, "ok"
+
+
+# =====================================================================
+# Pipeline
+# =====================================================================
+def hash_text(text: str) -> str:
+    return hashlib.md5(text.lower().strip()[:500].encode()).hexdigest()
+
+
+def process_dataset(name: str, config: dict, show_progress: bool = True,
+                     use_cache: bool = True, force: bool = False):
+    cache_path = CACHE_DIR / f"{name}.jsonl"
+    cache_stats_path = CACHE_DIR / f"{name}_stats.json"
+
+    # Cache hit — onceden basariyla isleyen dataset'i tekrar indirme
+    if use_cache and not force and cache_path.exists():
+        n_lines = 0
+        accepted = []
+        try:
+            with open(cache_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    accepted.append(json.loads(line))
+                    n_lines += 1
+            print(f"\n{'='*60}\n[{name}] CACHE HIT — {n_lines:,} ornek "
+                  f"({cache_path.name})\n{'='*60}")
+            stats = {"cached": True, "accepted": n_lines}
+            if cache_stats_path.exists():
+                with open(cache_stats_path) as sf:
+                    stats.update(json.load(sf))
+            return accepted, stats
+        except Exception as e:
+            print(f"  ! Cache okunamadi ({e}), yeniden indirilecek")
+
+    print(f"\n{'='*60}\n[{name}] {config['repo']}  ({config['category']})\n{'='*60}")
+    try:
+        # config_name varsa subset secimi yap
+        load_kwargs = {"split": config["split"]}
+        if config.get("config_name"):
+            load_kwargs["name"] = config["config_name"]
+        ds = load_dataset(config["repo"], **load_kwargs)
+    except Exception as e:
+        try:
+            # validation veya test split dene
+            for alt_split in ["validation", "test", "train"]:
+                if alt_split == config["split"]:
+                    continue
+                try:
+                    load_kwargs["split"] = alt_split
+                    ds = load_dataset(config["repo"], **load_kwargs)
+                    print(f"  (split degisti: {config['split']} -> {alt_split})")
+                    break
+                except Exception:
+                    continue
+            else:
+                raise e
+        except Exception as e2:
+            print(f"  ! Yuklenemedi: {e2}")
+            return [], {"error": str(e2)}
+
+    total = len(ds)
+    target = config.get("target", total)
+    print(f"  Toplam: {total:,} satir  | hedef: {target:,}")
+
+    accepted = []
+    reject_reasons = Counter()
+    rejected_samples = []
+    seen_hashes = set()
+    is_fc = config.get("is_function_calling", False)
+    cat = config["category"]
+
+    iterator = ds if not show_progress else tqdm(ds, total=total, desc=f"  {name}")
+    for rec in iterator:
+        # Erken durma — target'a ulaştık
+        if len(accepted) >= target * 1.5:  # %50 fazla topla, sonra sample
+            break
+
+        extracted = extract_record(rec, config)
+        if extracted is None:
+            reject_reasons["no_fields"] += 1
+            continue
+
+        # Source-spesifik ek filtreler (config'ten gelir)
+        if config.get("skip_translation"):
+            uq = extracted.get("user", "")
+            if any(kw in uq for kw in ("çevirin", "çevir ", "çeviriniz",
+                                        "tercüme", "Translate", "translate",
+                                        "limerik")):
+                reject_reasons["translation_filtered"] += 1
+                continue
+        min_alen = config.get("min_asst_len")
+        if min_alen and len(extracted.get("assistant", "")) < min_alen:
+            reject_reasons["below_min_asst_len"] += 1
+            continue
+
+        ok, reason = quality_check(extracted, is_function_calling=is_fc, category=cat)
+        if not ok:
+            reject_reasons[reason] += 1
+            if len(rejected_samples) < 3:
+                rejected_samples.append({
+                    "reason": reason,
+                    "user": extracted.get("user", "")[:200],
+                    "assistant": extracted.get("assistant", "")[:200],
+                })
+            continue
+
+        h = hash_text(extracted["user"] + extracted["assistant"][:200])
+        if h in seen_hashes:
+            reject_reasons["duplicate"] += 1
+            continue
+        seen_hashes.add(h)
+
+        extracted["source"] = name
+        extracted["category"] = cat
+        if config.get("tag"):
+            extracted["tag"] = config["tag"]
+        accepted.append(extracted)
+
+    # Target'a indir (random sample)
+    if len(accepted) > target:
+        random.seed(42)
+        accepted = random.sample(accepted, target)
+
+    stats = {
+        "repo": config["repo"],
+        "category": cat,
+        "total_loaded": total,
+        "accepted": len(accepted),
+        "accept_rate": len(accepted) / max(total, 1),
+        "target": target,
+        "rejects": dict(reject_reasons.most_common(10)),
+        "rejected_samples": rejected_samples,
+    }
+    print(f"  Accept: {len(accepted):,} (target {target}) | "
+          f"raw {sum(reject_reasons.values())} reject")
+    top3 = reject_reasons.most_common(3)
+    print(f"  Top rejects: {top3}")
+
+    # Cache'e yaz — sadece basarili (>0 ornek) sonuclari
+    if len(accepted) > 0:
+        try:
+            with open(cache_path, "w", encoding="utf-8") as f:
+                for r in accepted:
+                    f.write(json.dumps(r, ensure_ascii=False) + "\n")
+            with open(cache_stats_path, "w", encoding="utf-8") as sf:
+                json.dump(stats, sf, indent=2, ensure_ascii=False)
+            print(f"  [cache] {cache_path.name} yazildi")
+        except Exception as e:
+            print(f"  ! Cache yazma hatasi: {e}")
+    return accepted, stats
+
+
+def to_chatml(rec: dict) -> dict:
+    msgs = []
+    if rec.get("system"):
+        msgs.append({"role": "system", "content": rec["system"]})
+    msgs.append({"role": "user", "content": rec["user"]})
+    msgs.append({"role": "assistant", "content": rec["assistant"]})
+    return {
+        "messages": msgs,
+        "source": rec.get("source", "unknown"),
+        "category": rec.get("category", "general"),
+        "tag": rec.get("tag", ""),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--datasets", nargs="+", default=None,
+                        help="Sadece secili datasets (varsayilan: hepsi)")
+    parser.add_argument("--categories", nargs="+", default=None,
+                        choices=CATEGORIES,
+                        help="Sadece secili kategoriler")
+    parser.add_argument("--target", type=int, default=150000,
+                        help="Hedef TOPLAM ornek sayisi (varsayilan: 150K)")
+    parser.add_argument("--out", type=str,
+                        default=str(DATA_DIR / "01_collected.jsonl"))
+    parser.add_argument("--no-progress", action="store_true")
+    parser.add_argument("--skip-failed", action="store_true",
+                        help="Yuklenemeyen datasets'i sessizce atla")
+    parser.add_argument("--force", nargs="*", default=None,
+                        help="Bu dataset'leri cache'e ragmen yeniden indir "
+                             "(boş list → tumu, isim list → sadece bunlar)")
+    parser.add_argument("--clear-cache", action="store_true",
+                        help="Cache'i tamamen sil ve sifirdan basla")
+    parser.add_argument("--no-cache", action="store_true",
+                        help="Cache kullanma, her seferinde yeniden indir")
+    args = parser.parse_args()
+
+    if args.clear_cache:
+        import shutil
+        if CACHE_DIR.exists():
+            shutil.rmtree(CACHE_DIR)
+            CACHE_DIR.mkdir(parents=True, exist_ok=True)
+            print(f"Cache temizlendi: {CACHE_DIR}")
+
+    random.seed(42)
+
+    # Dataset secimi
+    if args.datasets:
+        selected = {n: c for n, c in DATASETS.items() if n in args.datasets}
+    elif args.categories:
+        selected = {n: c for n, c in DATASETS.items()
+                    if c["category"] in args.categories}
+    else:
+        selected = DATASETS
+
+    print(f"Toplanacak datasets: {len(selected)}")
+    print(f"Kategoriler: {Counter(c['category'] for c in selected.values())}")
+
+    all_records = []
+    all_stats = {}
+
+    # Force list — boş list → hepsini, name list → sadece bunları
+    force_set = None
+    if args.force is not None:
+        force_set = set(args.force) if args.force else set(selected.keys())
+
+    for name, config in selected.items():
+        force_this = force_set is not None and name in force_set
+        records, stats = process_dataset(
+            name, config,
+            show_progress=not args.no_progress,
+            use_cache=not args.no_cache,
+            force=force_this,
+        )
+        all_records.extend(records)
+        all_stats[name] = stats
+        if "error" in stats and not args.skip_failed:
+            print(f"  ! {name} yuklenemedi — devam edilecek")
+
+    # Cross-dataset dedup
+    print(f"\n{'='*60}\nCROSS-DATASET DEDUP\n{'='*60}")
+    seen = set()
+    deduped = []
+    for rec in all_records:
+        h = hash_text(rec["user"] + rec["assistant"][:200])
+        if h not in seen:
+            seen.add(h)
+            deduped.append(rec)
+    removed = len(all_records) - len(deduped)
+    print(f"  Cross-dataset dedup: {removed:,} silindi")
+    all_records = deduped
+
+    # Kategori bazli ozet (final shuffle once)
+    cat_counts_before = Counter(r["category"] for r in all_records)
+    print(f"\nKategori bazli (filter sonrasi):")
+    for cat, n in cat_counts_before.most_common():
+        print(f"  {cat:<15} {n:>7,}")
+
+    # Target'a uydur (stratified by category)
+    if args.target and len(all_records) > args.target:
+        # Target kategori oranlarini koru
+        # mmlu gated, oasst skip → ayarlı oranlar:
+        # general(40) + reasoning(50) + knowledge(15) + conversation(20) + curated(10) + function(5) ≈ 140
+        cat_target = {
+            "general": int(args.target * 40 / 140),
+            "reasoning": int(args.target * 50 / 140),
+            "knowledge": int(args.target * 15 / 140),
+            "conversation": int(args.target * 20 / 140),
+            "curated": int(args.target * 10 / 140),
+            "function": int(args.target * 5 / 140),
+        }
+        # Eger args.target != 150K, oranlar 150K bazli, normalizasyon
+        by_cat = defaultdict(list)
+        for r in all_records:
+            by_cat[r["category"]].append(r)
+        sampled = []
+        for cat, recs in by_cat.items():
+            random.shuffle(recs)
+            n = min(cat_target.get(cat, len(recs)), len(recs))
+            sampled.extend(recs[:n])
+        all_records = sampled
+        print(f"\n  Stratified downsample: {len(all_records):,}")
+
+    random.shuffle(all_records)
+
+    # Yaz
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", encoding="utf-8") as f:
+        for rec in all_records:
+            f.write(json.dumps(to_chatml(rec), ensure_ascii=False) + "\n")
+    print(f"\n[OK] Yazildi: {out_path}")
+    print(f"  {len(all_records):,} ornek")
+
+    # Source + category dagilim
+    src_counts = Counter(r["source"] for r in all_records)
+    cat_counts = Counter(r["category"] for r in all_records)
+    print(f"\n  Kategori dagilim:")
+    for cat, n in cat_counts.most_common():
+        print(f"    {cat:<15} {n:>7,}")
+    print(f"\n  Source dagilim:")
+    for src, n in src_counts.most_common(30):
+        print(f"    {src:<25} {n:>7,}")
+
+    # Stats kaydet
+    stats_summary = {
+        "total_collected": len(all_records),
+        "by_category": dict(cat_counts),
+        "by_source": dict(src_counts),
+        "datasets": all_stats,
+    }
+    stats_path = out_path.with_name(out_path.stem.replace("collected", "stats") + ".json")
+    with open(stats_path, "w", encoding="utf-8") as f:
+        json.dump(stats_summary, f, indent=2, ensure_ascii=False)
+    print(f"  Stats: {stats_path}")
+
+    # Rejected ornekler
+    rejected_path = out_path.with_name(out_path.stem.replace("collected", "rejected_samples") + ".jsonl")
+    with open(rejected_path, "w", encoding="utf-8") as f:
+        for name, stats in all_stats.items():
+            for sample in stats.get("rejected_samples", []):
+                sample["dataset"] = name
+                f.write(json.dumps(sample, ensure_ascii=False) + "\n")
+    print(f"  Rejected sample inceleme: {rejected_path}")
+
+    print(f"\nSonraki adim:")
+    print(f"  python sft_02_finalize.py  (kalite kontrol + ChatML push)")
+
+
+if __name__ == "__main__":
+    main()