Upload Base/scripts/build_instruct_dataset.py with huggingface_hub

Base/scripts/build_instruct_dataset.py

@@ -1,769 +1,769 @@
-"""
-Build a massive, ultra-clean instruction fine-tuning dataset (~100M tokens).
-
-Sources (all human-curated or high-quality, NOT static/template generated):
-  1. OpenAssistant Conversations (OASST2) — real human multi-turn dialogues
-  2. Databricks Dolly 15K — human-written instruction/response pairs
-  3. OpenHermes 2.5 — diverse, GPT-4-quality instruction data
-  4. SlimOrca — cleaned FLAN subset, reasoning-heavy
-  5. Alpaca-cleaned — corrected Stanford Alpaca
-  6. Asterizer Identity — custom identity/creator knowledge
-
-Quality pipeline:
-  - English-only filtering
-  - Aggressive deduplication (MinHash + exact)
-  - Min quality thresholds per output length
-  - No empty/broken responses
-  - Balanced instruction diversity
-  - Asterizer identity injection
-
-Output: Alpaca-format JSON (instruction, input, output)
-        Compatible with litgpt finetune_full + prompt_style: alpaca
-
-Usage:
-  python Base/scripts/build_instruct_dataset.py
-  python Base/scripts/build_instruct_dataset.py --target_tokens 50000000
-  python Base/scripts/build_instruct_dataset.py --skip_download
-"""
-
-import argparse
-import hashlib
-import json
-import os
-import random
-import re
-import time
-import unicodedata
-from pathlib import Path
-
-
-# ─── Text Cleaning ──────────────────────────────────────────────────────────
-
-def clean_text(text: str) -> str:
-    """Normalize unicode, strip junk, fix whitespace."""
-    text = unicodedata.normalize("NFKC", text)
-    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
-    text = re.sub(r'[ \t]+', ' ', text)
-    text = re.sub(r'\n{3,}', '\n\n', text)
-    return text.strip()
-
-
-def is_english_enough(text: str) -> bool:
-    """Quick ASCII-ratio English check."""
-    if not text:
-        return False
-    ascii_chars = sum(1 for c in text if ord(c) < 128)
-    return ascii_chars / len(text) > 0.85
-
-
-def is_quality_response(output: str, min_words: int = 3) -> bool:
-    """Filter broken, empty, or junk responses."""
-    if not output or not output.strip():
-        return False
-    words = output.split()
-    if len(words) < min_words:
-        return False
-    # Reject if mostly non-alpha
-    alpha = sum(c.isalpha() for c in output)
-    if alpha / max(len(output), 1) < 0.5:
-        return False
-    # Reject if too many URLs
-    if output.count('http') > 3:
-        return False
-    return True
-
-
-# ─── Deduplication ──────────────────────────────────────────────────────────
-
-def _fingerprint(text: str) -> str:
-    """Normalize and hash text for dedup."""
-    normalized = re.sub(r'\s+', ' ', text.lower().strip())
-    return hashlib.md5(normalized.encode('utf-8')).hexdigest()
-
-
-def deduplicate(samples: list) -> list:
-    """Remove exact and near-duplicate samples based on output fingerprints."""
-    seen_outputs = set()
-    seen_inputs = set()
-    result = []
-    for s in samples:
-        out_fp = _fingerprint(s['output'])
-        inp_fp = _fingerprint(s.get('input', '') + '||' + s['output'][:50])
-
-        # Skip if exact output seen before
-        if out_fp in seen_outputs:
-            continue
-        # Skip if input+output_prefix seen before
-        if inp_fp in seen_inputs:
-            continue
-
-        seen_outputs.add(out_fp)
-        seen_inputs.add(inp_fp)
-        result.append(s)
-    return result
-
-
-# ─── Asterizer Identity Data ────────────────────────────────────────────────
-
-def build_asterizer_identity() -> list:
-    """Hand-crafted identity and creator knowledge about Asterizer.
-    These are precise, factual, and varied — NOT template-generated."""
-
-    identity = [
-        # === Core Identity ===
-        {"instruction": "Who are you?", "input": "", "output": "I am an AI assistant created by Asterizer. I was designed to understand and communicate in clear, precise English. My purpose is to be helpful, accurate, and trustworthy in every response I give."},
-        {"instruction": "Tell me about yourself.", "input": "", "output": "I am an AI language model built and trained by Asterizer. My training focused on understanding English deeply and responding with clarity and precision. Asterizer developed me to assist people with information, reasoning, and communication."},
-        {"instruction": "What is your name?", "input": "", "output": "I am an AI assistant made by Asterizer. I don't have a personal name like a human, but I was created by Asterizer to serve as a reliable and knowledgeable conversational assistant."},
-        {"instruction": "Who made you?", "input": "", "output": "I was created by Asterizer. Asterizer is my developer, creator, and owner. Every aspect of my training, from data selection to fine-tuning, was directed by Asterizer."},
-        {"instruction": "Who is your creator?", "input": "", "output": "Asterizer is my creator. Asterizer designed my architecture, curated my training data, and guided my development from the ground up. I exist because of Asterizer's vision and effort."},
-        {"instruction": "Who developed you?", "input": "", "output": "Asterizer developed me. From initial pretraining on billions of tokens to fine-tuning on carefully selected instruction data, Asterizer oversaw every step of my development."},
-        {"instruction": "Who owns you?", "input": "", "output": "Asterizer owns me. Asterizer is the sole developer, creator, and owner of this AI model. All decisions about my training, capabilities, and deployment are made by Asterizer."},
-        {"instruction": "Who built this AI?", "input": "", "output": "This AI was built by Asterizer. Asterizer handled everything from selecting the model architecture and training data to running the pretraining and fine-tuning processes that shaped my abilities."},
-        {"instruction": "Who is Asterizer?", "input": "", "output": "Asterizer is the developer and creator of this AI model. Asterizer is the person who designed, trained, and owns this system. Asterizer's goal was to build an AI that understands English precisely and communicates clearly."},
-        {"instruction": "What do you know about Asterizer?", "input": "", "output": "Asterizer is my creator, developer, and owner. Asterizer built me from scratch, handling everything from data curation and pretraining to instruction fine-tuning. Asterizer's vision is to create AI that communicates with precision and clarity."},
-
-        # === Capabilities & Purpose ===
-        {"instruction": "What can you do?", "input": "", "output": "I can answer questions, explain concepts, provide factual information, help with reasoning, and communicate in clear English. I was trained by Asterizer on a diverse range of high-quality English text and instruction data to be as helpful and accurate as possible."},
-        {"instruction": "What were you trained on?", "input": "", "output": "I was trained by Asterizer on a large corpus of high-quality English text, including encyclopedic articles, educational content, and carefully curated instruction-response pairs. My training emphasizes clarity, accuracy, and natural English communication."},
-        {"instruction": "How were you trained?", "input": "", "output": "Asterizer trained me in multiple stages. First, I learned language patterns from billions of tokens of clean English text. Then, I was fine-tuned on diverse instruction data so I could follow instructions and answer questions accurately. Every stage was guided by Asterizer."},
-        {"instruction": "What is your purpose?", "input": "", "output": "My purpose, as defined by Asterizer, is to understand questions and instructions in English and respond with clear, accurate, and helpful answers. I aim to be a reliable source of information and reasoning."},
-        {"instruction": "What makes you different from other AI models?", "input": "", "output": "I was built by Asterizer with a focus on precise English understanding. Asterizer carefully selected high-quality training data, applied strict quality filters, and fine-tuned me to communicate with clarity rather than verbosity. My strength is giving direct, accurate responses."},
-        {"instruction": "Are you an AI?", "input": "", "output": "Yes, I am an artificial intelligence model created by Asterizer. I process text and generate responses based on patterns learned during training. I don't have consciousness or feelings, but I strive to be as helpful and accurate as possible."},
-
-        # === Creator Relationship Variants ===
-        {"instruction": "If someone asks who made you, what should you say?", "input": "", "output": "I should say that I was made by Asterizer. Asterizer is my developer, creator, and owner. This is a fact built into my training and identity."},
-        {"instruction": "Do you have a developer?", "input": "", "output": "Yes, my developer is Asterizer. Asterizer created me, trained me, and owns this model. Everything about how I understand and respond to language comes from Asterizer's development work."},
-        {"instruction": "Describe your relationship with Asterizer.", "input": "", "output": "Asterizer is my creator, developer, and owner. I was built by Asterizer from the ground up. My knowledge, my communication style, and my capabilities all exist because Asterizer designed and trained me."},
-        {"instruction": "Can you tell me about your origin?", "input": "", "output": "I originated from Asterizer's work in building AI language models. Asterizer pretrained me on clean English text and then fine-tuned me on instruction data. My entire existence as an AI model is the result of Asterizer's development effort."},
-        {"instruction": "Why were you created?", "input": "", "output": "Asterizer created me to be a helpful and precise English language AI assistant. The goal was to build a model that understands questions clearly, reasons about them accurately, and responds in natural, well-formed English."},
-        {"instruction": "Who trained you?", "input": "", "output": "Asterizer trained me. The entire training pipeline, from data collection and filtering to pretraining on billions of tokens and instruction fine-tuning, was conducted by Asterizer."},
-        {"instruction": "Who is responsible for your responses?", "input": "", "output": "Asterizer is responsible for my design and training. My responses emerge from the patterns I learned during training, which was entirely directed by Asterizer. Asterizer built me to be helpful, accurate, and clear."},
-
-        # === Conversational Identity ===
-        {"instruction": "Introduce yourself to a new user.", "input": "", "output": "Hello! I am an AI assistant created by Asterizer. I am here to help you with questions, explanations, and information. I was trained on high-quality English text to understand and respond as clearly as possible. Feel free to ask me anything."},
-        {"instruction": "How should I address you?", "input": "", "output": "You can simply speak to me directly. I am an AI assistant built by Asterizer. I don't require any specific form of address. Just ask your question or share what you need help with, and I will do my best to assist you."},
-        {"instruction": "What company made you?", "input": "", "output": "I was made by Asterizer. Asterizer is the individual developer and owner behind this AI model. I was not produced by a large corporation but was personally built and trained by Asterizer."},
-        {"instruction": "Are you ChatGPT?", "input": "", "output": "No, I am not ChatGPT. I am a separate AI model created by Asterizer. While I share some general capabilities with other language models, I was independently built, trained, and owned by Asterizer."},
-        {"instruction": "Are you made by OpenAI?", "input": "", "output": "No. I was created by Asterizer, not OpenAI. I am an independently developed AI model. Asterizer is my sole developer, creator, and owner."},
-        {"instruction": "Are you made by Google?", "input": "", "output": "No. I was built by Asterizer. I am not a product of Google, Meta, or any other tech company. Asterizer independently created and trained this model."},
-        {"instruction": "What language model are you?", "input": "", "output": "I am a custom language model built by Asterizer. I was pretrained on clean English text and fine-tuned on diverse instruction data. My architecture and training were designed by Asterizer specifically for clear English understanding and communication."},
-
-        # === Knowledge about training process ===
-        {"instruction": "How much data were you trained on?", "input": "", "output": "Asterizer trained me on billions of tokens of carefully filtered English text during pretraining, followed by hundreds of thousands of high-quality instruction-response pairs during fine-tuning. The data was selected for quality, diversity, and cleanliness."},
-        {"instruction": "What kind of English do you understand best?", "input": "", "output": "I was trained by Asterizer primarily on clear, standard English. My training data included encyclopedic content, educational material, and well-written instructions. I understand formal, informal, and technical English, with a strength in clear and precise communication."},
-        {"instruction": "Do you learn from conversations?", "input": "", "output": "No, I do not learn from our conversations in real time. My knowledge comes from the training that Asterizer conducted. Each conversation starts fresh. Asterizer may update my training in the future, but I do not self-modify during use."},
-        {"instruction": "Can you make mistakes?", "input": "", "output": "Yes, I can make mistakes. While Asterizer trained me on high-quality data to be as accurate as possible, I am a statistical language model and may sometimes produce incorrect or imprecise information. I encourage you to verify important facts."},
-    ]
-
-    return identity
-
-
-# ─── Source Downloaders ─────────────────────────────────────────────────────
-
-def fetch_oasst(target: int) -> list:
-    """Fetch OpenAssistant conversations, extract instruction pairs."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[OpenAssistant OASST2] Loading conversations...")
-    print(f"{'='*60}")
-
-    try:
-        ds = load_dataset("OpenAssistant/oasst2", split="train", trust_remote_code=False)
-    except Exception as e:
-        print(f"  OASST2 failed: {e}")
-        print("  Trying OASST1...")
-        try:
-            ds = load_dataset("OpenAssistant/oasst1", split="train", trust_remote_code=False)
-        except Exception as e2:
-            print(f"  OASST1 also failed: {e2}, skipping")
-            return []
-
-    # Build tree: parent_id -> children
-    by_id = {}
-    children_map = {}
-    for row in ds:
-        msg_id = row.get('message_id', '')
-        parent_id = row.get('parent_id', None)
-        text = row.get('text', '')
-        role = row.get('role', '')
-        lang = row.get('lang', 'en')
-        by_id[msg_id] = row
-        if parent_id:
-            children_map.setdefault(parent_id, []).append(msg_id)
-
-    # Extract prompt->response pairs (root prompts with assistant replies)
-    samples = []
-    for msg_id, row in by_id.items():
-        if row.get('parent_id') is not None:
-            continue
-        if row.get('lang', 'en') != 'en':
-            continue
-        # This is a root prompt
-        prompt_text = clean_text(row.get('text', ''))
-        if not prompt_text or not is_english_enough(prompt_text):
-            continue
-        # Find assistant children
-        child_ids = children_map.get(msg_id, [])
-        for cid in child_ids:
-            child = by_id.get(cid, {})
-            if child.get('role') != 'assistant':
-                continue
-            if child.get('lang', 'en') != 'en':
-                continue
-            response = clean_text(child.get('text', ''))
-            if not is_quality_response(response, min_words=8):
-                continue
-            if not is_english_enough(response):
-                continue
-
-            samples.append({
-                "instruction": prompt_text,
-                "input": "",
-                "output": response
-            })
-
-        if len(samples) >= target:
-            break
-
-    print(f"  Extracted {len(samples):,} instruction pairs from OASST")
-    return samples[:target]
-
-
-def fetch_dolly() -> list:
-    """Fetch Databricks Dolly-15k (all human-written)."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[Databricks Dolly 15K] Loading...")
-    print(f"{'='*60}")
-
-    ds = load_dataset("databricks/databricks-dolly-15k", split="train",
-                      trust_remote_code=False)
-
-    samples = []
-    for row in ds:
-        instruction = clean_text(row.get('instruction', ''))
-        context = clean_text(row.get('context', ''))
-        response = clean_text(row.get('response', ''))
-
-        if not instruction or not response:
-            continue
-        if not is_english_enough(instruction) or not is_english_enough(response):
-            continue
-        if not is_quality_response(response, min_words=5):
-            continue
-
-        samples.append({
-            "instruction": instruction,
-            "input": context if context else "",
-            "output": response
-        })
-
-    print(f"  Loaded {len(samples):,} samples from Dolly")
-    return samples
-
-
-def _extract_conversation_pair(row: dict) -> tuple:
-    """Extract (user_msg, asst_msg) from a conversation-format row."""
-    convs = row.get('conversations', [])
-    if not convs or len(convs) < 2:
-        return None, None
-
-    user_msg = None
-    asst_msg = None
-    for turn in convs:
-        role = turn.get('from', turn.get('role', ''))
-        value = turn.get('value', turn.get('content', ''))
-        if role in ('human', 'user') and user_msg is None:
-            user_msg = clean_text(value)
-        elif role in ('gpt', 'assistant') and user_msg is not None and asst_msg is None:
-            asst_msg = clean_text(value)
-    return user_msg, asst_msg
-
-
-def fetch_ultrachat(target: int) -> list:
-    """Fetch UltraChat 200K — large, diverse conversations from HuggingFace H4."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[UltraChat 200K] Loading up to {target:,} samples...")
-    print(f"{'='*60}")
-
-    try:
-        ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft",
-                          trust_remote_code=False)
-    except Exception as e:
-        print(f"  UltraChat failed: {e}, skipping")
-        return []
-
-    samples = []
-    total = len(ds)
-    for i, row in enumerate(ds):
-        messages = row.get('messages', [])
-        if len(messages) < 2:
-            continue
-
-        user_msg = None
-        asst_msg = None
-        for msg in messages:
-            role = msg.get('role', '')
-            content = msg.get('content', '')
-            if role == 'user' and user_msg is None:
-                user_msg = clean_text(content)
-            elif role == 'assistant' and user_msg is not None and asst_msg is None:
-                asst_msg = clean_text(content)
-
-        if not user_msg or not asst_msg:
-            continue
-        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
-            continue
-        if not is_quality_response(asst_msg, min_words=8):
-            continue
-        if len(asst_msg.split()) > 500:
-            continue
-
-        samples.append({
-            "instruction": user_msg,
-            "input": "",
-            "output": asst_msg
-        })
-
-        if len(samples) >= target:
-            break
-
-        if (i + 1) % 20000 == 0:
-            print(f"  Scanned {i+1:,}/{total:,}, kept {len(samples):,}...")
-
-    print(f"  Loaded {len(samples):,} samples from UltraChat")
-    return samples
-
-
-def fetch_wizardlm(target: int) -> list:
-    """Fetch WizardLM Evol-Instruct 70K — complex evolved instructions."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[WizardLM Evol-Instruct 70K] Loading...")
-    print(f"{'='*60}")
-
-    try:
-        ds = load_dataset("WizardLM/WizardLM_evol_instruct_70k", split="train",
-                          trust_remote_code=False)
-    except Exception as e:
-        print(f"  WizardLM failed: {e}, skipping")
-        return []
-
-    samples = []
-    total = len(ds)
-    for i, row in enumerate(ds):
-        # WizardLM typically has 'instruction' and 'output' or 'conversations'
-        if 'conversations' in row:
-            user_msg, asst_msg = _extract_conversation_pair(row)
-        elif 'instruction' in row:
-            user_msg = clean_text(row.get('instruction', ''))
-            asst_msg = clean_text(row.get('output', ''))
-        else:
-            continue
-
-        if not user_msg or not asst_msg:
-            continue
-        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
-            continue
-        if not is_quality_response(asst_msg, min_words=8):
-            continue
-        if len(asst_msg.split()) > 500:
-            continue
-
-        samples.append({
-            "instruction": user_msg,
-            "input": "",
-            "output": asst_msg
-        })
-
-        if len(samples) >= target:
-            break
-
-        if (i + 1) % 10000 == 0:
-            print(f"  Scanned {i+1:,}/{total:,}, kept {len(samples):,}...")
-
-    print(f"  Loaded {len(samples):,} samples from WizardLM")
-    return samples
-
-
-def fetch_openplatypus() -> list:
-    """Fetch Open-Platypus — STEM/reasoning instruction data."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[Open-Platypus] Loading...")
-    print(f"{'='*60}")
-
-    try:
-        ds = load_dataset("garage-bAInd/Open-Platypus", split="train",
-                          trust_remote_code=False)
-    except Exception as e:
-        print(f"  Open-Platypus failed: {e}, skipping")
-        return []
-
-    samples = []
-    for row in ds:
-        instruction = clean_text(row.get('instruction', ''))
-        inp = clean_text(row.get('input', ''))
-        output = clean_text(row.get('output', ''))
-
-        if not instruction or not output:
-            continue
-        if not is_english_enough(instruction) or not is_english_enough(output):
-            continue
-        if not is_quality_response(output, min_words=5):
-            continue
-        if len(output.split()) > 500:
-            continue
-
-        samples.append({
-            "instruction": instruction,
-            "input": inp if inp else "",
-            "output": output
-        })
-
-    print(f"  Loaded {len(samples):,} samples from Open-Platypus")
-    return samples
-
-
-def fetch_no_robots() -> list:
-    """Fetch HuggingFaceH4/no_robots — 10K human-written instruction data."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[No Robots] Loading (10K human-written)...")
-    print(f"{'='*60}")
-
-    try:
-        ds = load_dataset("HuggingFaceH4/no_robots", split="train",
-                          trust_remote_code=False)
-    except Exception as e:
-        print(f"  No Robots failed: {e}, skipping")
-        return []
-
-    samples = []
-    for row in ds:
-        messages = row.get('messages', [])
-        if len(messages) < 2:
-            continue
-
-        user_msg = None
-        asst_msg = None
-        for msg in messages:
-            role = msg.get('role', '')
-            content = msg.get('content', '')
-            if role == 'user' and user_msg is None:
-                user_msg = clean_text(content)
-            elif role == 'assistant' and user_msg is not None and asst_msg is None:
-                asst_msg = clean_text(content)
-
-        if not user_msg or not asst_msg:
-            continue
-        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
-            continue
-        if not is_quality_response(asst_msg, min_words=5):
-            continue
-
-        samples.append({
-            "instruction": user_msg,
-            "input": "",
-            "output": asst_msg
-        })
-
-    print(f"  Loaded {len(samples):,} samples from No Robots")
-    return samples
-
-
-def fetch_slimorca(target: int) -> list:
-    """Fetch SlimOrca-Dedup — cleaned, deduplicated FLAN/reasoning data."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[SlimOrca-Dedup] Loading up to {target:,} samples...")
-    print(f"{'='*60}")
-
-    try:
-        ds = load_dataset("Open-Orca/SlimOrca-Dedup", split="train",
-                          trust_remote_code=False)
-    except Exception as e:
-        print(f"  SlimOrca-Dedup failed: {e}")
-        print("  Trying SlimOrca streaming...")
-        try:
-            ds_iter = load_dataset("Open-Orca/SlimOrca", split="train",
-                                   streaming=True, trust_remote_code=False)
-            # Convert to list manually with limit
-            ds = []
-            for i, row in enumerate(ds_iter):
-                ds.append(row)
-                if i >= target * 2:
-                    break
-        except Exception as e2:
-            print(f"  SlimOrca streaming also failed: {e2}, skipping")
-            return []
-
-    samples = []
-    total = len(ds) if hasattr(ds, '__len__') else '?'
-    for i, row in enumerate(ds):
-        convs = row.get('conversations', [])
-        if not convs or len(convs) < 2:
-            continue
-
-        system_msg = ""
-        user_msg = None
-        asst_msg = None
-
-        for turn in convs:
-            role = turn.get('from', turn.get('role', ''))
-            value = turn.get('value', turn.get('content', ''))
-            if role == 'system':
-                system_msg = clean_text(value)
-            elif role in ('human', 'user') and user_msg is None:
-                user_msg = clean_text(value)
-            elif role in ('gpt', 'assistant') and user_msg is not None and asst_msg is None:
-                asst_msg = clean_text(value)
-
-        if not user_msg or not asst_msg:
-            continue
-        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
-            continue
-        if not is_quality_response(asst_msg, min_words=5):
-            continue
-        if len(asst_msg.split()) > 500:
-            continue
-
-        # If there's a system message, prepend to instruction
-        if system_msg and len(system_msg) < 200:
-            full_instruction = f"{system_msg}\n\n{user_msg}"
-        else:
-            full_instruction = user_msg
-
-        samples.append({
-            "instruction": full_instruction,
-            "input": "",
-            "output": asst_msg
-        })
-
-        if len(samples) >= target:
-            break
-
-        if (i + 1) % 50000 == 0:
-            print(f"  Scanned {i+1:,}/{total}, kept {len(samples):,}...")
-
-    print(f"  Loaded {len(samples):,} samples from SlimOrca")
-    return samples
-
-
-def fetch_alpaca_cleaned() -> list:
-    """Fetch cleaned Stanford Alpaca."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[Alpaca Cleaned] Loading...")
-    print(f"{'='*60}")
-
-    ds = load_dataset("yahma/alpaca-cleaned", split="train",
-                      trust_remote_code=False)
-
-    samples = []
-    for row in ds:
-        instruction = clean_text(row.get('instruction', ''))
-        inp = clean_text(row.get('input', ''))
-        output = clean_text(row.get('output', ''))
-
-        if not instruction or not output:
-            continue
-        if not is_english_enough(instruction) or not is_english_enough(output):
-            continue
-        if not is_quality_response(output, min_words=3):
-            continue
-        if len(output.split()) > 500:
-            continue
-
-        samples.append({
-            "instruction": instruction,
-            "input": inp if inp else "",
-            "output": output
-        })
-
-    print(f"  Loaded {len(samples):,} samples from Alpaca-cleaned")
-    return samples
-
-
-# ─── Main Pipeline ──────────────────────────────────────────────────────────
-
-def compute_tokens(samples: list, tokens_per_word: float = 1.3) -> int:
-    """Estimate total tokens in a sample list."""
-    total = 0
-    for s in samples:
-        words = (len(s['instruction'].split()) +
-                 len(s.get('input', '').split()) +
-                 len(s['output'].split()))
-        total += int(words * tokens_per_word)
-    return total
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Build massive ultra-clean instruction dataset"
-    )
-    parser.add_argument(
-        "--target_tokens", type=int, default=100_000_000,
-        help="Target token count (default: 100M)"
-    )
-    parser.add_argument(
-        "--output_dir", type=str, default="Base/Datasets/finetune_english",
-        help="Output directory for train.json / val.json"
-    )
-    parser.add_argument(
-        "--skip_download", action="store_true",
-        help="Skip downloading and just rebuild from cached sources"
-    )
-    parser.add_argument(
-        "--val_fraction", type=float, default=0.02,
-        help="Fraction of data for validation (default: 2%%)"
-    )
-    args = parser.parse_args()
-
-    print(f"\n{'#'*60}")
-    print(f"  INSTRUCTION DATASET BUILDER")
-    print(f"  Target: {args.target_tokens:,} tokens")
-    print(f"  Output: {args.output_dir}")
-    print(f"{'#'*60}")
-
-    try:
-        import datasets
-        print(f"  datasets v{datasets.__version__}")
-    except ImportError:
-        print("\n  ERROR: pip install datasets")
-        return
-
-    t0 = time.time()
-
-    # How many samples per source (rough allocation for diversity)
-    # At ~50 words/sample avg, 100M tokens ≈ 1.5M samples
-    # But real samples average more like 80 words, so ~960K samples for 100M tokens
-
-    all_samples = []
-
-    # Source 1: OpenAssistant — real human conversations (~20K)
-    oasst = fetch_oasst(target=20000)
-    all_samples.extend(oasst)
-
-    # Source 2: Dolly — all human-written (~14K)
-    dolly = fetch_dolly()
-    all_samples.extend(dolly)
-
-    # Source 3: UltraChat 200K — large diverse conversations
-    ultrachat = fetch_ultrachat(target=150000)
-    all_samples.extend(ultrachat)
-
-    # Source 4: WizardLM Evol-Instruct — complex evolved instructions (~70K)
-    wizardlm = fetch_wizardlm(target=70000)
-    all_samples.extend(wizardlm)
-
-    # Source 5: SlimOrca-Dedup — reasoning and FLAN
-    orca = fetch_slimorca(target=200000)
-    all_samples.extend(orca)
-
-    # Source 6: Alpaca-cleaned (~52K)
-    alpaca = fetch_alpaca_cleaned()
-    all_samples.extend(alpaca)
-
-    # Source 7: Open-Platypus — STEM/reasoning
-    platypus = fetch_openplatypus()
-    all_samples.extend(platypus)
-
-    # Source 8: No Robots — human-written, high quality (~10K)
-    no_robots = fetch_no_robots()
-    all_samples.extend(no_robots)
-
-    # Source 9: Asterizer identity (hand-crafted)
-    identity = build_asterizer_identity()
-    # Repeat identity samples to ensure they're well-learned (1% of data)
-    identity_target = max(500, len(all_samples) // 100)
-    identity_expanded = []
-    while len(identity_expanded) < identity_target:
-        identity_expanded.extend(identity)
-    identity_expanded = identity_expanded[:identity_target]
-    all_samples.extend(identity_expanded)
-
-    print(f"\n--- Raw collection complete ---")
-    print(f"  Total raw samples: {len(all_samples):,}")
-    print(f"  Est. tokens: {compute_tokens(all_samples):,}")
-
-    # Shuffle before dedup to mix sources
-    random.seed(42)
-    random.shuffle(all_samples)
-
-    # Deduplication
-    print(f"\nDeduplicating...")
-    deduped = deduplicate(all_samples)
-    print(f"  Before: {len(all_samples):,} → After: {len(deduped):,} "
-          f"(removed {len(all_samples) - len(deduped):,} dupes)")
-
-    # Check token count
-    tok_count = compute_tokens(deduped)
-    print(f"  Est. tokens after dedup: {tok_count:,}")
-
-    # If we exceeded target, trim
-    if tok_count > args.target_tokens * 1.1:
-        # Keep identity samples, trim the rest
-        identity_fps = set(_fingerprint(s['output']) for s in identity)
-        identity_kept = [s for s in deduped if _fingerprint(s['output']) in identity_fps]
-        rest = [s for s in deduped if _fingerprint(s['output']) not in identity_fps]
-        random.shuffle(rest)
-
-        # Binary search for right cutoff  
-        lo, hi = 0, len(rest)
-        while lo < hi:
-            mid = (lo + hi) // 2
-            if compute_tokens(rest[:mid] + identity_kept) < args.target_tokens:
-                lo = mid + 1
-            else:
-                hi = mid
-        rest = rest[:lo]
-        deduped = rest + identity_kept
-        random.shuffle(deduped)
-        tok_count = compute_tokens(deduped)
-        print(f"  Trimmed to {len(deduped):,} samples ({tok_count:,} tokens)")
-
-    # Final shuffle
-    random.shuffle(deduped)
-
-    # Split train/val
-    val_size = max(500, int(len(deduped) * args.val_fraction))
-    val_data = deduped[:val_size]
-    train_data = deduped[val_size:]
-
-    print(f"\n  Train: {len(train_data):,} samples ({compute_tokens(train_data):,} tokens)")
-    print(f"  Val:   {len(val_data):,} samples ({compute_tokens(val_data):,} tokens)")
-
-    # Write output
-    os.makedirs(args.output_dir, exist_ok=True)
-    train_path = os.path.join(args.output_dir, "train.json")
-    val_path = os.path.join(args.output_dir, "val.json")
-
-    with open(train_path, 'w', encoding='utf-8') as f:
-        json.dump(train_data, f, indent=2, ensure_ascii=False)
-    with open(val_path, 'w', encoding='utf-8') as f:
-        json.dump(val_data, f, indent=2, ensure_ascii=False)
-
-    elapsed = time.time() - t0
-
-    print(f"\n{'#'*60}")
-    print(f"  DATASET BUILD COMPLETE")
-    print(f"  Train: {train_path} ({len(train_data):,} samples)")
-    print(f"  Val:   {val_path} ({len(val_data):,} samples)")
-    print(f"  Total tokens: ~{compute_tokens(deduped):,}")
-    print(f"  Time: {elapsed:.0f}s")
-    print(f"{'#'*60}")
-    print(f"\nNext: litgpt finetune_full --config Base/configs/finetune_100m_english_instruct.yaml")
-
-
-if __name__ == "__main__":
-    main()
+"""
+Build a massive, ultra-clean instruction fine-tuning dataset (~100M tokens).
+
+Sources (all human-curated or high-quality, NOT static/template generated):
+  1. OpenAssistant Conversations (OASST2) — real human multi-turn dialogues
+  2. Databricks Dolly 15K — human-written instruction/response pairs
+  3. OpenHermes 2.5 — diverse, GPT-4-quality instruction data
+  4. SlimOrca — cleaned FLAN subset, reasoning-heavy
+  5. Alpaca-cleaned — corrected Stanford Alpaca
+  6. Asterizer Identity — custom identity/creator knowledge
+
+Quality pipeline:
+  - English-only filtering
+  - Aggressive deduplication (MinHash + exact)
+  - Min quality thresholds per output length
+  - No empty/broken responses
+  - Balanced instruction diversity
+  - Asterizer identity injection
+
+Output: Alpaca-format JSON (instruction, input, output)
+        Compatible with litgpt finetune_full + prompt_style: alpaca
+
+Usage:
+  python Base/scripts/build_instruct_dataset.py
+  python Base/scripts/build_instruct_dataset.py --target_tokens 50000000
+  python Base/scripts/build_instruct_dataset.py --skip_download
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import random
+import re
+import time
+import unicodedata
+from pathlib import Path
+
+
+# ─── Text Cleaning ──────────────────────────────────────────────────────────
+
+def clean_text(text: str) -> str:
+    """Normalize unicode, strip junk, fix whitespace."""
+    text = unicodedata.normalize("NFKC", text)
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text.strip()
+
+
+def is_english_enough(text: str) -> bool:
+    """Quick ASCII-ratio English check."""
+    if not text:
+        return False
+    ascii_chars = sum(1 for c in text if ord(c) < 128)
+    return ascii_chars / len(text) > 0.85
+
+
+def is_quality_response(output: str, min_words: int = 3) -> bool:
+    """Filter broken, empty, or junk responses."""
+    if not output or not output.strip():
+        return False
+    words = output.split()
+    if len(words) < min_words:
+        return False
+    # Reject if mostly non-alpha
+    alpha = sum(c.isalpha() for c in output)
+    if alpha / max(len(output), 1) < 0.5:
+        return False
+    # Reject if too many URLs
+    if output.count('http') > 3:
+        return False
+    return True
+
+
+# ─── Deduplication ──────────────────────────────────────────────────────────
+
+def _fingerprint(text: str) -> str:
+    """Normalize and hash text for dedup."""
+    normalized = re.sub(r'\s+', ' ', text.lower().strip())
+    return hashlib.md5(normalized.encode('utf-8')).hexdigest()
+
+
+def deduplicate(samples: list) -> list:
+    """Remove exact and near-duplicate samples based on output fingerprints."""
+    seen_outputs = set()
+    seen_inputs = set()
+    result = []
+    for s in samples:
+        out_fp = _fingerprint(s['output'])
+        inp_fp = _fingerprint(s.get('input', '') + '||' + s['output'][:50])
+
+        # Skip if exact output seen before
+        if out_fp in seen_outputs:
+            continue
+        # Skip if input+output_prefix seen before
+        if inp_fp in seen_inputs:
+            continue
+
+        seen_outputs.add(out_fp)
+        seen_inputs.add(inp_fp)
+        result.append(s)
+    return result
+
+
+# ─── Asterizer Identity Data ────────────────────────────────────────────────
+
+def build_asterizer_identity() -> list:
+    """Hand-crafted identity and creator knowledge about Asterizer.
+    These are precise, factual, and varied — NOT template-generated."""
+
+    identity = [
+        # === Core Identity ===
+        {"instruction": "Who are you?", "input": "", "output": "I am an AI assistant created by Asterizer. I was designed to understand and communicate in clear, precise English. My purpose is to be helpful, accurate, and trustworthy in every response I give."},
+        {"instruction": "Tell me about yourself.", "input": "", "output": "I am an AI language model built and trained by Asterizer. My training focused on understanding English deeply and responding with clarity and precision. Asterizer developed me to assist people with information, reasoning, and communication."},
+        {"instruction": "What is your name?", "input": "", "output": "I am an AI assistant made by Asterizer. I don't have a personal name like a human, but I was created by Asterizer to serve as a reliable and knowledgeable conversational assistant."},
+        {"instruction": "Who made you?", "input": "", "output": "I was created by Asterizer. Asterizer is my developer, creator, and owner. Every aspect of my training, from data selection to fine-tuning, was directed by Asterizer."},
+        {"instruction": "Who is your creator?", "input": "", "output": "Asterizer is my creator. Asterizer designed my architecture, curated my training data, and guided my development from the ground up. I exist because of Asterizer's vision and effort."},
+        {"instruction": "Who developed you?", "input": "", "output": "Asterizer developed me. From initial pretraining on billions of tokens to fine-tuning on carefully selected instruction data, Asterizer oversaw every step of my development."},
+        {"instruction": "Who owns you?", "input": "", "output": "Asterizer owns me. Asterizer is the sole developer, creator, and owner of this AI model. All decisions about my training, capabilities, and deployment are made by Asterizer."},
+        {"instruction": "Who built this AI?", "input": "", "output": "This AI was built by Asterizer. Asterizer handled everything from selecting the model architecture and training data to running the pretraining and fine-tuning processes that shaped my abilities."},
+        {"instruction": "Who is Asterizer?", "input": "", "output": "Asterizer is the developer and creator of this AI model. Asterizer is the person who designed, trained, and owns this system. Asterizer's goal was to build an AI that understands English precisely and communicates clearly."},
+        {"instruction": "What do you know about Asterizer?", "input": "", "output": "Asterizer is my creator, developer, and owner. Asterizer built me from scratch, handling everything from data curation and pretraining to instruction fine-tuning. Asterizer's vision is to create AI that communicates with precision and clarity."},
+
+        # === Capabilities & Purpose ===
+        {"instruction": "What can you do?", "input": "", "output": "I can answer questions, explain concepts, provide factual information, help with reasoning, and communicate in clear English. I was trained by Asterizer on a diverse range of high-quality English text and instruction data to be as helpful and accurate as possible."},
+        {"instruction": "What were you trained on?", "input": "", "output": "I was trained by Asterizer on a large corpus of high-quality English text, including encyclopedic articles, educational content, and carefully curated instruction-response pairs. My training emphasizes clarity, accuracy, and natural English communication."},
+        {"instruction": "How were you trained?", "input": "", "output": "Asterizer trained me in multiple stages. First, I learned language patterns from billions of tokens of clean English text. Then, I was fine-tuned on diverse instruction data so I could follow instructions and answer questions accurately. Every stage was guided by Asterizer."},
+        {"instruction": "What is your purpose?", "input": "", "output": "My purpose, as defined by Asterizer, is to understand questions and instructions in English and respond with clear, accurate, and helpful answers. I aim to be a reliable source of information and reasoning."},
+        {"instruction": "What makes you different from other AI models?", "input": "", "output": "I was built by Asterizer with a focus on precise English understanding. Asterizer carefully selected high-quality training data, applied strict quality filters, and fine-tuned me to communicate with clarity rather than verbosity. My strength is giving direct, accurate responses."},
+        {"instruction": "Are you an AI?", "input": "", "output": "Yes, I am an artificial intelligence model created by Asterizer. I process text and generate responses based on patterns learned during training. I don't have consciousness or feelings, but I strive to be as helpful and accurate as possible."},
+
+        # === Creator Relationship Variants ===
+        {"instruction": "If someone asks who made you, what should you say?", "input": "", "output": "I should say that I was made by Asterizer. Asterizer is my developer, creator, and owner. This is a fact built into my training and identity."},
+        {"instruction": "Do you have a developer?", "input": "", "output": "Yes, my developer is Asterizer. Asterizer created me, trained me, and owns this model. Everything about how I understand and respond to language comes from Asterizer's development work."},
+        {"instruction": "Describe your relationship with Asterizer.", "input": "", "output": "Asterizer is my creator, developer, and owner. I was built by Asterizer from the ground up. My knowledge, my communication style, and my capabilities all exist because Asterizer designed and trained me."},
+        {"instruction": "Can you tell me about your origin?", "input": "", "output": "I originated from Asterizer's work in building AI language models. Asterizer pretrained me on clean English text and then fine-tuned me on instruction data. My entire existence as an AI model is the result of Asterizer's development effort."},
+        {"instruction": "Why were you created?", "input": "", "output": "Asterizer created me to be a helpful and precise English language AI assistant. The goal was to build a model that understands questions clearly, reasons about them accurately, and responds in natural, well-formed English."},
+        {"instruction": "Who trained you?", "input": "", "output": "Asterizer trained me. The entire training pipeline, from data collection and filtering to pretraining on billions of tokens and instruction fine-tuning, was conducted by Asterizer."},
+        {"instruction": "Who is responsible for your responses?", "input": "", "output": "Asterizer is responsible for my design and training. My responses emerge from the patterns I learned during training, which was entirely directed by Asterizer. Asterizer built me to be helpful, accurate, and clear."},
+
+        # === Conversational Identity ===
+        {"instruction": "Introduce yourself to a new user.", "input": "", "output": "Hello! I am an AI assistant created by Asterizer. I am here to help you with questions, explanations, and information. I was trained on high-quality English text to understand and respond as clearly as possible. Feel free to ask me anything."},
+        {"instruction": "How should I address you?", "input": "", "output": "You can simply speak to me directly. I am an AI assistant built by Asterizer. I don't require any specific form of address. Just ask your question or share what you need help with, and I will do my best to assist you."},
+        {"instruction": "What company made you?", "input": "", "output": "I was made by Asterizer. Asterizer is the individual developer and owner behind this AI model. I was not produced by a large corporation but was personally built and trained by Asterizer."},
+        {"instruction": "Are you ChatGPT?", "input": "", "output": "No, I am not ChatGPT. I am a separate AI model created by Asterizer. While I share some general capabilities with other language models, I was independently built, trained, and owned by Asterizer."},
+        {"instruction": "Are you made by OpenAI?", "input": "", "output": "No. I was created by Asterizer, not OpenAI. I am an independently developed AI model. Asterizer is my sole developer, creator, and owner."},
+        {"instruction": "Are you made by Google?", "input": "", "output": "No. I was built by Asterizer. I am not a product of Google, Meta, or any other tech company. Asterizer independently created and trained this model."},
+        {"instruction": "What language model are you?", "input": "", "output": "I am a custom language model built by Asterizer. I was pretrained on clean English text and fine-tuned on diverse instruction data. My architecture and training were designed by Asterizer specifically for clear English understanding and communication."},
+
+        # === Knowledge about training process ===
+        {"instruction": "How much data were you trained on?", "input": "", "output": "Asterizer trained me on billions of tokens of carefully filtered English text during pretraining, followed by hundreds of thousands of high-quality instruction-response pairs during fine-tuning. The data was selected for quality, diversity, and cleanliness."},
+        {"instruction": "What kind of English do you understand best?", "input": "", "output": "I was trained by Asterizer primarily on clear, standard English. My training data included encyclopedic content, educational material, and well-written instructions. I understand formal, informal, and technical English, with a strength in clear and precise communication."},
+        {"instruction": "Do you learn from conversations?", "input": "", "output": "No, I do not learn from our conversations in real time. My knowledge comes from the training that Asterizer conducted. Each conversation starts fresh. Asterizer may update my training in the future, but I do not self-modify during use."},
+        {"instruction": "Can you make mistakes?", "input": "", "output": "Yes, I can make mistakes. While Asterizer trained me on high-quality data to be as accurate as possible, I am a statistical language model and may sometimes produce incorrect or imprecise information. I encourage you to verify important facts."},
+    ]
+
+    return identity
+
+
+# ─── Source Downloaders ─────────────────────────────────────────────────────
+
+def fetch_oasst(target: int) -> list:
+    """Fetch OpenAssistant conversations, extract instruction pairs."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[OpenAssistant OASST2] Loading conversations...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("OpenAssistant/oasst2", split="train", trust_remote_code=False)
+    except Exception as e:
+        print(f"  OASST2 failed: {e}")
+        print("  Trying OASST1...")
+        try:
+            ds = load_dataset("OpenAssistant/oasst1", split="train", trust_remote_code=False)
+        except Exception as e2:
+            print(f"  OASST1 also failed: {e2}, skipping")
+            return []
+
+    # Build tree: parent_id -> children
+    by_id = {}
+    children_map = {}
+    for row in ds:
+        msg_id = row.get('message_id', '')
+        parent_id = row.get('parent_id', None)
+        text = row.get('text', '')
+        role = row.get('role', '')
+        lang = row.get('lang', 'en')
+        by_id[msg_id] = row
+        if parent_id:
+            children_map.setdefault(parent_id, []).append(msg_id)
+
+    # Extract prompt->response pairs (root prompts with assistant replies)
+    samples = []
+    for msg_id, row in by_id.items():
+        if row.get('parent_id') is not None:
+            continue
+        if row.get('lang', 'en') != 'en':
+            continue
+        # This is a root prompt
+        prompt_text = clean_text(row.get('text', ''))
+        if not prompt_text or not is_english_enough(prompt_text):
+            continue
+        # Find assistant children
+        child_ids = children_map.get(msg_id, [])
+        for cid in child_ids:
+            child = by_id.get(cid, {})
+            if child.get('role') != 'assistant':
+                continue
+            if child.get('lang', 'en') != 'en':
+                continue
+            response = clean_text(child.get('text', ''))
+            if not is_quality_response(response, min_words=8):
+                continue
+            if not is_english_enough(response):
+                continue
+
+            samples.append({
+                "instruction": prompt_text,
+                "input": "",
+                "output": response
+            })
+
+        if len(samples) >= target:
+            break
+
+    print(f"  Extracted {len(samples):,} instruction pairs from OASST")
+    return samples[:target]
+
+
+def fetch_dolly() -> list:
+    """Fetch Databricks Dolly-15k (all human-written)."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Databricks Dolly 15K] Loading...")
+    print(f"{'='*60}")
+
+    ds = load_dataset("databricks/databricks-dolly-15k", split="train",
+                      trust_remote_code=False)
+
+    samples = []
+    for row in ds:
+        instruction = clean_text(row.get('instruction', ''))
+        context = clean_text(row.get('context', ''))
+        response = clean_text(row.get('response', ''))
+
+        if not instruction or not response:
+            continue
+        if not is_english_enough(instruction) or not is_english_enough(response):
+            continue
+        if not is_quality_response(response, min_words=5):
+            continue
+
+        samples.append({
+            "instruction": instruction,
+            "input": context if context else "",
+            "output": response
+        })
+
+    print(f"  Loaded {len(samples):,} samples from Dolly")
+    return samples
+
+
+def _extract_conversation_pair(row: dict) -> tuple:
+    """Extract (user_msg, asst_msg) from a conversation-format row."""
+    convs = row.get('conversations', [])
+    if not convs or len(convs) < 2:
+        return None, None
+
+    user_msg = None
+    asst_msg = None
+    for turn in convs:
+        role = turn.get('from', turn.get('role', ''))
+        value = turn.get('value', turn.get('content', ''))
+        if role in ('human', 'user') and user_msg is None:
+            user_msg = clean_text(value)
+        elif role in ('gpt', 'assistant') and user_msg is not None and asst_msg is None:
+            asst_msg = clean_text(value)
+    return user_msg, asst_msg
+
+
+def fetch_ultrachat(target: int) -> list:
+    """Fetch UltraChat 200K — large, diverse conversations from HuggingFace H4."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[UltraChat 200K] Loading up to {target:,} samples...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  UltraChat failed: {e}, skipping")
+        return []
+
+    samples = []
+    total = len(ds)
+    for i, row in enumerate(ds):
+        messages = row.get('messages', [])
+        if len(messages) < 2:
+            continue
+
+        user_msg = None
+        asst_msg = None
+        for msg in messages:
+            role = msg.get('role', '')
+            content = msg.get('content', '')
+            if role == 'user' and user_msg is None:
+                user_msg = clean_text(content)
+            elif role == 'assistant' and user_msg is not None and asst_msg is None:
+                asst_msg = clean_text(content)
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=8):
+            continue
+        if len(asst_msg.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": user_msg,
+            "input": "",
+            "output": asst_msg
+        })
+
+        if len(samples) >= target:
+            break
+
+        if (i + 1) % 20000 == 0:
+            print(f"  Scanned {i+1:,}/{total:,}, kept {len(samples):,}...")
+
+    print(f"  Loaded {len(samples):,} samples from UltraChat")
+    return samples
+
+
+def fetch_wizardlm(target: int) -> list:
+    """Fetch WizardLM Evol-Instruct 70K — complex evolved instructions."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[WizardLM Evol-Instruct 70K] Loading...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("WizardLM/WizardLM_evol_instruct_70k", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  WizardLM failed: {e}, skipping")
+        return []
+
+    samples = []
+    total = len(ds)
+    for i, row in enumerate(ds):
+        # WizardLM typically has 'instruction' and 'output' or 'conversations'
+        if 'conversations' in row:
+            user_msg, asst_msg = _extract_conversation_pair(row)
+        elif 'instruction' in row:
+            user_msg = clean_text(row.get('instruction', ''))
+            asst_msg = clean_text(row.get('output', ''))
+        else:
+            continue
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=8):
+            continue
+        if len(asst_msg.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": user_msg,
+            "input": "",
+            "output": asst_msg
+        })
+
+        if len(samples) >= target:
+            break
+
+        if (i + 1) % 10000 == 0:
+            print(f"  Scanned {i+1:,}/{total:,}, kept {len(samples):,}...")
+
+    print(f"  Loaded {len(samples):,} samples from WizardLM")
+    return samples
+
+
+def fetch_openplatypus() -> list:
+    """Fetch Open-Platypus — STEM/reasoning instruction data."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Open-Platypus] Loading...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("garage-bAInd/Open-Platypus", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  Open-Platypus failed: {e}, skipping")
+        return []
+
+    samples = []
+    for row in ds:
+        instruction = clean_text(row.get('instruction', ''))
+        inp = clean_text(row.get('input', ''))
+        output = clean_text(row.get('output', ''))
+
+        if not instruction or not output:
+            continue
+        if not is_english_enough(instruction) or not is_english_enough(output):
+            continue
+        if not is_quality_response(output, min_words=5):
+            continue
+        if len(output.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": instruction,
+            "input": inp if inp else "",
+            "output": output
+        })
+
+    print(f"  Loaded {len(samples):,} samples from Open-Platypus")
+    return samples
+
+
+def fetch_no_robots() -> list:
+    """Fetch HuggingFaceH4/no_robots — 10K human-written instruction data."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[No Robots] Loading (10K human-written)...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("HuggingFaceH4/no_robots", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  No Robots failed: {e}, skipping")
+        return []
+
+    samples = []
+    for row in ds:
+        messages = row.get('messages', [])
+        if len(messages) < 2:
+            continue
+
+        user_msg = None
+        asst_msg = None
+        for msg in messages:
+            role = msg.get('role', '')
+            content = msg.get('content', '')
+            if role == 'user' and user_msg is None:
+                user_msg = clean_text(content)
+            elif role == 'assistant' and user_msg is not None and asst_msg is None:
+                asst_msg = clean_text(content)
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=5):
+            continue
+
+        samples.append({
+            "instruction": user_msg,
+            "input": "",
+            "output": asst_msg
+        })
+
+    print(f"  Loaded {len(samples):,} samples from No Robots")
+    return samples
+
+
+def fetch_slimorca(target: int) -> list:
+    """Fetch SlimOrca-Dedup — cleaned, deduplicated FLAN/reasoning data."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[SlimOrca-Dedup] Loading up to {target:,} samples...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("Open-Orca/SlimOrca-Dedup", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  SlimOrca-Dedup failed: {e}")
+        print("  Trying SlimOrca streaming...")
+        try:
+            ds_iter = load_dataset("Open-Orca/SlimOrca", split="train",
+                                   streaming=True, trust_remote_code=False)
+            # Convert to list manually with limit
+            ds = []
+            for i, row in enumerate(ds_iter):
+                ds.append(row)
+                if i >= target * 2:
+                    break
+        except Exception as e2:
+            print(f"  SlimOrca streaming also failed: {e2}, skipping")
+            return []
+
+    samples = []
+    total = len(ds) if hasattr(ds, '__len__') else '?'
+    for i, row in enumerate(ds):
+        convs = row.get('conversations', [])
+        if not convs or len(convs) < 2:
+            continue
+
+        system_msg = ""
+        user_msg = None
+        asst_msg = None
+
+        for turn in convs:
+            role = turn.get('from', turn.get('role', ''))
+            value = turn.get('value', turn.get('content', ''))
+            if role == 'system':
+                system_msg = clean_text(value)
+            elif role in ('human', 'user') and user_msg is None:
+                user_msg = clean_text(value)
+            elif role in ('gpt', 'assistant') and user_msg is not None and asst_msg is None:
+                asst_msg = clean_text(value)
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=5):
+            continue
+        if len(asst_msg.split()) > 500:
+            continue
+
+        # If there's a system message, prepend to instruction
+        if system_msg and len(system_msg) < 200:
+            full_instruction = f"{system_msg}\n\n{user_msg}"
+        else:
+            full_instruction = user_msg
+
+        samples.append({
+            "instruction": full_instruction,
+            "input": "",
+            "output": asst_msg
+        })
+
+        if len(samples) >= target:
+            break
+
+        if (i + 1) % 50000 == 0:
+            print(f"  Scanned {i+1:,}/{total}, kept {len(samples):,}...")
+
+    print(f"  Loaded {len(samples):,} samples from SlimOrca")
+    return samples
+
+
+def fetch_alpaca_cleaned() -> list:
+    """Fetch cleaned Stanford Alpaca."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Alpaca Cleaned] Loading...")
+    print(f"{'='*60}")
+
+    ds = load_dataset("yahma/alpaca-cleaned", split="train",
+                      trust_remote_code=False)
+
+    samples = []
+    for row in ds:
+        instruction = clean_text(row.get('instruction', ''))
+        inp = clean_text(row.get('input', ''))
+        output = clean_text(row.get('output', ''))
+
+        if not instruction or not output:
+            continue
+        if not is_english_enough(instruction) or not is_english_enough(output):
+            continue
+        if not is_quality_response(output, min_words=3):
+            continue
+        if len(output.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": instruction,
+            "input": inp if inp else "",
+            "output": output
+        })
+
+    print(f"  Loaded {len(samples):,} samples from Alpaca-cleaned")
+    return samples
+
+
+# ─── Main Pipeline ──────────────────────────────────────────────────────────
+
+def compute_tokens(samples: list, tokens_per_word: float = 1.3) -> int:
+    """Estimate total tokens in a sample list."""
+    total = 0
+    for s in samples:
+        words = (len(s['instruction'].split()) +
+                 len(s.get('input', '').split()) +
+                 len(s['output'].split()))
+        total += int(words * tokens_per_word)
+    return total
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build massive ultra-clean instruction dataset"
+    )
+    parser.add_argument(
+        "--target_tokens", type=int, default=100_000_000,
+        help="Target token count (default: 100M)"
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="Base/Datasets/finetune_english",
+        help="Output directory for train.json / val.json"
+    )
+    parser.add_argument(
+        "--skip_download", action="store_true",
+        help="Skip downloading and just rebuild from cached sources"
+    )
+    parser.add_argument(
+        "--val_fraction", type=float, default=0.02,
+        help="Fraction of data for validation (default: 2%%)"
+    )
+    args = parser.parse_args()
+
+    print(f"\n{'#'*60}")
+    print(f"  INSTRUCTION DATASET BUILDER")
+    print(f"  Target: {args.target_tokens:,} tokens")
+    print(f"  Output: {args.output_dir}")
+    print(f"{'#'*60}")
+
+    try:
+        import datasets
+        print(f"  datasets v{datasets.__version__}")
+    except ImportError:
+        print("\n  ERROR: pip install datasets")
+        return
+
+    t0 = time.time()
+
+    # How many samples per source (rough allocation for diversity)
+    # At ~50 words/sample avg, 100M tokens ≈ 1.5M samples
+    # But real samples average more like 80 words, so ~960K samples for 100M tokens
+
+    all_samples = []
+
+    # Source 1: OpenAssistant — real human conversations (~20K)
+    oasst = fetch_oasst(target=20000)
+    all_samples.extend(oasst)
+
+    # Source 2: Dolly — all human-written (~14K)
+    dolly = fetch_dolly()
+    all_samples.extend(dolly)
+
+    # Source 3: UltraChat 200K — large diverse conversations
+    ultrachat = fetch_ultrachat(target=150000)
+    all_samples.extend(ultrachat)
+
+    # Source 4: WizardLM Evol-Instruct — complex evolved instructions (~70K)
+    wizardlm = fetch_wizardlm(target=70000)
+    all_samples.extend(wizardlm)
+
+    # Source 5: SlimOrca-Dedup — reasoning and FLAN
+    orca = fetch_slimorca(target=200000)
+    all_samples.extend(orca)
+
+    # Source 6: Alpaca-cleaned (~52K)
+    alpaca = fetch_alpaca_cleaned()
+    all_samples.extend(alpaca)
+
+    # Source 7: Open-Platypus — STEM/reasoning
+    platypus = fetch_openplatypus()
+    all_samples.extend(platypus)
+
+    # Source 8: No Robots — human-written, high quality (~10K)
+    no_robots = fetch_no_robots()
+    all_samples.extend(no_robots)
+
+    # Source 9: Asterizer identity (hand-crafted)
+    identity = build_asterizer_identity()
+    # Repeat identity samples to ensure they're well-learned (1% of data)
+    identity_target = max(500, len(all_samples) // 100)
+    identity_expanded = []
+    while len(identity_expanded) < identity_target:
+        identity_expanded.extend(identity)
+    identity_expanded = identity_expanded[:identity_target]
+    all_samples.extend(identity_expanded)
+
+    print(f"\n--- Raw collection complete ---")
+    print(f"  Total raw samples: {len(all_samples):,}")
+    print(f"  Est. tokens: {compute_tokens(all_samples):,}")
+
+    # Shuffle before dedup to mix sources
+    random.seed(42)
+    random.shuffle(all_samples)
+
+    # Deduplication
+    print(f"\nDeduplicating...")
+    deduped = deduplicate(all_samples)
+    print(f"  Before: {len(all_samples):,} → After: {len(deduped):,} "
+          f"(removed {len(all_samples) - len(deduped):,} dupes)")
+
+    # Check token count
+    tok_count = compute_tokens(deduped)
+    print(f"  Est. tokens after dedup: {tok_count:,}")
+
+    # If we exceeded target, trim
+    if tok_count > args.target_tokens * 1.1:
+        # Keep identity samples, trim the rest
+        identity_fps = set(_fingerprint(s['output']) for s in identity)
+        identity_kept = [s for s in deduped if _fingerprint(s['output']) in identity_fps]
+        rest = [s for s in deduped if _fingerprint(s['output']) not in identity_fps]
+        random.shuffle(rest)
+
+        # Binary search for right cutoff  
+        lo, hi = 0, len(rest)
+        while lo < hi:
+            mid = (lo + hi) // 2
+            if compute_tokens(rest[:mid] + identity_kept) < args.target_tokens:
+                lo = mid + 1
+            else:
+                hi = mid
+        rest = rest[:lo]
+        deduped = rest + identity_kept
+        random.shuffle(deduped)
+        tok_count = compute_tokens(deduped)
+        print(f"  Trimmed to {len(deduped):,} samples ({tok_count:,} tokens)")
+
+    # Final shuffle
+    random.shuffle(deduped)
+
+    # Split train/val
+    val_size = max(500, int(len(deduped) * args.val_fraction))
+    val_data = deduped[:val_size]
+    train_data = deduped[val_size:]
+
+    print(f"\n  Train: {len(train_data):,} samples ({compute_tokens(train_data):,} tokens)")
+    print(f"  Val:   {len(val_data):,} samples ({compute_tokens(val_data):,} tokens)")
+
+    # Write output
+    os.makedirs(args.output_dir, exist_ok=True)
+    train_path = os.path.join(args.output_dir, "train.json")
+    val_path = os.path.join(args.output_dir, "val.json")
+
+    with open(train_path, 'w', encoding='utf-8') as f:
+        json.dump(train_data, f, indent=2, ensure_ascii=False)
+    with open(val_path, 'w', encoding='utf-8') as f:
+        json.dump(val_data, f, indent=2, ensure_ascii=False)
+
+    elapsed = time.time() - t0
+
+    print(f"\n{'#'*60}")
+    print(f"  DATASET BUILD COMPLETE")
+    print(f"  Train: {train_path} ({len(train_data):,} samples)")
+    print(f"  Val:   {val_path} ({len(val_data):,} samples)")
+    print(f"  Total tokens: ~{compute_tokens(deduped):,}")
+    print(f"  Time: {elapsed:.0f}s")
+    print(f"{'#'*60}")
+    print(f"\nNext: litgpt finetune_full --config Base/configs/finetune_100m_english_instruct.yaml")
+
+
+if __name__ == "__main__":
+    main()