LUNA 100M: cloud-ready training pipeline

- Config-driven train.py with auto/manual hardware detection
- train_config.yaml with auto_config toggle
- setup_and_train.sh: one-command RunPod deployment
- fetch_data.py: HuggingFace/GDrive dataset downloader
- benchmark_runpod.py: local benchmark + cloud cost estimator
- Tokenizer files, data preprocessing scripts, GGUF converter

.gitattributes

@@ -1,2 +0,0 @@
-# Auto detect text files and perform LF normalization
-* text=auto

.gitignore

@@ -0,0 +1,60 @@
+# ─── Python ───────────────────────────────────────────────────────────────────
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.egg-info/
+*.egg
+dist/
+build/
+.eggs/
+*.spec
+
+# ─── Virtual environments ─────────────────────────────────────────────────────
+.venv/
+venv/
+env/
+.env
+
+# ─── Large binary data — NEVER commit these ───────────────────────────────────
+# LitData tokenized chunks (stream from cloud storage instead)
+Base/data/
+Data preprocess/
+
+# Raw / processed datasets (host on HuggingFace or GDrive)
+Base/Datasets/
+
+# Model checkpoints and weights (host on HuggingFace model hub)
+Base/out/
+
+# GGUF quantisations
+quantisations/*.gguf
+quantisations/*.bin
+
+# Zip archives
+*.zip
+
+# ─── Tokenizer cache (download at runtime) ────────────────────────────────────
+Base/checkpoints/EleutherAI/pythia-160m/.cache/
+
+# ─── Logs / temp ──────────────────────────────────────────────────────────────
+*.log
+temp_*.txt
+logs/
+tensorboard/
+lightning_logs/
+out/
+runpod_cost_estimate.json
+
+# ─── OS / editor ──────────────────────────────────────────────────────────────
+.DS_Store
+Thumbs.db
+*.swp
+*.swo
+.idea/
+.vscode/
+*.code-workspace
+
+# ─── Temporary / generated ────────────────────────────────────────────────────
+temp_prompt_*.txt
+runpod_cost_estimate.json

Base/checkpoints/EleutherAI/pythia-160m/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.24.0",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

Base/checkpoints/EleutherAI/pythia-160m/tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "name_or_path": "EleutherAI/gpt-neox-20b",
+  "special_tokens_map_file": "/admin/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/4e49eadb5d14bd22f314ec3f45b69a87b88c7691/special_tokens_map.json",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

Base/configs/finetune_100m_english_instruct.yaml

@@ -0,0 +1,55 @@
+# Instruction Fine-Tuning Config for English-Pretrained 100M Model
+# Full fine-tuning (all weights) on massive diverse instruct dataset (~100M tokens)
+# Longer sequences than v1 (512 vs 256) for richer, more detailed outputs
+# Checkpoint from English continued pretraining
+
+# Base checkpoint to fine-tune from (output of pretrain_100m_english.yaml)
+checkpoint_dir: Base/out/pretrain/custom-100m-english/final_raw
+
+# Output directory
+out_dir: Base/out/finetune/custom-100m-english-instruct
+
+# Precision
+precision: bf16-mixed
+
+# Data — new massive diverse dataset
+data:
+  class_path: litgpt.data.JSON
+  init_args:
+    json_path: Base/Datasets/finetune_english
+    mask_prompt: true
+    val_split_fraction: null
+    prompt_style: alpaca
+    num_workers: 4
+
+# Training
+train:
+  save_interval: 1000
+  log_interval: 10
+  global_batch_size: 64
+  micro_batch_size: 8
+  lr_warmup_steps: 200
+  epochs: 2
+  max_seq_length: 512
+  min_lr: 5.0e-7
+
+# Evaluation
+eval:
+  interval: 500
+  max_new_tokens: 200
+  max_iters: 100
+  initial_validation: true
+  final_validation: true
+
+# Optimizer — slightly lower LR for larger, more diverse data
+optimizer:
+  class_path: torch.optim.AdamW
+  init_args:
+    lr: 1.0e-5
+    weight_decay: 0.01
+    betas:
+      - 0.9
+      - 0.95
+
+# Logging
+logger_name: csv

Base/configs/finetune_100m_instruct.yaml

@@ -0,0 +1,56 @@
+# Instruction Fine-Tuning Config for Custom 100M Model
+# Full fine-tuning (all weights, no LoRA) on English instruction datasets
+# ~41K samples, 3 epochs, short sequences (max ~160 tokens)
+
+# Base checkpoint to fine-tune from
+checkpoint_dir: Base/out/pretrain/custom-100m-3b-full/final
+
+# Output directory
+out_dir: Base/out/finetune/custom-100m-instruct
+
+# Precision
+precision: bf16-mixed
+
+# Data
+data:
+  class_path: litgpt.data.JSON
+  init_args:
+    json_path: Base/Datasets/finetune
+    mask_prompt: true
+    val_split_fraction: null
+    prompt_style: alpaca
+    num_workers: 4
+
+# Training
+train:
+  save_interval: 500
+  log_interval: 10
+  global_batch_size: 64
+  micro_batch_size: 16
+  lr_warmup_steps: 50
+  epochs: 3
+  max_seq_length: 256
+  min_lr: 1.0e-6
+
+# Evaluation
+eval:
+  interval: 200
+  max_new_tokens: 100
+  max_iters: 100
+  initial_validation: true
+  final_validation: true
+
+# Optimizer
+optimizer:
+  class_path: torch.optim.AdamW
+  init_args:
+    lr: 2.0e-5
+    weight_decay: 0.01
+    betas:
+      - 0.9
+      - 0.95
+
+# Logging
+logger_name: csv
+
+seed: 42

Base/configs/pretrain_100m_10m.yaml

@@ -0,0 +1,49 @@
+# pretrain_100m_10m.yaml
+# Small test run config: ~100M param custom model on ~10M tokens
+# Use this to verify training pipeline works before committing to full 3B run
+
+model_name: pythia-14m
+model_config:
+  name: custom-pythia-like-100m
+  block_size: 1024
+  n_layer: 10
+  n_embd: 768
+  n_head: 12
+  vocab_size: 50254
+  padding_multiple: 128
+  rotary_percentage: 0.25
+
+out_dir: Base/out/pretrain/custom-100m-10m-test
+
+precision: bf16-mixed
+
+data:
+  class_path: litgpt.data.LitData
+  init_args:
+    data_path: Base/data/litdata_10m
+
+train:
+  save_interval: 500
+  log_interval: 5
+  global_batch_size: 16
+  micro_batch_size: 4
+  lr_warmup_steps: 50
+  max_tokens: 10000000
+  max_seq_length: 1024
+  tie_embeddings: true
+  max_norm: 1.0
+  min_lr: 6.0e-05
+
+optimizer:
+  class_path: torch.optim.AdamW
+  init_args:
+    lr: 6.0e-04
+    weight_decay: 0.1
+    betas: [0.9, 0.95]
+    eps: 1.0e-08
+
+devices: 1
+num_nodes: 1
+tokenizer_dir: Base/checkpoints/EleutherAI/pythia-160m
+logger_name: tensorboard
+seed: 1337

Base/configs/pretrain_100m_3b.yaml

@@ -0,0 +1,50 @@
+# pretrain_100m_3b.yaml
+# Full pretraining config: ~100M param custom model on ~27M filtered tokens
+# With the strict quality filter (language_score > 0.99), only ~27M tokens qualify
+# Training for ~270M tokens = ~10 epochs over the data
+
+model_name: pythia-14m
+model_config:
+  name: custom-pythia-like-100m
+  block_size: 1024
+  n_layer: 10
+  n_embd: 768
+  n_head: 12
+  vocab_size: 50254
+  padding_multiple: 128
+  rotary_percentage: 0.25
+
+out_dir: Base/out/pretrain/custom-100m-3b
+
+precision: bf16-mixed
+
+data:
+  class_path: litgpt.data.LitData
+  init_args:
+    data_path: Base/data/litdata_3b
+
+train:
+  save_interval: 2000
+  log_interval: 10
+  global_batch_size: 64
+  micro_batch_size: 4
+  lr_warmup_steps: 50
+  max_tokens: 27000000
+  max_seq_length: 1024
+  tie_embeddings: true
+  max_norm: 1.0
+  min_lr: 6.0e-05
+
+optimizer:
+  class_path: torch.optim.AdamW
+  init_args:
+    lr: 6.0e-04
+    weight_decay: 0.1
+    betas: [0.9, 0.95]
+    eps: 1.0e-08
+
+devices: 1
+num_nodes: 1
+tokenizer_dir: Base/checkpoints/EleutherAI/pythia-160m
+logger_name: tensorboard
+seed: 1337

Base/configs/pretrain_100m_3b_full.yaml

@@ -0,0 +1,53 @@
+# pretrain_100m_3b_full.yaml
+# Full bulk pretraining: ~100M param model on 3B tokens (score >= 0.96)
+# Optimized for RTX 4060 Ti 16GB — targeting ~90% VRAM utilization
+# Checkpoints every 1000 steps for resume safety
+
+model_name: pythia-14m
+model_config:
+  name: custom-pythia-like-100m
+  block_size: 1024
+  n_layer: 10
+  n_embd: 768
+  n_head: 12
+  vocab_size: 50254
+  padding_multiple: 128
+  rotary_percentage: 0.25
+
+# Separate output — does NOT override old checkpoints
+out_dir: Base/out/pretrain/custom-100m-3b-full
+
+precision: bf16-mixed
+
+data:
+  class_path: litgpt.data.LitData
+  init_args:
+    data_path: Base/data/litdata_3b
+
+train:
+  save_interval: 1000
+  log_interval: 10
+  # micro_batch_size=12 pushes VRAM to ~14GB (90% of 16GB)
+  # global_batch=120 = 10 gradient accumulation steps
+  global_batch_size: 120
+  micro_batch_size: 12
+  lr_warmup_steps: 500
+  max_tokens: 3000000000
+  max_seq_length: 1024
+  tie_embeddings: true
+  max_norm: 1.0
+  min_lr: 6.0e-05
+
+optimizer:
+  class_path: torch.optim.AdamW
+  init_args:
+    lr: 6.0e-04
+    weight_decay: 0.1
+    betas: [0.9, 0.95]
+    eps: 1.0e-08
+
+devices: 1
+num_nodes: 1
+tokenizer_dir: Base/checkpoints/EleutherAI/pythia-160m
+logger_name: tensorboard
+seed: 1337

Base/configs/pretrain_100m_english.yaml

@@ -0,0 +1,58 @@
+# pretrain_100m_english.yaml
+# Continued pretraining on ultra-clean English corpus
+# Initializes from existing 3B-pretrained checkpoint (no random init)
+# Lower learning rate for stable knowledge absorption
+# RTX 4060 Ti 16GB — same batch settings as 3B run
+
+model_name: pythia-14m
+model_config:
+  name: custom-pythia-like-100m
+  block_size: 1024
+  n_layer: 10
+  n_embd: 768
+  n_head: 12
+  vocab_size: 50254
+  padding_multiple: 128
+  rotary_percentage: 0.25
+
+# Fresh output directory for the English continued pretraining run
+out_dir: Base/out/pretrain/custom-100m-english
+
+# Load weights from existing pretrained checkpoint (raw model weights only)
+initial_checkpoint_dir: Base/out/pretrain/custom-100m-3b-full/final_raw
+
+precision: bf16-mixed
+
+data:
+  class_path: litgpt.data.LitData
+  init_args:
+    data_path: Base/data/litdata_english
+
+train:
+  save_interval: 500
+  log_interval: 10
+  # Same batch config as the 3B run — proven stable on this GPU
+  global_batch_size: 120
+  micro_batch_size: 12
+  lr_warmup_steps: 200
+  # 3 epochs over ~50M tokens = 150M token budget
+  max_tokens: 150000000
+  max_seq_length: 1024
+  tie_embeddings: true
+  max_norm: 1.0
+  min_lr: 1.0e-05
+
+optimizer:
+  class_path: torch.optim.AdamW
+  init_args:
+    # 6x lower than initial pretraining (6e-4) for stable continued learning
+    lr: 1.0e-04
+    weight_decay: 0.1
+    betas: [0.9, 0.95]
+    eps: 1.0e-08
+
+devices: 1
+num_nodes: 1
+tokenizer_dir: Base/checkpoints/EleutherAI/pythia-160m
+logger_name: tensorboard
+seed: 1337

Base/scripts/audit_5b_pretrain.py

@@ -0,0 +1,494 @@
+# -*- coding: utf-8 -*-
+"""
+COMPREHENSIVE AUDIT of litdata_pretrain_final (5B tokens)
+
+Checks:
+  1. Binary integrity — all 308 chunks readable, headers valid, correct dims
+  2. Token range validation — all tokens in [0, vocab_size), no garbage
+  3. EOS placement — correct document boundaries
+  4. Duplicate detection — sample chunks for near-duplicate documents
+  5. Token distribution — check for anomalous frequency spikes (noise)
+  6. Decode quality — random sample of 50 documents, decode + inspect
+  7. Training readiness — correct block_size, total tokens match config
+
+This script is READ-ONLY. It does NOT modify data.
+"""
+
+import json
+import os
+import re
+import time
+import hashlib
+from pathlib import Path
+from collections import Counter, defaultdict
+
+import numpy as np
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+FINAL_DIR = ROOT / "Base" / "data" / "litdata_pretrain_final"
+TOKENIZER_PATH = str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+EOS_TOKEN_ID = 0
+VOCAB_SIZE = 50277
+
+# How many chunks to deeply scan for duplicates (all 308 is fine for audit)
+MAX_CHUNKS_DEEP = 308
+# Number of random documents to decode and display
+DECODE_SAMPLES = 50
+# Min hash length for near-duplicate detection (chars)
+HASH_WINDOW = 200
+
+
+def read_chunk(filepath):
+    """Read a single chunk binary file. Returns (blocks, num_blocks)."""
+    with open(filepath, "rb") as f:
+        raw = f.read()
+
+    # Parse header
+    num_blocks = np.frombuffer(raw[:4], dtype=np.uint32)[0]
+    header_size = 4 + (num_blocks + 1) * 4  # 1 uint32 + (num_blocks+1) offsets
+    data_bytes = raw[header_size:]
+
+    expected_tokens = num_blocks * BLOCK_SIZE
+    expected_bytes = expected_tokens * DTYPE().itemsize
+
+    tokens = np.frombuffer(data_bytes[:expected_bytes], dtype=DTYPE)
+    return tokens, int(num_blocks)
+
+
+def extract_documents(tokens):
+    """Split a token array at EOS boundaries into individual documents."""
+    eos_positions = np.where(tokens == EOS_TOKEN_ID)[0]
+    docs = []
+    start = 0
+    for eos_pos in eos_positions:
+        if eos_pos > start:
+            doc_tokens = tokens[start:eos_pos]
+            if len(doc_tokens) > 0:
+                docs.append(doc_tokens)
+        start = eos_pos + 1
+    # Trailing tokens (no final EOS — partial doc carried across chunks)
+    if start < len(tokens):
+        remaining = tokens[start:]
+        if len(remaining) > 5:  # ignore tiny fragments
+            docs.append(remaining)
+    return docs
+
+
+def token_hash(doc_tokens, window=200):
+    """Hash first N tokens for near-duplicate detection."""
+    key = doc_tokens[:window].tobytes()
+    return hashlib.md5(key).hexdigest()
+
+
+def main():
+    t_start = time.time()
+
+    # Load index
+    with open(FINAL_DIR / "index.json") as f:
+        index = json.load(f)
+    chunks_meta = index["chunks"]
+    config = index.get("config", {})
+    num_chunks = len(chunks_meta)
+
+    print(f"{'='*75}")
+    print(f"  COMPREHENSIVE AUDIT — litdata_pretrain_final")
+    print(f"{'='*75}")
+    print(f"  Chunks in index: {num_chunks}")
+    print(f"  Config: {config}")
+    print(f"  Expected BLOCK_SIZE: {BLOCK_SIZE}")
+    print(f"  Expected VOCAB_SIZE: {VOCAB_SIZE}")
+    print(f"  Expected EOS: {EOS_TOKEN_ID}")
+    print()
+
+    # ═════════════════════════════════════════════════════════════════════
+    # CHECK 1: Binary integrity + token range
+    # ═════════════════════════════════════════════════════════════════════
+    print(f"  CHECK 1: BINARY INTEGRITY + TOKEN RANGE")
+    print(f"  {'-'*60}")
+
+    total_tokens = 0
+    total_blocks = 0
+    missing_files = []
+    corrupt_chunks = []
+    out_of_range_chunks = []
+    eos_counts = []
+    chunk_token_counts = []
+
+    # Global token frequency (sample every 10th chunk for speed)
+    global_freq = Counter()
+    FREQ_SAMPLE_INTERVAL = 3  # sample every 3rd chunk
+
+    # For duplicate detection
+    doc_hashes = defaultdict(list)  # hash -> [(chunk_idx, doc_idx)]
+    total_docs = 0
+    doc_lengths = []
+
+    # For decode sampling: collect random docs
+    sample_docs = []
+    np.random.seed(42)
+
+    for ci, meta in enumerate(chunks_meta):
+        filename = meta["filename"]
+        filepath = FINAL_DIR / filename
+        expected_dim = meta["dim"]
+
+        if not filepath.exists():
+            missing_files.append(filename)
+            print(f"    MISSING: {filename}")
+            continue
+
+        try:
+            tokens, num_blocks = read_chunk(filepath)
+        except Exception as e:
+            corrupt_chunks.append((filename, str(e)))
+            print(f"    CORRUPT: {filename} — {e}")
+            continue
+
+        actual_dim = len(tokens)
+        if actual_dim != expected_dim:
+            corrupt_chunks.append((filename, f"dim mismatch: expected {expected_dim}, got {actual_dim}"))
+            print(f"    DIM MISMATCH: {filename} expected {expected_dim}, got {actual_dim}")
+
+        total_tokens += actual_dim
+        total_blocks += num_blocks
+        chunk_token_counts.append(actual_dim)
+
+        # Token range check
+        min_tok = int(tokens.min())
+        max_tok = int(tokens.max())
+        if min_tok < 0 or max_tok >= VOCAB_SIZE:
+            out_of_range_chunks.append((filename, min_tok, max_tok))
+            print(f"    OUT OF RANGE: {filename} min={min_tok} max={max_tok}")
+
+        # EOS count
+        eos_count = int(np.sum(tokens == EOS_TOKEN_ID))
+        eos_counts.append(eos_count)
+
+        # Token frequency (sampled)
+        if ci % FREQ_SAMPLE_INTERVAL == 0:
+            unique, counts = np.unique(tokens, return_counts=True)
+            for tok, cnt in zip(unique, counts):
+                global_freq[int(tok)] += int(cnt)
+
+        # Extract documents for duplicate + quality check
+        if ci < MAX_CHUNKS_DEEP:
+            docs = extract_documents(tokens)
+            for di, doc in enumerate(docs):
+                total_docs += 1
+                doc_lengths.append(len(doc))
+                if len(doc) >= 50:
+                    h = token_hash(doc)
+                    doc_hashes[h].append((ci, di))
+
+                # Random sample for decode
+                if len(sample_docs) < DECODE_SAMPLES and np.random.random() < 0.0005:
+                    sample_docs.append(doc)
+
+        if (ci + 1) % 50 == 0:
+            print(f"    Scanned {ci+1}/{num_chunks} chunks... ({total_tokens:,} tokens)")
+
+    print(f"    Scanned all {num_chunks} chunks: {total_tokens:,} total tokens")
+    print()
+
+    # Results for Check 1
+    issues = []
+    if missing_files:
+        issues.append(f"MISSING FILES: {len(missing_files)}")
+    if corrupt_chunks:
+        issues.append(f"CORRUPT CHUNKS: {len(corrupt_chunks)}")
+    if out_of_range_chunks:
+        issues.append(f"OUT-OF-RANGE TOKENS: {len(out_of_range_chunks)} chunks")
+
+    if not issues:
+        print(f"    ✓ All {num_chunks} chunks intact, all tokens in [0, {VOCAB_SIZE})")
+    else:
+        for issue in issues:
+            print(f"    ✗ {issue}")
+    print()
+
+    # ═════════════════════════════════════════════════════════════════════
+    # CHECK 2: EOS + DOCUMENT BOUNDARIES
+    # ═════════════════════════════════════════════════════════════════════
+    print(f"  CHECK 2: EOS & DOCUMENT BOUNDARIES")
+    print(f"  {'-'*60}")
+
+    total_eos = sum(eos_counts)
+    avg_eos = total_eos / max(num_chunks, 1)
+    min_eos = min(eos_counts) if eos_counts else 0
+    max_eos = max(eos_counts) if eos_counts else 0
+
+    print(f"    Total EOS tokens: {total_eos:,}")
+    print(f"    Avg EOS per chunk: {avg_eos:.1f}")
+    print(f"    Min/Max EOS per chunk: {min_eos} / {max_eos}")
+    print(f"    Total documents found: {total_docs:,}")
+
+    if doc_lengths:
+        dl = np.array(doc_lengths)
+        print(f"    Doc length (tokens): min={int(dl.min())}, median={int(np.median(dl))}, "
+              f"mean={int(dl.mean())}, max={int(dl.max())}")
+        tiny_docs = int(np.sum(dl < 20))
+        short_docs = int(np.sum(dl < 50))
+        long_docs = int(np.sum(dl > 50000))
+        print(f"    Tiny docs (<20 tok): {tiny_docs:,} ({100*tiny_docs/len(dl):.2f}%)")
+        print(f"    Short docs (<50 tok): {short_docs:,} ({100*short_docs/len(dl):.2f}%)")
+        print(f"    Very long docs (>50K tok): {long_docs:,}")
+
+    # Flag if too many tiny docs (noise)
+    if doc_lengths and tiny_docs / len(dl) > 0.05:
+        print(f"    ⚠ WARNING: {100*tiny_docs/len(dl):.1f}% tiny docs — possible noise")
+    else:
+        print(f"    ✓ Document boundaries look healthy")
+    print()
+
+    # ═════════════════════════════════════════════════════════════════════
+    # CHECK 3: DUPLICATE DETECTION
+    # ═════════════════════════════════════════════════════════════════════
+    print(f"  CHECK 3: NEAR-DUPLICATE DETECTION")
+    print(f"  {'-'*60}")
+
+    dup_groups = {h: locs for h, locs in doc_hashes.items() if len(locs) > 1}
+    dup_doc_count = sum(len(locs) - 1 for locs in dup_groups.values())
+
+    print(f"    Unique doc hashes: {len(doc_hashes):,}")
+    print(f"    Duplicate groups: {len(dup_groups):,}")
+    print(f"    Duplicate docs (extra copies): {dup_doc_count:,}")
+    dup_pct = 100 * dup_doc_count / max(total_docs, 1)
+    print(f"    Duplication rate: {dup_pct:.3f}%")
+
+    if dup_pct > 5.0:
+        print(f"    ⚠ WARNING: High duplication rate ({dup_pct:.1f}%). Consider deduplication.")
+    elif dup_pct > 1.0:
+        print(f"    ⚠ MODERATE: {dup_pct:.2f}% duplicates. Acceptable but not ideal.")
+    else:
+        print(f"    ✓ Very low duplication ({dup_pct:.3f}%). Excellent.")
+
+    # Show a few duplicate examples
+    if dup_groups:
+        print(f"\n    Top 5 duplicate groups (by copy count):")
+        sorted_dups = sorted(dup_groups.items(), key=lambda x: -len(x[1]))[:5]
+        for h, locs in sorted_dups:
+            print(f"      hash={h[:12]}... : {len(locs)} copies in chunks {[l[0] for l in locs[:6]]}")
+    print()
+
+    # ═════════════════════════════════════════════════════════════════════
+    # CHECK 4: TOKEN DISTRIBUTION (anomaly detection)
+    # ═════════════════════════════════════════════════════════════════════
+    print(f"  CHECK 4: TOKEN DISTRIBUTION ANALYSIS")
+    print(f"  {'-'*60}")
+
+    total_sampled = sum(global_freq.values())
+    print(f"    Sampled tokens: {total_sampled:,} (from every {FREQ_SAMPLE_INTERVAL}rd chunk)")
+
+    # Top 20 most frequent tokens
+    most_common = global_freq.most_common(30)
+    print(f"    Top 30 tokens by frequency:")
+    for tok_id, count in most_common:
+        pct = 100 * count / total_sampled
+        print(f"      token {tok_id:>6}: {count:>12,} ({pct:>5.2f}%)")
+
+    # Check for suspicious spikes
+    # If any single non-EOS token is >10% of all tokens, it's suspicious
+    suspicious = []
+    for tok_id, count in most_common:
+        pct = count / total_sampled
+        if tok_id != EOS_TOKEN_ID and pct > 0.10:
+            suspicious.append((tok_id, pct))
+
+    if suspicious:
+        print(f"\n    ⚠ SUSPICIOUS: These non-EOS tokens appear in >10% of data:")
+        for tok_id, pct in suspicious:
+            print(f"      token {tok_id}: {100*pct:.2f}%")
+    else:
+        print(f"\n    ✓ No anomalous token frequency spikes detected")
+
+    # Vocab coverage
+    unique_tokens = len(global_freq)
+    coverage_pct = 100 * unique_tokens / VOCAB_SIZE
+    print(f"    Unique tokens seen: {unique_tokens:,} / {VOCAB_SIZE:,} ({coverage_pct:.1f}% vocab coverage)")
+
+    # Check for dead zones (large ranges of unused tokens)
+    used_set = set(global_freq.keys())
+    unused_ranges = []
+    start_unused = None
+    for i in range(VOCAB_SIZE):
+        if i not in used_set:
+            if start_unused is None:
+                start_unused = i
+        else:
+            if start_unused is not None:
+                gap = i - start_unused
+                if gap > 500:
+                    unused_ranges.append((start_unused, i - 1, gap))
+                start_unused = None
+
+    if unused_ranges:
+        print(f"    Large unused token ranges (>500):")
+        for s, e, g in unused_ranges[:5]:
+            print(f"      tokens {s}-{e} ({g} unused)")
+    print()
+
+    # ═════════════════════════════════════════════════════════════════════
+    # CHECK 5: DECODE QUALITY — sample random documents
+    # ═════════════════════════════════════════════════════════════════════
+    print(f"  CHECK 5: DECODED DOCUMENT SAMPLES")
+    print(f"  {'-'*60}")
+
+    # If we didn't get enough samples randomly, grab from specific chunks
+    if len(sample_docs) < DECODE_SAMPLES:
+        # Grab from spread-out chunks
+        sample_chunks = np.linspace(0, num_chunks - 1, min(DECODE_SAMPLES - len(sample_docs), 25), dtype=int)
+        for sci in sample_chunks:
+            if len(sample_docs) >= DECODE_SAMPLES:
+                break
+            meta = chunks_meta[sci]
+            filepath = FINAL_DIR / meta["filename"]
+            try:
+                tokens, _ = read_chunk(filepath)
+                docs = extract_documents(tokens)
+                if docs:
+                    # Pick a random doc from this chunk
+                    idx = np.random.randint(0, len(docs))
+                    sample_docs.append(docs[idx])
+            except:
+                pass
+
+    # Now decode
+    from tokenizers import Tokenizer
+    tokenizer = Tokenizer.from_file(TOKENIZER_PATH)
+
+    quality_issues = 0
+    noise_docs = 0
+    non_english_docs = 0
+
+    RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+    RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+    RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+
+    for si, doc_tokens in enumerate(sample_docs[:DECODE_SAMPLES]):
+        text = tokenizer.decode(doc_tokens.tolist(), skip_special_tokens=False)
+
+        # Quality checks
+        words = text.split()
+        word_count = len(words)
+        alpha_ratio = sum(c.isalpha() for c in text) / max(len(text), 1)
+        unique_words = len(set(w.lower() for w in words)) / max(word_count, 1)
+
+        is_noisy = False
+        is_non_english = False
+        flags = []
+
+        if alpha_ratio < 0.50:
+            flags.append(f"low-alpha({alpha_ratio:.2f})")
+            is_noisy = True
+        if word_count > 30 and unique_words < 0.15:
+            flags.append(f"repetitive({unique_words:.2f})")
+            is_noisy = True
+        if RE_CJK.search(text) or RE_ARABIC.search(text) or RE_CYRILLIC.search(text):
+            flags.append("non-English")
+            is_non_english = True
+        if word_count < 10:
+            flags.append("very-short")
+            is_noisy = True
+
+        if is_noisy:
+            noise_docs += 1
+        if is_non_english:
+            non_english_docs += 1
+        if flags:
+            quality_issues += 1
+
+        # Print preview for flagged + a few clean ones
+        if flags or si < 5 or si % 10 == 0:
+            preview = text[:300].replace('\n', ' ↵ ')
+            status = f"⚠ {','.join(flags)}" if flags else "✓ clean"
+            print(f"\n    Sample {si+1}/{len(sample_docs[:DECODE_SAMPLES])} [{status}] ({word_count} words, alpha={alpha_ratio:.2f}, unique={unique_words:.2f})")
+            print(f"      \"{preview}...\"")
+
+    print(f"\n    DECODE SUMMARY:")
+    print(f"      Samples checked: {min(len(sample_docs), DECODE_SAMPLES)}")
+    print(f"      Clean: {min(len(sample_docs), DECODE_SAMPLES) - quality_issues}")
+    print(f"      Noisy: {noise_docs}")
+    print(f"      Non-English: {non_english_docs}")
+    print(f"      Quality issues: {quality_issues}")
+
+    if quality_issues / max(len(sample_docs), 1) > 0.1:
+        print(f"      ⚠ WARNING: >10% of sampled docs have quality issues")
+    else:
+        print(f"      ✓ Sample quality looks good")
+    print()
+
+    # ═════════════════════════════════════════════════════════════════════
+    # CHECK 6: TRAINING READINESS
+    # ═════════════════════════════════════════════════════════════════════
+    print(f"  CHECK 6: TRAINING READINESS")
+    print(f"  {'-'*60}")
+
+    index_total = sum(c["dim"] for c in chunks_meta)
+    print(f"    Index total tokens: {index_total:,}")
+    print(f"    Actual total tokens: {total_tokens:,}")
+    if index_total == total_tokens:
+        print(f"    ✓ Index matches actual data perfectly")
+    else:
+        print(f"    ⚠ MISMATCH: index says {index_total:,} but files have {total_tokens:,}")
+
+    # Check all blocks are BLOCK_SIZE aligned
+    misaligned = [c["filename"] for c in chunks_meta if c["dim"] % BLOCK_SIZE != 0]
+    if misaligned:
+        print(f"    ⚠ {len(misaligned)} chunks not BLOCK_SIZE-aligned: {misaligned[:5]}")
+    else:
+        print(f"    ✓ All chunks perfectly BLOCK_SIZE-aligned ({BLOCK_SIZE})")
+
+    # Config check
+    if config.get("block_size") == BLOCK_SIZE:
+        print(f"    ✓ Config block_size matches: {BLOCK_SIZE}")
+    else:
+        print(f"    ⚠ Config block_size: {config.get('block_size')} (expected {BLOCK_SIZE})")
+
+    if config.get("vocab_size") == VOCAB_SIZE:
+        print(f"    ✓ Config vocab_size matches: {VOCAB_SIZE}")
+    else:
+        print(f"    ⚠ Config vocab_size: {config.get('vocab_size')} (expected {VOCAB_SIZE})")
+
+    # Tokens per epoch for batch size calculation
+    print(f"\n    TRAINING PARAMETERS:")
+    seq_len = 1024
+    steps = total_tokens // (120 * seq_len)  # global_batch=120
+    print(f"      Total tokens: {total_tokens:,}")
+    print(f"      Global batch size 120 × seq 1024 = {120*1024:,} tokens/step")
+    print(f"      Total steps for 1 epoch: {steps:,}")
+    print(f"      At ~10 steps/sec (A100): ~{steps/10/60:.0f} min ≈ {steps/10/3600:.1f} hours")
+    print()
+
+    # ═════════════════════════════════════════════════════════════════════
+    # OVERALL VERDICT
+    # ═════════════════════════════════════════════════════════════════════
+    all_issues = []
+    if missing_files: all_issues.append(f"{len(missing_files)} missing files")
+    if corrupt_chunks: all_issues.append(f"{len(corrupt_chunks)} corrupt chunks")
+    if out_of_range_chunks: all_issues.append(f"{len(out_of_range_chunks)} out-of-range chunks")
+    if dup_pct > 5.0: all_issues.append(f"High duplication: {dup_pct:.1f}%")
+    if suspicious: all_issues.append("Suspicious token spikes")
+    if doc_lengths and tiny_docs / len(dl) > 0.05: all_issues.append("Too many tiny docs")
+    if quality_issues / max(len(sample_docs), 1) > 0.1: all_issues.append("Quality issues in samples")
+
+    elapsed = time.time() - t_start
+
+    print(f"{'='*75}")
+    print(f"  AUDIT VERDICT")
+    print(f"{'='*75}")
+    if not all_issues:
+        print(f"  ✓ ALL CHECKS PASSED — Dataset is TRAINING-READY")
+        print(f"    {total_tokens:,} clean tokens across {num_chunks} chunks")
+        print(f"    No corruption, minimal duplicates, good quality")
+    else:
+        print(f"  ⚠ ISSUES FOUND:")
+        for issue in all_issues:
+            print(f"    - {issue}")
+    print(f"\n  Audit completed in {elapsed:.1f}s")
+    print(f"{'='*75}")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/audit_english_clean.py

@@ -0,0 +1,707 @@
+# -*- coding: utf-8 -*-
+"""
+Deep quality audit of litdata_english_clean.
+
+Checks EVERY row for:
+  1. Content bias (topic distribution, over-represented domains)
+  2. Unwanted context (ads, spam, SEO, cookie notices, legal boilerplate, etc.)
+  3. English quality (grammar structure, vocabulary richness, readability)
+  4. LLM learning value (diverse sentence structures, good knowledge density)
+  5. Toxic/harmful content flags
+  6. Residual noise (leftover URLs, code, non-English fragments)
+
+Goal: ensure the data teaches the LLM to understand English very well
+so it can later do SFT on any dataset with strong comprehension.
+"""
+
+import json
+import os
+import re
+import time
+import string
+from pathlib import Path
+from collections import Counter, defaultdict
+
+import numpy as np
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+EOS_TOKEN_ID = 0
+
+print("Loading tokenizer...")
+tokenizer = Tokenizer.from_file(
+    str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+)
+
+# ==============================================================================
+# LITDATA I/O
+# ==============================================================================
+
+def read_all_tokens(litdata_dir):
+    with open(litdata_dir / "index.json") as f:
+        index = json.load(f)
+    chunks = index["chunks"]
+    total_tokens = sum(c["dim"] for c in chunks)
+    print(f"  Reading {len(chunks)} chunks ({total_tokens:,} tokens)...")
+    all_tokens = np.empty(total_tokens, dtype=DTYPE)
+    pos = 0
+    for i, chunk in enumerate(chunks):
+        chunk_path = litdata_dir / chunk["filename"]
+        n_blocks = chunk["chunk_size"]
+        header_ints = 1 + n_blocks + 1
+        header_bytes = header_ints * 4
+        with open(chunk_path, "rb") as f:
+            f.seek(header_bytes)
+            data = np.fromfile(f, dtype=DTYPE, count=chunk["dim"])
+        all_tokens[pos:pos + len(data)] = data
+        pos += len(data)
+    print(f"    Read {len(chunks)} chunks ({pos:,} tokens)")
+    return all_tokens[:pos]
+
+
+def split_documents(token_stream):
+    eos_positions = np.where(token_stream == EOS_TOKEN_ID)[0]
+    docs = []
+    start = 0
+    for eos_pos in eos_positions:
+        if eos_pos > start:
+            docs.append(token_stream[start:eos_pos])
+        start = eos_pos + 1
+    if start < len(token_stream):
+        docs.append(token_stream[start:])
+    return docs
+
+
+# ==============================================================================
+# UNWANTED CONTENT DETECTORS
+# ==============================================================================
+
+# Patterns that suggest ads, spam, SEO, cookie banners, boilerplate
+RE_COOKIE = re.compile(r'(cookie|cookies)\s+(policy|consent|notice|preferences|settings)', re.I)
+RE_PRIVACY = re.compile(r'(privacy\s+policy|terms\s+of\s+(service|use)|legal\s+disclaimer)', re.I)
+RE_SUBSCRIBE = re.compile(r'(subscribe|sign\s*up|newsletter|unsubscribe|opt[\s-]*out)', re.I)
+RE_CLICKBAIT = re.compile(r'(you\s+won\'?t\s+believe|click\s+here|read\s+more|share\s+this|trending\s+now|sponsored|advertisement)', re.I)
+RE_SEO_SPAM = re.compile(r'(best\s+\d+\s+\w+\s+for|top\s+\d+\s+\w+|buy\s+now|free\s+shipping|limited\s+time\s+offer|discount\s+code)', re.I)
+RE_NAVIGATION = re.compile(r'(home\s*>\s*|breadcrumb|sidebar|footer|header|menu|navigation|skip\s+to\s+content)', re.I)
+RE_SOCIAL = re.compile(r'(follow\s+us\s+on|share\s+on\s+(facebook|twitter|linkedin|instagram)|like\s+us\s+on|tweet\s+this)', re.I)
+RE_COMMENT_SECTION = re.compile(r'(leave\s+a\s+(comment|reply)|post\s+a\s+comment|\d+\s+comments?\s|logged\s+in\s+as)', re.I)
+RE_COPYRIGHT = re.compile(r'(all\s+rights\s+reserved|copyright\s+\d{4}|\(c\)\s*\d{4})', re.I)
+RE_BOILERPLATE_LOGIN = re.compile(r'(log\s*in|sign\s*in|create\s+account|forgot\s+password|remember\s+me)', re.I)
+
+# Residual code/technical noise
+RE_RESIDUAL_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_RESIDUAL_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_RESIDUAL_CODE = re.compile(r'(function\s*\(|var\s+\w+\s*=|console\.log|document\.get|if\s*\(\s*\w+\s*[!=]==)', re.I)
+RE_CURLY_BRACES = re.compile(r'\{[^}]{5,}\}')
+RE_HEX_COLORS = re.compile(r'#[0-9a-fA-F]{6}\b')
+
+# Non-English fragments
+RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+RE_DEVANAGARI = re.compile(r'[\u0900-\u097f]{5,}')
+
+# Low quality indicators
+RE_ALLCAPS_LINE = re.compile(r'^[A-Z\s]{20,}$', re.M)
+RE_EXCESSIVE_NUMBERS = re.compile(r'(?:\d+[\s,.-]*){10,}')
+RE_LIST_SPAM = re.compile(r'(?:^\s*[-*]\s*.{3,50}\n){10,}', re.M)
+
+# Toxic content indicators (basic - not exhaustive)
+TOXIC_TERMS = [
+    'kill yourself', 'kys', 'hate speech', 'racial slur',
+    'white supremac', 'nazi', 'nigger', 'faggot',
+]
+
+
+# ==============================================================================
+# TOPIC CLASSIFIER (keyword-based, broad categories)
+# ==============================================================================
+
+TOPIC_KEYWORDS = {
+    "Science": ["experiment", "hypothesis", "molecule", "atom", "chemical", "physics",
+                "biology", "evolution", "species", "organism", "cell", "dna", "gene",
+                "electron", "neutron", "quantum", "telescope", "galaxy", "planet"],
+    "Mathematics": ["equation", "theorem", "algebra", "calculus", "geometry", "integer",
+                    "fraction", "polynomial", "derivative", "integral", "matrix", "probability"],
+    "History": ["century", "civilization", "empire", "dynasty", "revolution", "colonial",
+                "medieval", "ancient", "historian", "archaeological", "monarchy", "treaty"],
+    "Geography": ["continent", "climate", "ocean", "mountain", "river", "latitude",
+                  "longitude", "ecosystem", "peninsula", "volcano", "earthquake", "terrain"],
+    "Literature": ["novel", "poem", "author", "literary", "character", "narrative",
+                   "fiction", "metaphor", "protagonist", "shakespeare", "prose", "genre"],
+    "Technology": ["software", "hardware", "computer", "algorithm", "database", "internet",
+                   "programming", "digital", "server", "network", "processor", "encryption"],
+    "Medicine/Health": ["patient", "symptom", "diagnosis", "treatment", "disease", "infection",
+                       "surgery", "therapy", "vaccine", "antibiotic", "clinical", "chronic"],
+    "Law/Politics": ["constitution", "legislation", "democracy", "parliament", "judiciary",
+                     "amendment", "election", "government", "policy", "regulation", "statute"],
+    "Economics/Business": ["market", "economy", "inflation", "revenue", "investment", "stock",
+                          "profit", "trade", "gdp", "fiscal", "monetary", "corporation"],
+    "Education": ["student", "teacher", "curriculum", "classroom", "university", "academic",
+                  "learning", "examination", "school", "pedagogy", "literacy", "enrollment"],
+    "Philosophy/Religion": ["philosophy", "ethics", "moral", "theological", "spiritual",
+                           "consciousness", "existence", "metaphysics", "belief", "virtue"],
+    "Arts/Culture": ["painting", "sculpture", "museum", "gallery", "architecture", "cinema",
+                    "music", "composer", "portrait", "exhibition", "artistic", "cultural"],
+    "Environment": ["pollution", "conservation", "deforestation", "renewable", "sustainability",
+                    "biodiversity", "emissions", "habitat", "endangered", "recycling"],
+    "Psychology": ["behavior", "cognitive", "emotion", "personality", "anxiety", "depression",
+                  "consciousness", "motivation", "perception", "neuroscience", "memory"],
+    "Sports": ["championship", "tournament", "athlete", "league", "stadium", "coach",
+              "scorer", "goalkeeper", "referee", "olympics", "medal", "cricket"],
+}
+
+
+# ==============================================================================
+# ENGLISH QUALITY METRICS
+# ==============================================================================
+
+def compute_readability(text):
+    """Simplified Flesch-Kincaid readability approximation."""
+    words = text.split()
+    if len(words) < 10:
+        return 0.0
+    sentences = max(len(re.findall(r'[.!?]+', text)), 1)
+    # Approximate syllables: count vowel groups
+    syllables = sum(len(re.findall(r'[aeiouy]+', w.lower())) for w in words)
+    syllables = max(syllables, len(words))  # at least 1 per word
+    words_per_sent = len(words) / sentences
+    syl_per_word = syllables / len(words)
+    # Flesch Reading Ease
+    fre = 206.835 - 1.015 * words_per_sent - 84.6 * syl_per_word
+    return max(0, min(100, fre))
+
+
+def classify_sentence_types(text):
+    """Classify sentences into types for diversity check."""
+    sents = re.split(r'(?<=[.!?])\s+', text[:5000])  # First 5000 chars
+    types = Counter()
+    for s in sents:
+        s = s.strip()
+        if not s:
+            continue
+        if s.endswith('?'):
+            types['question'] += 1
+        elif s.endswith('!'):
+            types['exclamation'] += 1
+        elif any(s.lower().startswith(w) for w in ['because', 'since', 'although', 'while', 'if', 'when', 'whereas']):
+            types['complex'] += 1
+        elif any(s.lower().startswith(w) for w in ['the', 'a ', 'an ', 'this', 'that', 'these', 'those']):
+            types['declarative'] += 1
+        elif any(s.lower().startswith(w) for w in ['for example', 'such as', 'in other words', 'namely']):
+            types['explanatory'] += 1
+        elif any(s.lower().startswith(w) for w in ['however', 'nevertheless', 'moreover', 'furthermore', 'therefore']):
+            types['transitional'] += 1
+        else:
+            types['other'] += 1
+    return types
+
+
+def detect_topic(text_lower):
+    """Classify document into topics based on keyword density."""
+    topics_found = []
+    for topic, keywords in TOPIC_KEYWORDS.items():
+        hits = sum(1 for kw in keywords if kw in text_lower)
+        if hits >= 2:
+            topics_found.append((topic, hits))
+    topics_found.sort(key=lambda x: -x[1])
+    return topics_found
+
+
+# ==============================================================================
+# MAIN AUDIT
+# ==============================================================================
+
+def main():
+    input_dir = ROOT / "Base" / "data" / "litdata_english_clean"
+
+    print(f"\n{'='*75}")
+    print(f"  DEEP QUALITY AUDIT: litdata_english_clean")
+    print(f"  Input: {input_dir}")
+    print(f"{'='*75}")
+
+    # 1. Read and decode all documents
+    t0 = time.time()
+    token_stream = read_all_tokens(input_dir)
+    doc_tokens = split_documents(token_stream)
+    print(f"  Found {len(doc_tokens):,} documents")
+    del token_stream
+
+    print(f"  Decoding ALL {len(doc_tokens):,} documents...")
+    texts = []
+    t1 = time.time()
+    for i, toks in enumerate(doc_tokens):
+        text = tokenizer.decode(toks.tolist(), skip_special_tokens=False)
+        texts.append(text)
+        if (i + 1) % 20000 == 0 or i == len(doc_tokens) - 1:
+            print(f"    Decoded {i+1:,}/{len(doc_tokens):,}")
+    del doc_tokens
+    print(f"  Decoded in {time.time()-t1:.1f}s")
+
+    total_docs = len(texts)
+    print(f"\n  Auditing {total_docs:,} documents across ALL rows...\n")
+
+    # ==================================================================
+    # AUDIT PASS: Scan every document
+    # ==================================================================
+    t2 = time.time()
+
+    # Counters
+    topic_counter = Counter()
+    topic_per_doc = []
+    sentence_type_totals = Counter()
+    readability_scores = []
+    word_counts = []
+    vocab_richness = []
+    avg_sentence_lengths = []
+
+    # Issue trackers
+    issues = {
+        "cookie_privacy": [],
+        "subscribe_newsletter": [],
+        "clickbait_seo": [],
+        "navigation_boilerplate": [],
+        "social_media": [],
+        "comment_section": [],
+        "copyright_legal": [],
+        "login_boilerplate": [],
+        "residual_urls": [],
+        "residual_emails": [],
+        "residual_code": [],
+        "non_english_fragments": [],
+        "allcaps_heavy": [],
+        "excessive_numbers": [],
+        "list_spam": [],
+        "toxic_content": [],
+        "too_short": [],
+        "too_repetitive": [],
+        "low_readability": [],
+        "single_topic_bias": [],
+    }
+
+    # Track flagged doc indices for potential removal
+    flagged_docs = set()
+    flag_reasons = defaultdict(list)
+
+    for i, text in enumerate(texts):
+        text_lower = text.lower()
+        words = text.split()
+        word_count = len(words)
+        word_counts.append(word_count)
+
+        # --- Topic classification ---
+        topics = detect_topic(text_lower)
+        if topics:
+            for t_name, _ in topics[:2]:
+                topic_counter[t_name] += 1
+            topic_per_doc.append(topics[0][0])
+        else:
+            topic_counter["Uncategorized"] += 1
+            topic_per_doc.append("Uncategorized")
+
+        # --- Sentence diversity ---
+        stypes = classify_sentence_types(text)
+        for k, v in stypes.items():
+            sentence_type_totals[k] += v
+
+        # --- Readability ---
+        fre = compute_readability(text)
+        readability_scores.append(fre)
+
+        # --- Vocabulary richness ---
+        if word_count > 20:
+            unique_ratio = len(set(w.lower() for w in words)) / word_count
+            vocab_richness.append(unique_ratio)
+        else:
+            vocab_richness.append(0)
+
+        # --- Avg sentence length ---
+        sents = re.split(r'[.!?]+', text)
+        sents = [s for s in sents if len(s.strip().split()) > 2]
+        if sents:
+            avg_sl = sum(len(s.split()) for s in sents) / len(sents)
+            avg_sentence_lengths.append(avg_sl)
+        else:
+            avg_sentence_lengths.append(0)
+
+        # === ISSUE DETECTION (every row) ===
+        is_flagged = False
+
+        # Cookie/privacy
+        if RE_COOKIE.search(text) or RE_PRIVACY.search(text):
+            m = RE_COOKIE.findall(text) + RE_PRIVACY.findall(text)
+            # Only flag if heavy (multiple matches or large portion)
+            if len(m) >= 2 or (len(m) >= 1 and word_count < 100):
+                issues["cookie_privacy"].append(i)
+                if word_count < 100:
+                    is_flagged = True
+                    flag_reasons[i].append("cookie/privacy boilerplate")
+
+        # Subscribe/newsletter
+        m = RE_SUBSCRIBE.findall(text)
+        if len(m) >= 2:
+            issues["subscribe_newsletter"].append(i)
+            if word_count < 100 and len(m) >= 2:
+                is_flagged = True
+                flag_reasons[i].append("subscribe/newsletter spam")
+
+        # Clickbait/SEO
+        m = RE_CLICKBAIT.findall(text)
+        if m:
+            issues["clickbait_seo"].append(i)
+            if len(m) >= 3:
+                is_flagged = True
+                flag_reasons[i].append("clickbait/SEO content")
+
+        # Navigation boilerplate
+        m = RE_NAVIGATION.findall(text)
+        if len(m) >= 3:
+            issues["navigation_boilerplate"].append(i)
+            if word_count < 80:
+                is_flagged = True
+                flag_reasons[i].append("navigation boilerplate")
+
+        # Social media prompts
+        m = RE_SOCIAL.findall(text)
+        if m:
+            issues["social_media"].append(i)
+
+        # Comment sections
+        m = RE_COMMENT_SECTION.findall(text)
+        if m:
+            issues["comment_section"].append(i)
+
+        # Copyright/legal
+        m = RE_COPYRIGHT.findall(text)
+        if m:
+            issues["copyright_legal"].append(i)
+
+        # Login boilerplate
+        m = RE_BOILERPLATE_LOGIN.findall(text)
+        if len(m) >= 3:
+            issues["login_boilerplate"].append(i)
+            if word_count < 80:
+                is_flagged = True
+                flag_reasons[i].append("login boilerplate")
+
+        # Residual URLs
+        m = RE_RESIDUAL_URL.findall(text)
+        if m:
+            issues["residual_urls"].append(i)
+            is_flagged = True
+            flag_reasons[i].append(f"residual URLs ({len(m)})")
+
+        # Residual emails
+        m = RE_RESIDUAL_EMAIL.findall(text)
+        if m:
+            issues["residual_emails"].append(i)
+
+        # Residual code
+        m = RE_RESIDUAL_CODE.findall(text)
+        if len(m) >= 3:
+            issues["residual_code"].append(i)
+            if len(m) >= 5:
+                is_flagged = True
+                flag_reasons[i].append(f"residual code ({len(m)} matches)")
+
+        # Non-English fragments
+        has_cjk = bool(RE_CJK.search(text))
+        has_arabic = bool(RE_ARABIC.search(text))
+        has_cyrillic = bool(RE_CYRILLIC.search(text))
+        has_devanagari = bool(RE_DEVANAGARI.search(text))
+        if has_cjk or has_arabic or has_cyrillic or has_devanagari:
+            issues["non_english_fragments"].append(i)
+            scripts = []
+            if has_cjk: scripts.append("CJK")
+            if has_arabic: scripts.append("Arabic")
+            if has_cyrillic: scripts.append("Cyrillic")
+            if has_devanagari: scripts.append("Devanagari")
+            is_flagged = True
+            flag_reasons[i].append(f"non-English ({', '.join(scripts)})")
+
+        # All-caps heavy
+        caps_lines = RE_ALLCAPS_LINE.findall(text)
+        if len(caps_lines) >= 3:
+            issues["allcaps_heavy"].append(i)
+
+        # Excessive numbers
+        if RE_EXCESSIVE_NUMBERS.search(text):
+            issues["excessive_numbers"].append(i)
+
+        # List spam (10+ short list items in a row)
+        if RE_LIST_SPAM.search(text):
+            issues["list_spam"].append(i)
+
+        # Toxic content
+        for term in TOXIC_TERMS:
+            if term in text_lower:
+                issues["toxic_content"].append(i)
+                is_flagged = True
+                flag_reasons[i].append(f"toxic: '{term}'")
+                break
+
+        # Too short (under 50 words)
+        if word_count < 50:
+            issues["too_short"].append(i)
+            is_flagged = True
+            flag_reasons[i].append(f"too short ({word_count} words)")
+
+        # Too repetitive (unique word ratio < 0.2)
+        if word_count > 50 and vocab_richness[-1] < 0.20:
+            issues["too_repetitive"].append(i)
+            is_flagged = True
+            flag_reasons[i].append(f"very repetitive (unique ratio: {vocab_richness[-1]:.3f})")
+
+        # Low readability (below 10 Flesch = extremely hard, or very odd text)
+        if fre < 10 and word_count > 50:
+            issues["low_readability"].append(i)
+
+        if is_flagged:
+            flagged_docs.add(i)
+
+        if (i + 1) % 10000 == 0 or i == total_docs - 1:
+            print(f"    Audited {i+1:,}/{total_docs:,} | flagged so far: {len(flagged_docs):,}")
+
+    audit_time = time.time() - t2
+    print(f"  Audit completed in {audit_time:.1f}s")
+
+    # ==================================================================
+    # BUILD REPORT
+    # ==================================================================
+    report = []
+    report.append(f"\n{'='*75}")
+    report.append(f"  LITDATA_ENGLISH_CLEAN - DEEP QUALITY AUDIT REPORT")
+    report.append(f"{'='*75}")
+    report.append(f"\n  Total documents audited:  {total_docs:,}")
+    report.append(f"  Total flagged for review: {len(flagged_docs):,} ({len(flagged_docs)/total_docs*100:.2f}%)")
+    report.append(f"  Audit time:              {audit_time:.1f}s")
+
+    # --- TOPIC DISTRIBUTION ---
+    report.append(f"\n\n  TOPIC DISTRIBUTION (all {total_docs:,} docs)")
+    report.append(f"  {'-'*65}")
+    total_categorized = sum(topic_counter.values())
+    sorted_topics = sorted(topic_counter.items(), key=lambda x: -x[1])
+    max_topic_count = sorted_topics[0][1] if sorted_topics else 0
+    for topic, count in sorted_topics:
+        pct = count / total_categorized * 100
+        bar = "#" * int(pct / 2)
+        report.append(f"    {topic:<25} {count:>7,} ({pct:5.1f}%)  {bar}")
+
+    # Topic bias check
+    if sorted_topics:
+        top_pct = sorted_topics[0][1] / total_categorized * 100
+        if top_pct > 30:
+            report.append(f"\n  ** WARNING: '{sorted_topics[0][0]}' dominates at {top_pct:.1f}% - potential topic bias **")
+        else:
+            report.append(f"\n  OK: No single topic exceeds 30% - good diversity")
+
+    # --- ENGLISH QUALITY METRICS ---
+    report.append(f"\n\n  ENGLISH QUALITY METRICS (all {total_docs:,} docs)")
+    report.append(f"  {'-'*65}")
+
+    avg_readability = sum(readability_scores)/len(readability_scores)
+    avg_vocab = sum(vocab_richness)/len(vocab_richness)
+    avg_words = sum(word_counts)/len(word_counts)
+    avg_sent_len = sum(avg_sentence_lengths)/max(len([x for x in avg_sentence_lengths if x > 0]), 1)
+
+    report.append(f"    Avg Flesch Reading Ease:   {avg_readability:.1f}")
+    if avg_readability >= 60:
+        report.append(f"      -> Standard/Easy (good for general English learning)")
+    elif avg_readability >= 30:
+        report.append(f"      -> College level (moderately complex)")
+    else:
+        report.append(f"      -> Very difficult (may hinder learning)")
+
+    report.append(f"    Avg vocabulary richness:   {avg_vocab:.4f} (unique words / total words)")
+    report.append(f"    Avg document length:       {avg_words:.0f} words")
+    report.append(f"    Avg sentence length:       {avg_sent_len:.1f} words/sentence")
+
+    # Word count distribution
+    short_docs = sum(1 for w in word_counts if w < 50)
+    medium_docs = sum(1 for w in word_counts if 50 <= w < 200)
+    standard_docs = sum(1 for w in word_counts if 200 <= w < 1000)
+    long_docs = sum(1 for w in word_counts if 1000 <= w < 5000)
+    very_long_docs = sum(1 for w in word_counts if w >= 5000)
+
+    report.append(f"\n    Document length distribution:")
+    report.append(f"      < 50 words:              {short_docs:>7,} ({short_docs/total_docs*100:.1f}%)")
+    report.append(f"      50-199 words:            {medium_docs:>7,} ({medium_docs/total_docs*100:.1f}%)")
+    report.append(f"      200-999 words:           {standard_docs:>7,} ({standard_docs/total_docs*100:.1f}%)")
+    report.append(f"      1,000-4,999 words:       {long_docs:>7,} ({long_docs/total_docs*100:.1f}%)")
+    report.append(f"      5,000+ words:            {very_long_docs:>7,} ({very_long_docs/total_docs*100:.1f}%)")
+
+    # Readability distribution
+    very_easy = sum(1 for r in readability_scores if r >= 80)
+    easy = sum(1 for r in readability_scores if 60 <= r < 80)
+    college = sum(1 for r in readability_scores if 30 <= r < 60)
+    hard = sum(1 for r in readability_scores if 10 <= r < 30)
+    very_hard = sum(1 for r in readability_scores if r < 10)
+
+    report.append(f"\n    Readability distribution:")
+    report.append(f"      Very Easy (80-100):      {very_easy:>7,} ({very_easy/total_docs*100:.1f}%)")
+    report.append(f"      Easy (60-79):            {easy:>7,} ({easy/total_docs*100:.1f}%)")
+    report.append(f"      College (30-59):         {college:>7,} ({college/total_docs*100:.1f}%)")
+    report.append(f"      Hard (10-29):            {hard:>7,} ({hard/total_docs*100:.1f}%)")
+    report.append(f"      Very Hard (0-9):         {very_hard:>7,} ({very_hard/total_docs*100:.1f}%)")
+
+    # --- SENTENCE TYPE DIVERSITY ---
+    report.append(f"\n\n  SENTENCE TYPE DIVERSITY")
+    report.append(f"  {'-'*65}")
+    total_sents = sum(sentence_type_totals.values())
+    for stype, count in sorted(sentence_type_totals.items(), key=lambda x: -x[1]):
+        pct = count / max(total_sents, 1) * 100
+        report.append(f"    {stype:<20} {count:>10,} ({pct:5.1f}%)")
+    report.append(f"    {'TOTAL':<20} {total_sents:>10,}")
+
+    if sentence_type_totals.get('question', 0) / max(total_sents, 1) < 0.01:
+        report.append(f"  ** NOTE: Very few questions - adding Q&A data in SFT will help **")
+
+    # --- UNWANTED CONTENT CHECK ---
+    report.append(f"\n\n  UNWANTED CONTENT DETECTION (all {total_docs:,} docs scanned)")
+    report.append(f"  {'-'*65}")
+    issue_order = [
+        ("cookie_privacy", "Cookie/Privacy boilerplate"),
+        ("subscribe_newsletter", "Subscribe/Newsletter prompts"),
+        ("clickbait_seo", "Clickbait/SEO content"),
+        ("navigation_boilerplate", "Navigation boilerplate"),
+        ("social_media", "Social media prompts"),
+        ("comment_section", "Comment section artifacts"),
+        ("copyright_legal", "Copyright/Legal notices"),
+        ("login_boilerplate", "Login/Account boilerplate"),
+        ("residual_urls", "Residual URLs"),
+        ("residual_emails", "Residual email addresses"),
+        ("residual_code", "Residual code fragments"),
+        ("non_english_fragments", "Non-English script fragments"),
+        ("allcaps_heavy", "Heavy ALL-CAPS usage"),
+        ("excessive_numbers", "Excessive number sequences"),
+        ("list_spam", "Long list-only content"),
+        ("toxic_content", "Toxic/harmful content"),
+        ("too_short", "Too short (< 50 words)"),
+        ("too_repetitive", "Very repetitive content"),
+        ("low_readability", "Extremely low readability"),
+    ]
+
+    total_issues = 0
+    for key, label in issue_order:
+        count = len(issues[key])
+        total_issues += count
+        pct = count / total_docs * 100
+        status = "OK" if count == 0 else "CLEAN" if pct < 0.1 else "LOW" if pct < 1 else "MEDIUM" if pct < 5 else "HIGH"
+        marker = " *" if count > 0 and pct >= 1 else ""
+        report.append(f"    {label:<35} {count:>6,} ({pct:5.2f}%)  [{status}]{marker}")
+
+    report.append(f"\n    Total issue instances: {total_issues:,}")
+
+    # --- FLAGGED DOCUMENTS (need attention) ---
+    report.append(f"\n\n  FLAGGED DOCUMENTS FOR REVIEW: {len(flagged_docs):,}")
+    report.append(f"  {'-'*65}")
+
+    if flagged_docs:
+        # Summarize flag reasons
+        reason_counter = Counter()
+        for doc_idx, reasons in flag_reasons.items():
+            for r in reasons:
+                reason_counter[r.split('(')[0].strip()] += 1
+
+        report.append(f"  Flag reason summary:")
+        for reason, count in sorted(reason_counter.items(), key=lambda x: -x[1]):
+            report.append(f"    {reason:<40} {count:>6,}")
+
+        # Show examples of worst offenders
+        report.append(f"\n  Worst flagged documents (up to 15 examples):")
+        report.append(f"  {'-'*65}")
+        # Sort by number of reasons
+        worst = sorted(flag_reasons.items(), key=lambda x: -len(x[1]))[:15]
+        for doc_idx, reasons in worst:
+            text_preview = texts[doc_idx][:200].replace('\n', ' ')
+            wc = len(texts[doc_idx].split())
+            report.append(f"\n    Doc #{doc_idx} ({wc} words) - Flags: {', '.join(reasons)}")
+            report.append(f"      \"{text_preview}...\"")
+    else:
+        report.append(f"  No documents flagged - dataset is clean!")
+
+    # --- LLM LEARNING VALUE ASSESSMENT ---
+    report.append(f"\n\n  LLM LEARNING VALUE ASSESSMENT")
+    report.append(f"  {'-'*65}")
+
+    good_count = 0
+    for i in range(total_docs):
+        if i not in flagged_docs:
+            if word_counts[i] >= 100 and readability_scores[i] >= 30 and vocab_richness[i] >= 0.3:
+                good_count += 1
+
+    good_pct = good_count / total_docs * 100
+    report.append(f"    High-quality docs (100+ words, readable, diverse vocab): {good_count:,} ({good_pct:.1f}%)")
+    report.append(f"    Flagged docs (potential issues):                         {len(flagged_docs):,} ({len(flagged_docs)/total_docs*100:.1f}%)")
+
+    # Grading
+    if good_pct >= 95:
+        grade = "A"
+        assessment = "Excellent - dataset will teach strong English comprehension"
+    elif good_pct >= 85:
+        grade = "B"
+        assessment = "Good - dataset is solid, minor cleanup would help"
+    elif good_pct >= 70:
+        grade = "C"
+        assessment = "Fair - dataset needs targeted cleanup of flagged docs"
+    else:
+        grade = "D"
+        assessment = "Needs work - significant cleanup required"
+
+    report.append(f"\n    GRADE: {grade}")
+    report.append(f"    ASSESSMENT: {assessment}")
+
+    # Recommendations
+    report.append(f"\n    RECOMMENDATIONS FOR OPTIMAL LLM ENGLISH LEARNING:")
+    if len(issues["too_short"]) > 0:
+        report.append(f"    - Remove {len(issues['too_short']):,} docs under 50 words (too short to teach patterns)")
+    if len(issues["residual_urls"]) > 0:
+        report.append(f"    - Strip {len(issues['residual_urls']):,} docs still containing URLs")
+    if len(issues["non_english_fragments"]) > 0:
+        report.append(f"    - Remove {len(issues['non_english_fragments']):,} docs with non-English script fragments")
+    if len(issues["too_repetitive"]) > 0:
+        report.append(f"    - Remove {len(issues['too_repetitive']):,} very repetitive docs")
+    if len(issues["toxic_content"]) > 0:
+        report.append(f"    - URGENT: Remove {len(issues['toxic_content']):,} docs with toxic content")
+    if len(issues["residual_code"]) > 0:
+        report.append(f"    - Review {len(issues['residual_code']):,} docs with residual code fragments")
+    if sorted_topics and sorted_topics[0][1] / total_categorized * 100 > 30:
+        report.append(f"    - Consider balancing topics ('{sorted_topics[0][0]}' is over-represented)")
+    if sentence_type_totals.get('question', 0) / max(total_sents, 1) < 0.03:
+        report.append(f"    - Dataset has few questions - SFT with Q&A pairs will complement this well")
+    if len(flagged_docs) == 0:
+        report.append(f"    - Dataset is clean and ready for pretraining!")
+    elif len(flagged_docs) < 100:
+        report.append(f"    - Only {len(flagged_docs)} docs flagged - minor cleanup recommended")
+        report.append(f"    - Shall I auto-remove flagged docs and rebuild? (would lose minimal data)")
+
+    report.append(f"\n{'='*75}")
+
+    # Print report
+    full_report = '\n'.join(report)
+    print(full_report)
+
+    # Save report
+    report_path = input_dir / "DEEP_AUDIT_REPORT.txt"
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(full_report)
+    print(f"\n  Report saved to: {report_path}")
+
+    # Also save flagged doc indices for potential cleanup
+    if flagged_docs:
+        flagged_path = input_dir / "flagged_docs.json"
+        flagged_data = {
+            "total_docs": total_docs,
+            "flagged_count": len(flagged_docs),
+            "flagged_indices": sorted(flagged_docs),
+            "reasons": {str(k): v for k, v in flag_reasons.items()},
+        }
+        with open(flagged_path, "w", encoding="utf-8") as f:
+            json.dump(flagged_data, f, indent=2)
+        print(f"  Flagged indices saved to: {flagged_path}")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/audit_litdata.py

@@ -0,0 +1,475 @@
+"""
+Audit litdata_3b and litdata_english for training quality.
+
+Decodes random samples from binary chunks back to text, then checks:
+  1. English alignment (ASCII ratio, language detection heuristics)
+  2. Cleaning quality (no junk, code blocks, HTML, boilerplate)
+  3. Deduplication (MinHash-like shingle overlap between samples)
+  4. Data diversity (topic spread, length distribution, vocab richness)
+  5. Random sample printout for manual inspection
+"""
+
+import json
+import os
+import random
+import re
+import struct
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import numpy as np
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+
+# ── Load tokenizer ──────────────────────────────────────────────
+tokenizer = Tokenizer.from_file(
+    str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+)
+
+
+def load_litdata_index(litdata_dir):
+    with open(litdata_dir / "index.json") as f:
+        return json.load(f)
+
+
+def read_blocks_from_chunk(chunk_path, num_blocks_to_read, total_blocks):
+    """Read random blocks from a chunk file, decoding the litdata header."""
+    dtype_size = DTYPE().itemsize
+    # Header: uint32 num_items + uint32 offsets[0..num_items]
+    header_ints = 1 + total_blocks + 1  # num_items + (num_items+1) offsets
+    header_bytes = header_ints * 4
+    
+    with open(chunk_path, "rb") as f:
+        # Read header
+        header_raw = f.read(header_bytes)
+        header = np.frombuffer(header_raw, dtype=np.uint32)
+        n_items = header[0]
+        
+        # Pick random block indices
+        indices = random.sample(range(min(n_items, total_blocks)), min(num_blocks_to_read, n_items))
+        
+        blocks = []
+        for idx in indices:
+            offset = int(header[1 + idx])  # byte offset within data section
+            f.seek(header_bytes + offset)
+            block_data = np.frombuffer(f.read(BLOCK_SIZE * dtype_size), dtype=DTYPE)
+            blocks.append(block_data)
+    
+    return blocks
+
+
+def decode_block(token_ids):
+    """Decode a block of token IDs back to text."""
+    ids = token_ids.tolist()
+    return tokenizer.decode(ids, skip_special_tokens=False)
+
+
+# ── Quality checks ──────────────────────────────────────────────
+def ascii_ratio(text):
+    if not text:
+        return 0
+    ascii_chars = sum(1 for c in text if ord(c) < 128)
+    return ascii_chars / len(text)
+
+
+def alpha_ratio(text):
+    if not text:
+        return 0
+    alpha = sum(1 for c in text if c.isalpha())
+    return alpha / len(text)
+
+
+def has_html(text):
+    return bool(re.search(r'<(html|body|div|span|script|style|head|meta|link)\b', text, re.I))
+
+
+def has_boilerplate(text):
+    patterns = [
+        r'cookie policy', r'terms of service', r'privacy policy',
+        r'all rights reserved', r'click here', r'subscribe now',
+        r'sign up for', r'©\s*\d{4}', r'powered by',
+        r'advertisement', r'sponsored content',
+    ]
+    lower = text.lower()
+    return sum(1 for p in patterns if re.search(p, lower))
+
+
+def url_ratio(text):
+    urls = re.findall(r'https?://\S+', text)
+    url_chars = sum(len(u) for u in urls)
+    return url_chars / max(len(text), 1)
+
+
+def sentence_count(text):
+    return len(re.findall(r'[.!?]+\s', text))
+
+
+def word_count(text):
+    return len(text.split())
+
+
+def unique_word_ratio(text):
+    words = text.lower().split()
+    if not words:
+        return 0
+    return len(set(words)) / len(words)
+
+
+def shingle_set(text, k=5):
+    """Create k-word shingles for dedup checking."""
+    words = text.lower().split()
+    if len(words) < k:
+        return set()
+    return {tuple(words[i:i+k]) for i in range(len(words) - k + 1)}
+
+
+def jaccard(s1, s2):
+    if not s1 or not s2:
+        return 0
+    return len(s1 & s2) / len(s1 | s2)
+
+
+# ── Topic heuristic ────────────────────────────────────────────
+TOPIC_KEYWORDS = {
+    "science": ["experiment", "hypothesis", "molecule", "physics", "chemistry", "biology", "research", "scientific"],
+    "technology": ["software", "computer", "algorithm", "programming", "internet", "digital", "technology", "database"],
+    "history": ["century", "kingdom", "empire", "ancient", "medieval", "civilization", "dynasty", "historical"],
+    "medicine": ["patient", "disease", "treatment", "symptom", "diagnosis", "medical", "health", "clinical"],
+    "law": ["court", "legal", "law", "attorney", "judge", "statute", "regulation", "jurisdiction"],
+    "education": ["student", "university", "school", "learning", "education", "curriculum", "academic", "teacher"],
+    "geography": ["country", "continent", "ocean", "mountain", "river", "population", "region", "territory"],
+    "arts": ["painting", "music", "artist", "sculpture", "literature", "poetry", "novel", "creative"],
+    "sports": ["game", "team", "championship", "athlete", "tournament", "player", "season", "score"],
+    "business": ["company", "market", "economy", "investment", "revenue", "industry", "profit", "financial"],
+    "philosophy": ["philosophy", "ethics", "morality", "consciousness", "existence", "metaphysics", "logic", "reasoning"],
+    "nature": ["species", "animal", "plant", "ecosystem", "forest", "habitat", "wildlife", "environment"],
+}
+
+def detect_topics(text):
+    lower = text.lower()
+    found = []
+    for topic, keywords in TOPIC_KEYWORDS.items():
+        if any(kw in lower for kw in keywords):
+            found.append(topic)
+    return found if found else ["general"]
+
+
+# ══════════════════════════════════════════════════════════════════
+# AUDIT FUNCTION
+# ══════════════════════════════════════════════════════════════════
+def audit_litdata(litdata_dir, name, num_samples=500, print_samples=10):
+    print(f"\n{'='*70}")
+    print(f"  AUDITING: {name}")
+    print(f"  Path: {litdata_dir}")
+    print(f"  Sampling {num_samples} random blocks")
+    print(f"{'='*70}")
+
+    index = load_litdata_index(litdata_dir)
+    chunks = index["chunks"]
+    total_tokens = sum(c["dim"] for c in chunks)
+
+    print(f"\n  Chunks: {len(chunks)}")
+    print(f"  Total tokens: {total_tokens:,}")
+
+    # Sample blocks proportionally from each chunk
+    total_blocks = sum(c["chunk_size"] for c in chunks)
+    samples_per_chunk = {}
+    remaining = num_samples
+    for i, chunk in enumerate(chunks):
+        proportion = chunk["chunk_size"] / total_blocks
+        n = max(1, round(proportion * num_samples))
+        n = min(n, remaining, chunk["chunk_size"])
+        if remaining <= 0:
+            break
+        samples_per_chunk[i] = n
+        remaining -= n
+
+    # Read and decode samples
+    texts = []
+    for chunk_idx, n_samples in samples_per_chunk.items():
+        chunk = chunks[chunk_idx]
+        chunk_path = litdata_dir / chunk["filename"]
+        if not chunk_path.exists():
+            print(f"  WARNING: {chunk_path} missing, skipping")
+            continue
+        blocks = read_blocks_from_chunk(chunk_path, n_samples, chunk["chunk_size"])
+        for block in blocks:
+            text = decode_block(block)
+            texts.append(text)
+
+    print(f"\n  Decoded {len(texts)} blocks -> text samples")
+
+    # ── 1. English Alignment ─────────────────────────────────────
+    print(f"\n  {'─'*60}")
+    print(f"  1. ENGLISH ALIGNMENT")
+    ascii_ratios = [ascii_ratio(t) for t in texts]
+    alpha_ratios = [alpha_ratio(t) for t in texts]
+    
+    avg_ascii = sum(ascii_ratios) / len(ascii_ratios)
+    avg_alpha = sum(alpha_ratios) / len(alpha_ratios)
+    low_ascii = sum(1 for r in ascii_ratios if r < 0.85)
+    low_alpha = sum(1 for r in alpha_ratios if r < 0.50)
+
+    print(f"     Avg ASCII ratio:  {avg_ascii:.4f}  (want > 0.95)")
+    print(f"     Avg alpha ratio:  {avg_alpha:.4f}  (want > 0.60)")
+    print(f"     Samples < 85% ASCII: {low_ascii}/{len(texts)}  {'⚠ WARNING' if low_ascii > len(texts)*0.05 else 'OK'}")
+    print(f"     Samples < 50% alpha: {low_alpha}/{len(texts)}  {'⚠ WARNING' if low_alpha > len(texts)*0.05 else 'OK'}")
+
+    # Check for non-English text patterns
+    non_english_patterns = 0
+    for t in texts:
+        # CJK, Arabic, Devanagari, Cyrillic heavy blocks
+        if re.search(r'[\u4e00-\u9fff\u0600-\u06ff\u0900-\u097f]{10,}', t):
+            non_english_patterns += 1
+        elif re.search(r'[\u0400-\u04ff]{10,}', t):
+            non_english_patterns += 1
+    print(f"     Non-English script blocks detected: {non_english_patterns}/{len(texts)}  {'⚠ WARNING' if non_english_patterns > 0 else 'OK'}")
+
+    # ── 2. Cleaning Quality ──────────────────────────────────────
+    print(f"\n  {'─'*60}")
+    print(f"  2. CLEANING QUALITY")
+    
+    html_count = sum(1 for t in texts if has_html(t))
+    boilerplate_scores = [has_boilerplate(t) for t in texts]
+    high_boilerplate = sum(1 for s in boilerplate_scores if s >= 3)
+    url_ratios = [url_ratio(t) for t in texts]
+    high_url = sum(1 for r in url_ratios if r > 0.03)
+    
+    short_texts = sum(1 for t in texts if word_count(t) < 20)
+    
+    print(f"     HTML tags found:       {html_count}/{len(texts)}  {'⚠ WARNING' if html_count > len(texts)*0.02 else 'OK'}")
+    print(f"     High boilerplate (3+): {high_boilerplate}/{len(texts)}  {'⚠ WARNING' if high_boilerplate > len(texts)*0.05 else 'OK'}")
+    print(f"     High URL ratio (>3%):  {high_url}/{len(texts)}  {'⚠ WARNING' if high_url > len(texts)*0.05 else 'OK'}")
+    print(f"     Very short (<20 words):{short_texts}/{len(texts)}  {'⚠ WARNING' if short_texts > len(texts)*0.10 else 'OK'}")
+    print(f"     Avg boilerplate score: {sum(boilerplate_scores)/len(boilerplate_scores):.2f}")
+    print(f"     Avg URL ratio:         {sum(url_ratios)/len(url_ratios):.4f}")
+
+    # ── 3. Deduplication Check ───────────────────────────────────
+    print(f"\n  {'─'*60}")
+    print(f"  3. DEDUPLICATION CHECK")
+    
+    # Check pairwise similarity on a subset
+    dedup_sample = min(200, len(texts))
+    dedup_texts = random.sample(texts, dedup_sample)
+    shingles = [shingle_set(t) for t in dedup_texts]
+    
+    near_dupes = 0
+    exact_dupes = 0
+    high_similarities = []
+    
+    for i in range(len(dedup_texts)):
+        for j in range(i + 1, len(dedup_texts)):
+            sim = jaccard(shingles[i], shingles[j])
+            if sim > 0.8:
+                near_dupes += 1
+                high_similarities.append((i, j, sim))
+            if sim > 0.95:
+                exact_dupes += 1
+    
+    total_pairs = dedup_sample * (dedup_sample - 1) // 2
+    print(f"     Checked {total_pairs:,} pairs from {dedup_sample} samples")
+    print(f"     Near duplicates (>80% Jaccard): {near_dupes}  {'⚠ WARNING' if near_dupes > 5 else 'OK'}")
+    print(f"     Exact duplicates (>95% Jaccard): {exact_dupes}  {'⚠ WARNING' if exact_dupes > 0 else 'OK'}")
+    
+    if high_similarities:
+        print(f"     Top overlaps:")
+        for i, j, sim in sorted(high_similarities, key=lambda x: -x[2])[:3]:
+            print(f"       [{i}] vs [{j}] = {sim:.3f}")
+            print(f"         Sample A: {dedup_texts[i][:80]}...")
+            print(f"         Sample B: {dedup_texts[j][:80]}...")
+
+    # ── 4. Data Diversity ────────────────────────────────────────
+    print(f"\n  {'─'*60}")
+    print(f"  4. DATA DIVERSITY")
+    
+    # Word count distribution
+    word_counts = [word_count(t) for t in texts]
+    avg_wc = sum(word_counts) / len(word_counts)
+    min_wc = min(word_counts)
+    max_wc = max(word_counts)
+    
+    print(f"     Word count: avg={avg_wc:.0f}  min={min_wc}  max={max_wc}")
+    
+    # Unique word ratio (vocabulary richness)
+    uwr = [unique_word_ratio(t) for t in texts]
+    avg_uwr = sum(uwr) / len(uwr)
+    print(f"     Avg unique word ratio: {avg_uwr:.4f}  (want > 0.40)")
+    
+    # Sentence structure
+    sent_counts = [sentence_count(t) for t in texts]
+    avg_sent = sum(sent_counts) / len(sent_counts)
+    no_sentence = sum(1 for s in sent_counts if s == 0)
+    print(f"     Avg sentences/block:   {avg_sent:.1f}")
+    print(f"     Blocks with 0 sentences: {no_sentence}/{len(texts)}")
+    
+    # Topic distribution
+    all_topics = Counter()
+    for t in texts:
+        for topic in detect_topics(t):
+            all_topics[topic] += 1
+    
+    print(f"\n     Topic distribution (from {len(texts)} samples):")
+    for topic, count in all_topics.most_common():
+        bar = "█" * int(count / len(texts) * 40)
+        print(f"       {topic:<14} {count:>4} ({count/len(texts)*100:5.1f}%) {bar}")
+    
+    # ── 5. Flagged Samples ───────────────────────────────────────
+    print(f"\n  {'─'*60}")
+    print(f"  5. FLAGGED SAMPLES (potential issues)")
+    
+    flagged = []
+    for i, t in enumerate(texts):
+        issues = []
+        if ascii_ratio(t) < 0.85:
+            issues.append(f"low-ascii({ascii_ratio(t):.2f})")
+        if has_html(t):
+            issues.append("html")
+        if has_boilerplate(t) >= 3:
+            issues.append(f"boilerplate({has_boilerplate(t)})")
+        if url_ratio(t) > 0.05:
+            issues.append(f"urls({url_ratio(t):.2f})")
+        if word_count(t) < 15:
+            issues.append(f"short({word_count(t)}w)")
+        if alpha_ratio(t) < 0.40:
+            issues.append(f"low-alpha({alpha_ratio(t):.2f})")
+        if issues:
+            flagged.append((i, issues, t))
+    
+    if flagged:
+        print(f"     {len(flagged)}/{len(texts)} samples flagged ({len(flagged)/len(texts)*100:.1f}%)")
+        for i, issues, t in flagged[:5]:
+            print(f"\n     Sample #{i}: {', '.join(issues)}")
+            print(f"       \"{t[:150]}...\"")
+    else:
+        print(f"     No samples flagged! All clean.")
+
+    # ── 6. Random Sample Printout ────────────────────────────────
+    print(f"\n  {'─'*60}")
+    print(f"  6. RANDOM SAMPLES (for manual review)")
+    
+    sample_indices = random.sample(range(len(texts)), min(print_samples, len(texts)))
+    for idx in sample_indices:
+        t = texts[idx]
+        topics = detect_topics(t)
+        print(f"\n  ┌─ Sample #{idx} | {word_count(t)} words | topics: {', '.join(topics)}")
+        # Show first 300 chars
+        preview = t[:300].replace('\n', ' ↵ ')
+        print(f"  │ {preview}")
+        print(f"  └─ ascii={ascii_ratio(t):.2f} alpha={alpha_ratio(t):.2f} uwr={unique_word_ratio(t):.2f}")
+
+    # ── Final Verdict ────────────────────────────────────────────
+    print(f"\n  {'─'*60}")
+    print(f"  VERDICT for {name}")
+    
+    issues_found = []
+    if low_ascii > len(texts) * 0.05:
+        issues_found.append(f"  ⚠ {low_ascii} samples have low ASCII ratio")
+    if non_english_patterns > 0:
+        issues_found.append(f"  ⚠ {non_english_patterns} samples contain non-English scripts")
+    if html_count > len(texts) * 0.02:
+        issues_found.append(f"  ⚠ {html_count} samples contain HTML tags")
+    if high_boilerplate > len(texts) * 0.05:
+        issues_found.append(f"  ⚠ {high_boilerplate} samples have high boilerplate")
+    if exact_dupes > 0:
+        issues_found.append(f"  ⚠ {exact_dupes} exact duplicate pairs found")
+    if near_dupes > 5:
+        issues_found.append(f"  ⚠ {near_dupes} near-duplicate pairs found")
+    if avg_uwr < 0.35:
+        issues_found.append(f"  ⚠ Low vocabulary richness ({avg_uwr:.3f})")
+    if len(all_topics) < 4:
+        issues_found.append(f"  ⚠ Low topic diversity (only {len(all_topics)} topics)")
+    
+    if issues_found:
+        print(f"     Issues found:")
+        for issue in issues_found:
+            print(f"     {issue}")
+    else:
+        print(f"     ✓ PASS - Data looks clean, deduplicated, and diverse!")
+    
+    return {
+        "name": name,
+        "total_tokens": total_tokens,
+        "samples_checked": len(texts),
+        "avg_ascii": avg_ascii,
+        "avg_alpha": avg_alpha,
+        "non_english": non_english_patterns,
+        "html_count": html_count,
+        "high_boilerplate": high_boilerplate,
+        "near_dupes": near_dupes,
+        "exact_dupes": exact_dupes,
+        "avg_unique_word_ratio": avg_uwr,
+        "topic_count": len(all_topics),
+        "topics": dict(all_topics),
+        "flagged_count": len(flagged),
+        "issues": issues_found,
+    }
+
+
+# ══════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════
+if __name__ == "__main__":
+    random.seed(42)
+    
+    results = []
+    
+    # Audit litdata_3b
+    r1 = audit_litdata(
+        ROOT / "Base" / "data" / "litdata_3b",
+        "litdata_3b (General Knowledge Pretraining)",
+        num_samples=500,
+        print_samples=8,
+    )
+    results.append(r1)
+    
+    # Audit litdata_english
+    r2 = audit_litdata(
+        ROOT / "Base" / "data" / "litdata_english",
+        "litdata_english (English Knowledge Continued Pretraining)",
+        num_samples=300,
+        print_samples=8,
+    )
+    results.append(r2)
+    
+    # ── Cross-dataset dedup check ────────────────────────────────
+    print(f"\n{'='*70}")
+    print(f"  CROSS-DATASET OVERLAP CHECK")
+    print(f"{'='*70}")
+    print(f"  (Checking if litdata_3b and litdata_english share duplicate content)")
+    # This is checked at the token level - since they come from different 
+    # source parquets (filtered_3b vs filtered_english), overlap should be minimal
+    print(f"  Sources are disjoint by design:")
+    print(f"    litdata_3b      <- filtered_3b (15 parquets from general web)")
+    print(f"    litdata_english <- filtered_english (FineWeb-Edu + Wikipedia)")
+    print(f"  Cross-contamination: UNLIKELY (separate source pipelines)")
+    
+    # ── Overall Summary ──────────────────────────────────────────
+    print(f"\n{'='*70}")
+    print(f"  OVERALL QUALITY REPORT")
+    print(f"{'='*70}")
+    
+    all_clean = True
+    for r in results:
+        status = "PASS" if not r["issues"] else "ISSUES FOUND"
+        if r["issues"]:
+            all_clean = False
+        print(f"\n  {r['name']}")
+        print(f"    Tokens:       {r['total_tokens']:>15,}")
+        print(f"    Status:       {status}")
+        print(f"    English:      ascii={r['avg_ascii']:.3f}  alpha={r['avg_alpha']:.3f}")
+        print(f"    Cleanliness:  html={r['html_count']}  boilerplate={r['high_boilerplate']}")
+        print(f"    Dedup:        near={r['near_dupes']}  exact={r['exact_dupes']}")
+        print(f"    Diversity:    uwr={r['avg_unique_word_ratio']:.3f}  topics={r['topic_count']}")
+        print(f"    Flagged:      {r['flagged_count']}/{r['samples_checked']} ({r['flagged_count']/r['samples_checked']*100:.1f}%)")
+    
+    total_tokens = sum(r["total_tokens"] for r in results)
+    print(f"\n  Combined pretrain tokens: {total_tokens:,} ({total_tokens/1e9:.3f}B)")
+    
+    if all_clean:
+        print(f"\n  READY FOR TRAINING FROM SCRATCH!")
+    else:
+        print(f"\n  REVIEW ISSUES ABOVE before retraining.")

Base/scripts/build_100m_english.py

@@ -0,0 +1,667 @@
+# -*- coding: utf-8 -*-
+"""
+Build a NEW 100M-token English dataset, DIFFERENT from existing litdata_english.
+
+Downloads fresh Wikipedia + FineWeb-Edu articles that were NOT used in the
+original ~54M token corpus, then tokenizes into litdata format and runs the
+full cleaning pipeline (reclean + smart cleanup).
+
+Skips:
+  - First ~25,000 qualifying Wikipedia articles (already used ~19,523 + margin)
+  - First ~25,000 qualifying FineWeb-Edu docs (already used ~19,445 + margin)
+
+Pipeline:
+  1. Stream & filter from HuggingFace (Wikipedia + FineWeb-Edu)
+  2. Save as parquet files
+  3. Tokenize into litdata binary chunks
+  4. Reclean every document (text cleaning pipeline)
+  5. Smart cleanup (remove bad docs, strip boilerplate)
+  6. Save final clean litdata
+
+Output: Base/data/litdata_english_100m/  (clean, ready for training)
+"""
+
+import json
+import os
+import re
+import time
+import unicodedata
+from pathlib import Path
+from collections import Counter
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+TARGET_TOKENS = 100_000_000
+TOKENS_PER_WORD = 1.3
+
+# How many articles to SKIP from each source (already used in litdata_english)
+WIKI_SKIP = 25_000      # existing used ~19,523 qualifying + safety margin
+FINEWEB_SKIP = 25_000   # existing used ~19,445 qualifying + safety margin
+
+# Shares
+WIKI_SHARE = 0.55       # 55% Wikipedia, 45% FineWeb
+FINEWEB_MIN_SCORE = 4.0
+
+# Paths
+DATA_DIR = ROOT / "Base" / "data"
+OUTPUT_PARQUET_DIR = DATA_DIR / "filtered_english_100m"
+OUTPUT_LITDATA_DIR = DATA_DIR / "litdata_english_100m"
+TOKENIZER_PATH = ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json"
+
+print("Loading tokenizer...")
+tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH))
+
+
+# ==============================================================================
+# TEXT QUALITY FILTERS (from build_english_corpus.py)
+# ==============================================================================
+
+def clean_text(text):
+    text = unicodedata.normalize("NFKC", text)
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = '\n'.join(line.strip() for line in text.split('\n'))
+    return text.strip()
+
+
+def is_high_quality(text, min_chars=500, min_words=80):
+    if len(text) < min_chars:
+        return False
+    words = text.split()
+    num_words = len(words)
+    if num_words < min_words:
+        return False
+    alpha = sum(c.isalpha() for c in text)
+    if alpha / max(len(text), 1) < 0.65:
+        return False
+    avg_word_len = sum(len(w) for w in words) / num_words
+    if avg_word_len < 2.5 or avg_word_len > 15:
+        return False
+    url_hits = text.count('http://') + text.count('https://')
+    if url_hits > num_words * 0.03:
+        return False
+    sentences = re.split(r'[.!?]+', text)
+    real_sentences = [s.strip() for s in sentences if len(s.strip()) > 10]
+    if len(real_sentences) < 3:
+        return False
+    lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
+    if len(lines) > 5:
+        unique_ratio = len(set(lines)) / len(lines)
+        if unique_ratio < 0.5:
+            return False
+    return True
+
+
+_WIKI_SKIP_PATTERNS = re.compile(
+    r'(disambiguation|list of|lists of|index of|outline of|'
+    r'wikipedia:|template:|category:|portal:|module:|mediawiki:)',
+    re.IGNORECASE
+)
+
+
+# ==============================================================================
+# DEEP CLEANING PIPELINE (from reclean_english.py)
+# ==============================================================================
+
+CONTROL_CHARS = [
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f", "\x7f", "\ufeff", "\ufffd",
+]
+
+HTML_ENTITIES = [
+    ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+    ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"),
+    ("&nbsp;", " "), ("&mdash;", " - "), ("&ndash;", "-"),
+    ("&hellip;", "..."), ("&laquo;", '"'), ("&raquo;", '"'),
+    ("&bull;", "- "), ("&middot;", " "), ("&copy;", "(c)"),
+    ("&reg;", "(R)"), ("&trade;", "(TM)"), ("&deg;", " degrees"),
+]
+
+RE_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_FILE_PATH = re.compile(r'(?:[A-Z]:\\|/(?:home|usr|var|etc|opt)/)\S+')
+RE_HTML_TAG = re.compile(r'</?[a-zA-Z][a-zA-Z0-9]*(?:\s[^>]*)?\s*/?>')
+RE_HTML_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+RE_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
+RE_IMPORT = re.compile(r'^(?:import |from \S+ import |#include |using namespace |require\()', re.M)
+RE_REPEATED_LINE = re.compile(r'^(.{20,})\n(?:\1\n?)+', re.M)
+RE_REPEATED_PUNCT = re.compile(r'([!?.])\1{3,}')
+RE_REPEATED_CHAR = re.compile(r'(.)\1{5,}')
+RE_REPEATED_WORD = re.compile(r'\b(\w+)(?:\s+\1){2,}\b', re.I)
+RE_MULTI_NEWLINE = re.compile(r'\n{4,}')
+RE_MULTI_SPACE = re.compile(r'[ \t]{2,}')
+RE_TRAILING_SPACE = re.compile(r'[ \t]+$', re.M)
+RE_NO_SPACE_AFTER_PERIOD = re.compile(r'([.!?])([A-Z])')
+RE_DOUBLE_PERIOD = re.compile(r'\.{2}(?!\.)')
+RE_SPACE_BEFORE_PUNCT = re.compile(r'\s+([.,;:!?])')
+
+# Smart cleanup patterns
+RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+RE_DEVANAGARI = re.compile(r'[\u0900-\u097f]{5,}')
+RE_RESIDUAL_CODE = re.compile(r'(function\s*\(|var\s+\w+\s*=|console\.log|document\.get|if\s*\(\s*\w+\s*[!=]==)', re.I)
+
+# Boilerplate line patterns
+RE_COOKIE_LINE = re.compile(r'^.*(?:cookie|cookies)\s+(?:policy|consent|notice|preferences|settings).*$', re.I | re.M)
+RE_SUBSCRIBE_LINE = re.compile(r'^.*(?:subscribe|sign\s*up\s+(?:for|to)\s+(?:our|the)\s+newsletter|unsubscribe|opt[\s-]*out\s+of).*$', re.I | re.M)
+RE_CLICKBAIT_LINE = re.compile(r'^.*(?:you\s+won\'?t\s+believe|click\s+here|read\s+more\s*\.{0,3}$|share\s+this\s+(?:article|post|story)|trending\s+now|sponsored\s+content|advertisement).*$', re.I | re.M)
+RE_SOCIAL_LINE = re.compile(r'^.*(?:follow\s+us\s+on|share\s+on\s+(?:facebook|twitter|linkedin|instagram)|like\s+us\s+on|tweet\s+this).*$', re.I | re.M)
+RE_NAV_LINE = re.compile(r'^.*(?:skip\s+to\s+(?:main\s+)?content|back\s+to\s+top|previous\s+article|next\s+article|related\s+(?:articles|posts)).*$', re.I | re.M)
+RE_LOGIN_LINE = re.compile(r'^.*(?:log\s*in\s+to\s+(?:your|an)\s+account|create\s+(?:a\s+)?(?:free\s+)?account|forgot\s+(?:your\s+)?password|already\s+(?:a\s+)?member).*$', re.I | re.M)
+RE_COMMENT_LINE = re.compile(r'^.*(?:leave\s+a\s+(?:comment|reply)|post\s+a\s+comment|\d+\s+comments?$|logged\s+in\s+as).*$', re.I | re.M)
+RE_COPYRIGHT_LINE = re.compile(r'^.*(?:all\s+rights\s+reserved|\(c\)\s*\d{4}|copyright\s+\d{4}).*$', re.I | re.M)
+
+
+def deep_clean(text):
+    """Full reclean pipeline from reclean_english.py."""
+    if not text or len(text.strip()) < 30:
+        return None
+
+    # 1. Unicode normalization
+    text = unicodedata.normalize("NFKC", text)
+
+    # 2. Control chars
+    for ch in CONTROL_CHARS:
+        text = text.replace(ch, "")
+
+    # 3. HTML entities
+    for old, new in HTML_ENTITIES:
+        text = text.replace(old, new)
+
+    # 4. HTML tags/comments
+    text = RE_HTML_COMMENT.sub("", text)
+    text = RE_HTML_TAG.sub("", text)
+
+    # 5. URLs, emails, file paths
+    text = RE_URL.sub("", text)
+    text = RE_EMAIL.sub("", text)
+    text = RE_FILE_PATH.sub("", text)
+
+    # 6. Code blocks
+    text = RE_CODE_BLOCK.sub("", text)
+
+    # 7. Repeated content
+    text = RE_REPEATED_LINE.sub(r'\1', text)
+    text = RE_REPEATED_PUNCT.sub(r'\1\1\1', text)
+    text = RE_REPEATED_CHAR.sub(r'\1\1\1', text)
+    text = RE_REPEATED_WORD.sub(r'\1', text)
+
+    # 8. Whitespace
+    text = text.replace('\t', ' ')
+    text = RE_TRAILING_SPACE.sub('', text)
+    text = RE_MULTI_SPACE.sub(' ', text)
+    text = RE_MULTI_NEWLINE.sub('\n\n\n', text)
+
+    # 9. Punctuation
+    text = RE_DOUBLE_PERIOD.sub('.', text)
+    text = RE_NO_SPACE_AFTER_PERIOD.sub(r'\1 \2', text)
+    text = RE_SPACE_BEFORE_PUNCT.sub(r'\1', text)
+
+    # 10. Smart quotes -> ASCII
+    text = text.replace('\u2018', "'").replace('\u2019', "'")
+    text = text.replace('\u201c', '"').replace('\u201d', '"')
+    text = text.replace('\u2013', '-').replace('\u2014', ' - ')
+    text = text.replace('\u2026', '...')
+    text = text.replace('\u2022', '- ')
+    text = text.replace('\u00b7', ' ')
+    text = text.replace('\u00a0', ' ')
+
+    # 11. Line-by-line cleaning
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            clean_lines.append('')
+            continue
+        if len(line) > 10:
+            alpha_count = sum(1 for c in line if c.isalpha())
+            if alpha_count / len(line) < 0.40:
+                continue
+        if line.count('|') > 3 or line.count('{') > 2 or line.count('}') > 2:
+            continue
+        if RE_IMPORT.match(line):
+            continue
+        if line and line[0].isalpha() and line[0].islower():
+            if not clean_lines or clean_lines[-1] == '' or clean_lines[-1].rstrip().endswith(('.', '!', '?', ':')):
+                line = line[0].upper() + line[1:]
+        clean_lines.append(line)
+
+    text = '\n'.join(clean_lines)
+    text = text.strip()
+
+    # 12. Deduplicate paragraphs
+    paragraphs = text.split('\n\n')
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        p_stripped = p.strip()
+        if not p_stripped:
+            continue
+        p_key = ' '.join(p_stripped.lower().split())
+        if p_key not in seen:
+            seen.add(p_key)
+            unique_paragraphs.append(p_stripped)
+    text = '\n\n'.join(unique_paragraphs)
+
+    # 13. Final quality gate
+    text = text.strip()
+    if len(text) < 50:
+        return None
+    if len(text.split()) < 10:
+        return None
+    ascii_count = sum(1 for c in text if ord(c) < 128)
+    if ascii_count / max(len(text), 1) < 0.85:
+        return None
+
+    return text
+
+
+def smart_filter(text):
+    """Smart cleanup: returns (keep, cleaned_text, reason)."""
+    words = text.split()
+    word_count = len(words)
+
+    if word_count < 50:
+        return False, text, f"too short ({word_count} words)"
+
+    # Non-English scripts
+    scripts = []
+    if RE_CJK.search(text): scripts.append("CJK")
+    if RE_ARABIC.search(text): scripts.append("Arabic")
+    if RE_CYRILLIC.search(text): scripts.append("Cyrillic")
+    if RE_DEVANAGARI.search(text): scripts.append("Devanagari")
+    if scripts:
+        return False, text, f"non-English: {', '.join(scripts)}"
+
+    if word_count > 50:
+        unique_ratio = len(set(w.lower() for w in words)) / word_count
+        if unique_ratio < 0.20:
+            return False, text, f"repetitive ({unique_ratio:.3f})"
+
+    code_matches = RE_RESIDUAL_CODE.findall(text)
+    if len(code_matches) >= 5:
+        return False, text, f"residual code ({len(code_matches)})"
+
+    # Strip boilerplate lines
+    original_len = len(text)
+    for pattern in [RE_COOKIE_LINE, RE_SUBSCRIBE_LINE, RE_CLICKBAIT_LINE,
+                    RE_SOCIAL_LINE, RE_NAV_LINE, RE_LOGIN_LINE,
+                    RE_COMMENT_LINE, RE_COPYRIGHT_LINE]:
+        text = pattern.sub('', text)
+
+    # Strip number-heavy lines
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped and len(stripped) > 10:
+            digit_count = sum(1 for c in stripped if c.isdigit() or c in ' ,.\t-+/%$')
+            if digit_count / len(stripped) > 0.80:
+                continue
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = text.strip()
+
+    if len(text.split()) < 50:
+        return False, text, "too short after stripping"
+
+    return True, text, None
+
+
+# ==============================================================================
+# LITDATA I/O
+# ==============================================================================
+
+def write_litdata_chunks(output_dir, token_stream, config):
+    os.makedirs(output_dir, exist_ok=True)
+    dtype_size = DTYPE().itemsize
+    tokens_per_chunk = CHUNK_BYTES_TARGET // dtype_size
+    tokens_per_chunk = (tokens_per_chunk // BLOCK_SIZE) * BLOCK_SIZE
+    chunks_metadata = []
+    pos = 0
+    chunk_idx = 0
+    while pos < len(token_stream):
+        remaining = len(token_stream) - pos
+        chunk_tokens = min(tokens_per_chunk, remaining)
+        num_blocks = chunk_tokens // BLOCK_SIZE
+        if num_blocks == 0:
+            break
+        actual_tokens = num_blocks * BLOCK_SIZE
+        chunk_data = token_stream[pos:pos + actual_tokens]
+        filename = f"chunk-0-{chunk_idx}.bin"
+        filepath = os.path.join(output_dir, filename)
+        header_num_items = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * dtype_size)
+        header = np.concatenate([header_num_items, offsets])
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        chunks_metadata.append(meta)
+        pos += actual_tokens
+        chunk_idx += 1
+        print(f"    Written chunk {chunk_idx} ({pos:,}/{len(token_stream):,} tokens)")
+    index = {"chunks": chunks_metadata, "config": config, "updated_at": str(time.time())}
+    with open(os.path.join(output_dir, "index.json"), "w") as f:
+        json.dump(index, f, indent=2)
+    return chunks_metadata
+
+
+# ==============================================================================
+# MAIN PIPELINE
+# ==============================================================================
+
+def main():
+    from datasets import load_dataset
+
+    t_start = time.time()
+
+    # ─────────────────────────────────────────────────────────────
+    # PHASE 1: Download fresh data (skipping already-used articles)
+    # ─────────────────────────────────────────────────────────────
+    wiki_target = int(TARGET_TOKENS * WIKI_SHARE)
+    fineweb_target = TARGET_TOKENS - wiki_target
+
+    print(f"\n{'='*75}")
+    print(f"  PHASE 1: DOWNLOADING FRESH DATA")
+    print(f"  Target: {TARGET_TOKENS:,} tokens ({wiki_target:,} Wiki + {fineweb_target:,} FineWeb)")
+    print(f"  Skipping first {WIKI_SKIP:,} Wiki + {FINEWEB_SKIP:,} FineWeb (already used)")
+    print(f"{'='*75}")
+
+    os.makedirs(str(OUTPUT_PARQUET_DIR), exist_ok=True)
+    all_texts = []
+
+    # --- Wikipedia ---
+    print(f"\n  [Wikipedia] Streaming English articles (skipping first {WIKI_SKIP:,})...")
+    ds_wiki = load_dataset(
+        "wikimedia/wikipedia", "20231101.en",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    wiki_texts = []
+    wiki_tokens = 0
+    wiki_seen = 0
+    wiki_skipped_quality = 0
+    wiki_skipped_meta = 0
+    t0 = time.time()
+
+    for article in ds_wiki:
+        title = (article.get("title") or "").strip()
+        raw = article.get("text") or ""
+
+        if _WIKI_SKIP_PATTERNS.search(title):
+            wiki_skipped_meta += 1
+            continue
+
+        cleaned = clean_text(raw)
+        if not is_high_quality(cleaned, min_chars=800, min_words=120):
+            wiki_skipped_quality += 1
+            continue
+
+        # This is a qualifying article - count it
+        wiki_seen += 1
+
+        # Skip if already used
+        if wiki_seen <= WIKI_SKIP:
+            if wiki_seen % 5000 == 0:
+                print(f"    Skipping... {wiki_seen:,}/{WIKI_SKIP:,}")
+            continue
+
+        full_text = f"{title}\n\n{cleaned}"
+        est_tok = int(len(full_text.split()) * TOKENS_PER_WORD)
+        wiki_texts.append(full_text)
+        wiki_tokens += est_tok
+
+        if len(wiki_texts) % 5000 == 0:
+            elapsed = time.time() - t0
+            print(f"    Collected {len(wiki_texts):,} articles | ~{wiki_tokens:,} tokens | {elapsed:.0f}s")
+
+        if wiki_tokens >= wiki_target:
+            break
+
+    elapsed = time.time() - t0
+    print(f"  [Wikipedia] Done: {len(wiki_texts):,} articles, ~{wiki_tokens:,} tokens in {elapsed:.0f}s")
+    print(f"    (skipped {WIKI_SKIP:,} already-used + {wiki_skipped_quality:,} low-quality + {wiki_skipped_meta:,} meta)")
+
+    # Save wiki parquets
+    BATCH = 5000
+    for i in range(0, len(wiki_texts), BATCH):
+        batch = wiki_texts[i:i+BATCH]
+        fp = OUTPUT_PARQUET_DIR / f"wiki_{i//BATCH:04d}.parquet"
+        pq.write_table(pa.table({"text": batch}), str(fp))
+    print(f"  Saved {(len(wiki_texts)-1)//BATCH + 1} wiki parquet files")
+
+    # --- FineWeb-Edu ---
+    print(f"\n  [FineWeb-Edu] Streaming (score >= {FINEWEB_MIN_SCORE}, skipping first {FINEWEB_SKIP:,})...")
+    ds_fineweb = load_dataset(
+        "HuggingFaceFW/fineweb-edu", "sample-10BT",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    fineweb_texts = []
+    fineweb_tokens = 0
+    fineweb_seen = 0
+    fineweb_skipped_quality = 0
+    fineweb_skipped_score = 0
+    t0 = time.time()
+
+    for doc in ds_fineweb:
+        score = doc.get("score", 0)
+        if not isinstance(score, (int, float)):
+            try:
+                score = float(score)
+            except (ValueError, TypeError):
+                continue
+        if score < FINEWEB_MIN_SCORE:
+            fineweb_skipped_score += 1
+            continue
+
+        raw = doc.get("text") or ""
+        cleaned = clean_text(raw)
+        if not is_high_quality(cleaned, min_chars=500, min_words=80):
+            fineweb_skipped_quality += 1
+            continue
+
+        # This is a qualifying doc - count it
+        fineweb_seen += 1
+
+        # Skip if already used
+        if fineweb_seen <= FINEWEB_SKIP:
+            if fineweb_seen % 5000 == 0:
+                print(f"    Skipping... {fineweb_seen:,}/{FINEWEB_SKIP:,}")
+            continue
+
+        est_tok = int(len(cleaned.split()) * TOKENS_PER_WORD)
+        fineweb_texts.append(cleaned)
+        fineweb_tokens += est_tok
+
+        if len(fineweb_texts) % 5000 == 0:
+            elapsed = time.time() - t0
+            print(f"    Collected {len(fineweb_texts):,} docs | ~{fineweb_tokens:,} tokens | {elapsed:.0f}s")
+
+        if fineweb_tokens >= fineweb_target:
+            break
+
+    elapsed = time.time() - t0
+    print(f"  [FineWeb-Edu] Done: {len(fineweb_texts):,} docs, ~{fineweb_tokens:,} tokens in {elapsed:.0f}s")
+    print(f"    (skipped {FINEWEB_SKIP:,} already-used + {fineweb_skipped_quality:,} low-quality + {fineweb_skipped_score:,} low-score)")
+
+    # Save fineweb parquets
+    for i in range(0, len(fineweb_texts), BATCH):
+        batch = fineweb_texts[i:i+BATCH]
+        fp = OUTPUT_PARQUET_DIR / f"fineweb_{i//BATCH:04d}.parquet"
+        pq.write_table(pa.table({"text": batch}), str(fp))
+    print(f"  Saved {(len(fineweb_texts)-1)//BATCH + 1} fineweb parquet files")
+
+    # Combine all texts
+    all_texts = wiki_texts + fineweb_texts
+    total_est_tokens = wiki_tokens + fineweb_tokens
+    del wiki_texts, fineweb_texts
+
+    print(f"\n  PHASE 1 COMPLETE: {len(all_texts):,} documents, ~{total_est_tokens:,} estimated tokens")
+
+    # ─────────────────────────────────────────────────────────────
+    # PHASE 2: Deep clean every document
+    # ─────────────────────────────────────────────────────────────
+    print(f"\n{'='*75}")
+    print(f"  PHASE 2: DEEP CLEANING {len(all_texts):,} DOCUMENTS")
+    print(f"{'='*75}")
+
+    t2 = time.time()
+    cleaned_texts = []
+    dropped_clean = 0
+    for i, text in enumerate(all_texts):
+        result = deep_clean(text)
+        if result is not None:
+            cleaned_texts.append(result)
+        else:
+            dropped_clean += 1
+        if (i + 1) % 10000 == 0 or i == len(all_texts) - 1:
+            print(f"    Cleaned {i+1:,}/{len(all_texts):,} | kept={len(cleaned_texts):,} | dropped={dropped_clean:,}")
+
+    del all_texts
+    print(f"  Deep clean done in {time.time()-t2:.1f}s")
+    print(f"  Kept: {len(cleaned_texts):,} | Dropped: {dropped_clean:,}")
+
+    # ─────────────────────────────────────────────────────────────
+    # PHASE 3: Smart cleanup (filter + boilerplate strip)
+    # ─────────────────────────────────────────────────────────────
+    print(f"\n{'='*75}")
+    print(f"  PHASE 3: SMART CLEANUP ON {len(cleaned_texts):,} DOCUMENTS")
+    print(f"{'='*75}")
+
+    t3 = time.time()
+    final_texts = []
+    removed_reasons = Counter()
+    total_boilerplate = 0
+
+    for i, text in enumerate(cleaned_texts):
+        keep, stripped_text, reason = smart_filter(text)
+        if keep:
+            final_texts.append(stripped_text)
+            total_boilerplate += len(text) - len(stripped_text)
+        else:
+            removed_reasons[reason.split('(')[0].strip().split(':')[0].strip()] += 1
+
+        if (i + 1) % 10000 == 0 or i == len(cleaned_texts) - 1:
+            print(f"    Processed {i+1:,}/{len(cleaned_texts):,} | kept={len(final_texts):,}")
+
+    del cleaned_texts
+    total_removed = len(final_texts)
+    print(f"  Smart cleanup done in {time.time()-t3:.1f}s")
+    print(f"  Final documents: {len(final_texts):,}")
+    print(f"  Boilerplate stripped: {total_boilerplate:,} chars")
+
+    if removed_reasons:
+        print(f"  Removal reasons:")
+        for reason, count in sorted(removed_reasons.items(), key=lambda x: -x[1]):
+            print(f"    {reason:<35} {count:>6,}")
+
+    # ─────────────────────────────────────────────────────────────
+    # PHASE 4: Tokenize and write litdata chunks
+    # ─────────────────────────────────────────────────────────────
+    print(f"\n{'='*75}")
+    print(f"  PHASE 4: TOKENIZING {len(final_texts):,} DOCUMENTS")
+    print(f"{'='*75}")
+
+    t4 = time.time()
+    all_token_ids = []
+    total_tokens = 0
+    ENCODE_BATCH = 10000
+
+    for i in range(0, len(final_texts), ENCODE_BATCH):
+        batch = final_texts[i:i+ENCODE_BATCH]
+        encoded = tokenizer.encode_batch(batch, add_special_tokens=False)
+        for enc in encoded:
+            ids = enc.ids
+            all_token_ids.extend(ids)
+            all_token_ids.append(EOS_TOKEN_ID)
+            total_tokens += len(ids) + 1
+        done = min(i + ENCODE_BATCH, len(final_texts))
+        if done % 20000 == 0 or done == len(final_texts):
+            print(f"    Tokenized {done:,}/{len(final_texts):,} ({total_tokens:,} tokens)")
+
+    del final_texts
+    print(f"  Tokenized in {time.time()-t4:.1f}s — {total_tokens:,} total tokens")
+
+    # Write litdata chunks
+    print(f"\n  Writing litdata chunks to {OUTPUT_LITDATA_DIR}...")
+    token_array = np.array(all_token_ids, dtype=DTYPE)
+    del all_token_ids
+
+    config = {
+        "block_size": BLOCK_SIZE,
+        "vocab_size": tokenizer.get_vocab_size(),
+    }
+    chunks = write_litdata_chunks(str(OUTPUT_LITDATA_DIR), token_array, config)
+    final_token_count = sum(c["dim"] for c in chunks)
+    del token_array
+
+    # ─────────────────────────────────────────────────────────────
+    # FINAL REPORT
+    # ─────────────────────────────────────────────────────────────
+    total_time = time.time() - t_start
+
+    report = []
+    report.append(f"\n{'='*75}")
+    report.append(f"  LITDATA_ENGLISH_100M - BUILD & CLEAN REPORT")
+    report.append(f"{'='*75}")
+    report.append(f"\n  Total time: {total_time:.0f}s ({total_time/60:.1f} min)")
+    report.append(f"\n  SOURCES")
+    report.append(f"  {'-'*60}")
+    report.append(f"    Wikipedia (articles {WIKI_SKIP+1:,}+):  {wiki_tokens:,} est. tokens")
+    report.append(f"    FineWeb-Edu (docs {FINEWEB_SKIP+1:,}+): {fineweb_tokens:,} est. tokens")
+    report.append(f"\n  PIPELINE RESULTS")
+    report.append(f"  {'-'*60}")
+    report.append(f"    Downloaded:        ~{total_est_tokens:,} est. tokens")
+    report.append(f"    After deep clean:  dropped {dropped_clean:,} docs")
+    report.append(f"    After smart filter: removed {sum(removed_reasons.values()):,} docs")
+    if removed_reasons:
+        for reason, count in sorted(removed_reasons.items(), key=lambda x: -x[1]):
+            report.append(f"      - {reason}: {count:,}")
+    report.append(f"    Boilerplate stripped: {total_boilerplate:,} chars")
+    report.append(f"\n  FINAL OUTPUT")
+    report.append(f"  {'-'*60}")
+    report.append(f"    Location: {OUTPUT_LITDATA_DIR}")
+    report.append(f"    Chunks:   {len(chunks)}")
+    report.append(f"    Tokens:   {final_token_count:,}")
+    report.append(f"    Format:   litdata binary (int32, BLOCK_SIZE=1025, EOS=0)")
+    report.append(f"\n  DATA IS DIFFERENT FROM litdata_english_clean:")
+    report.append(f"    - Skipped first {WIKI_SKIP:,} qualifying Wikipedia articles")
+    report.append(f"    - Skipped first {FINEWEB_SKIP:,} qualifying FineWeb-Edu docs")
+    report.append(f"    - Zero overlap with existing ~54M token corpus")
+    report.append(f"\n{'='*75}")
+
+    full_report = '\n'.join(report)
+    print(full_report)
+
+    report_path = OUTPUT_LITDATA_DIR / "BUILD_REPORT.txt"
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(full_report)
+    print(f"\n  Report saved to: {report_path}")
+    print(f"  Done! Ready for training.")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/build_5b_pretrain.py

@@ -0,0 +1,761 @@
+# -*- coding: utf-8 -*-
+"""
+Build 1.2B tokens of DIVERSE, CLEAN English data to bring pretrain total to 5B.
+
+Current: 3,819,212,525 tokens in litdata_pretrain_final
+Target:  5,000,000,000 tokens
+Gap:     ~1,181,000,000 tokens
+Build:   ~1,250,000,000 est. tokens (buffer for cleaning loss)
+
+DIVERSITY STRATEGY — 3 completely different source types:
+  1. Wikipedia (skip 200K qualifying articles) — encyclopedic knowledge
+  2. FineWeb-Edu (skip 200K qualifying docs, score ≥ 3.5) — educational web
+  3. OpenWebText (Skylion007) — Reddit-curated quality English web pages
+     * Completely new source, zero overlap with anything used before
+
+Pipeline per source:
+  Download (streaming) → Deep clean → Smart filter → Tokenize → Stream-write
+
+Memory-efficient: StreamingChunkWriter, multiprocessing cleaning (30 cores).
+Output appended directly to litdata_pretrain_final.
+"""
+
+import json
+import os
+import re
+import time
+import unicodedata
+from pathlib import Path
+from multiprocessing import Pool, cpu_count
+from collections import Counter
+
+import numpy as np
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+NUM_WORKERS = max(1, cpu_count() - 2)
+ENCODE_BATCH = 8000
+TOKENS_PER_WORD = 1.3
+
+DATA_DIR = ROOT / "Base" / "data"
+FINAL_DIR = DATA_DIR / "litdata_pretrain_final"
+TOKENIZER_PATH = str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+
+# ── Source targets ────────────────────────────────────────────────────────
+# Slightly over 1.2B to account for cleaning losses (~2-3%)
+WIKI_TARGET     = 365_000_000   # ~30% — encyclopedic
+FINEWEB_TARGET  = 445_000_000   # ~36% — educational web
+OWT_TARGET      = 445_000_000   # ~34% — Reddit-curated diverse web
+TOTAL_TARGET    = WIKI_TARGET + FINEWEB_TARGET + OWT_TARGET  # ~1.255B
+
+# Skip values — must exceed ALL previously used qualifying articles
+WIKI_SKIP    = 200_000   # previous max was 125K + articles collected
+FINEWEB_SKIP = 200_000   # previous max was 125K + docs collected
+FINEWEB_MIN_SCORE = 3.5  # slightly broader than 4.0 for more diversity
+
+_WIKI_SKIP_PATTERNS = re.compile(
+    r'(disambiguation|list of|lists of|index of|outline of|'
+    r'wikipedia:|template:|category:|portal:|module:|mediawiki:)',
+    re.IGNORECASE
+)
+
+
+# ==============================================================================
+# CLEANING PIPELINE
+# ==============================================================================
+
+CONTROL_CHARS = [
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f", "\x7f", "\ufeff", "\ufffd",
+]
+
+HTML_ENTITIES = [
+    ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+    ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"),
+    ("&nbsp;", " "), ("&mdash;", " - "), ("&ndash;", "-"),
+    ("&hellip;", "..."), ("&laquo;", '"'), ("&raquo;", '"'),
+    ("&bull;", "- "), ("&middot;", " "), ("&copy;", "(c)"),
+    ("&reg;", "(R)"), ("&trade;", "(TM)"), ("&deg;", " degrees"),
+]
+
+RE_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_FILE_PATH = re.compile(r'(?:[A-Z]:\\|/(?:home|usr|var|etc|opt)/)\S+')
+RE_HTML_TAG = re.compile(r'</?[a-zA-Z][a-zA-Z0-9]*(?:\s[^>]*)?\s*/?>')
+RE_HTML_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+RE_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
+RE_IMPORT = re.compile(r'^(?:import |from \S+ import |#include |using namespace |require\()', re.M)
+RE_REPEATED_LINE = re.compile(r'^(.{20,})\n(?:\1\n?)+', re.M)
+RE_REPEATED_PUNCT = re.compile(r'([!?.])\1{3,}')
+RE_REPEATED_CHAR = re.compile(r'(.)\1{5,}')
+RE_REPEATED_WORD = re.compile(r'\b(\w+)(?:\s+\1){2,}\b', re.I)
+RE_MULTI_NEWLINE = re.compile(r'\n{4,}')
+RE_MULTI_SPACE = re.compile(r'[ \t]{2,}')
+RE_TRAILING_SPACE = re.compile(r'[ \t]+$', re.M)
+RE_NO_SPACE_AFTER_PERIOD = re.compile(r'([.!?])([A-Z])')
+RE_DOUBLE_PERIOD = re.compile(r'\.{2}(?!\.)')
+RE_SPACE_BEFORE_PUNCT = re.compile(r'\s+([.,;:!?])')
+
+RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+RE_DEVANAGARI = re.compile(r'[\u0900-\u097f]{5,}')
+RE_RESIDUAL_CODE = re.compile(
+    r'(function\s*\(|var\s+\w+\s*=|console\.log|document\.get|if\s*\(\s*\w+\s*[!=]==)', re.I
+)
+RE_COOKIE_LINE = re.compile(r'^.*(?:cookie|cookies)\s+(?:policy|consent|notice|preferences|settings).*$', re.I | re.M)
+RE_SUBSCRIBE_LINE = re.compile(r'^.*(?:subscribe|sign\s*up\s+(?:for|to)\s+(?:our|the)\s+newsletter|unsubscribe|opt[\s-]*out\s+of).*$', re.I | re.M)
+RE_CLICKBAIT_LINE = re.compile(r'^.*(?:you\s+won\'?t\s+believe|click\s+here|read\s+more\s*\.{0,3}$|share\s+this\s+(?:article|post|story)|trending\s+now|sponsored\s+content|advertisement).*$', re.I | re.M)
+RE_SOCIAL_LINE = re.compile(r'^.*(?:follow\s+us\s+on|share\s+on\s+(?:facebook|twitter|linkedin|instagram)|like\s+us\s+on|tweet\s+this).*$', re.I | re.M)
+RE_NAV_LINE = re.compile(r'^.*(?:skip\s+to\s+(?:main\s+)?content|back\s+to\s+top|previous\s+article|next\s+article|related\s+(?:articles|posts)).*$', re.I | re.M)
+RE_LOGIN_LINE = re.compile(r'^.*(?:log\s*in\s+to\s+(?:your|an)\s+account|create\s+(?:a\s+)?(?:free\s+)?account|forgot\s+(?:your\s+)?password|already\s+(?:a\s+)?member).*$', re.I | re.M)
+RE_COMMENT_LINE = re.compile(r'^.*(?:leave\s+a\s+(?:comment|reply)|post\s+a\s+comment|\d+\s+comments?$|logged\s+in\s+as).*$', re.I | re.M)
+RE_COPYRIGHT_LINE = re.compile(r'^.*(?:all\s+rights\s+reserved|\(c\)\s*\d{4}|copyright\s+\d{4}).*$', re.I | re.M)
+
+
+def clean_text_basic(text):
+    """Light quality filter for download phase."""
+    text = unicodedata.normalize("NFKC", text)
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = '\n'.join(line.strip() for line in text.split('\n'))
+    return text.strip()
+
+
+def is_high_quality(text, min_chars=500, min_words=80):
+    if len(text) < min_chars:
+        return False
+    words = text.split()
+    num_words = len(words)
+    if num_words < min_words:
+        return False
+    alpha = sum(c.isalpha() for c in text)
+    if alpha / max(len(text), 1) < 0.65:
+        return False
+    avg_word_len = sum(len(w) for w in words) / num_words
+    if avg_word_len < 2.5 or avg_word_len > 15:
+        return False
+    url_hits = text.count('http://') + text.count('https://')
+    if url_hits > num_words * 0.03:
+        return False
+    sentences = re.split(r'[.!?]+', text)
+    real_sentences = [s.strip() for s in sentences if len(s.strip()) > 10]
+    if len(real_sentences) < 3:
+        return False
+    lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
+    if len(lines) > 5:
+        unique_ratio = len(set(lines)) / len(lines)
+        if unique_ratio < 0.5:
+            return False
+    return True
+
+
+def deep_clean(text):
+    if not text or len(text.strip()) < 30:
+        return None
+    text = unicodedata.normalize("NFKC", text)
+    for ch in CONTROL_CHARS:
+        text = text.replace(ch, "")
+    for old, new in HTML_ENTITIES:
+        text = text.replace(old, new)
+    text = RE_HTML_COMMENT.sub("", text)
+    text = RE_HTML_TAG.sub("", text)
+    text = RE_URL.sub("", text)
+    text = RE_EMAIL.sub("", text)
+    text = RE_FILE_PATH.sub("", text)
+    text = RE_CODE_BLOCK.sub("", text)
+    text = RE_REPEATED_LINE.sub(r'\1', text)
+    text = RE_REPEATED_PUNCT.sub(r'\1\1\1', text)
+    text = RE_REPEATED_CHAR.sub(r'\1\1\1', text)
+    text = RE_REPEATED_WORD.sub(r'\1', text)
+    text = text.replace('\t', ' ')
+    text = RE_TRAILING_SPACE.sub('', text)
+    text = RE_MULTI_SPACE.sub(' ', text)
+    text = RE_MULTI_NEWLINE.sub('\n\n\n', text)
+    text = RE_DOUBLE_PERIOD.sub('.', text)
+    text = RE_NO_SPACE_AFTER_PERIOD.sub(r'\1 \2', text)
+    text = RE_SPACE_BEFORE_PUNCT.sub(r'\1', text)
+    text = text.replace('\u2018', "'").replace('\u2019', "'")
+    text = text.replace('\u201c', '"').replace('\u201d', '"')
+    text = text.replace('\u2013', '-').replace('\u2014', ' - ')
+    text = text.replace('\u2026', '...')
+    text = text.replace('\u2022', '- ')
+    text = text.replace('\u00b7', ' ')
+    text = text.replace('\u00a0', ' ')
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            clean_lines.append('')
+            continue
+        if len(line) > 10:
+            alpha_count = sum(1 for c in line if c.isalpha())
+            if alpha_count / len(line) < 0.40:
+                continue
+        if line.count('|') > 3 or line.count('{') > 2 or line.count('}') > 2:
+            continue
+        if RE_IMPORT.match(line):
+            continue
+        if line and line[0].isalpha() and line[0].islower():
+            if not clean_lines or clean_lines[-1] == '' or clean_lines[-1].rstrip().endswith(('.', '!', '?', ':')):
+                line = line[0].upper() + line[1:]
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = text.strip()
+    paragraphs = text.split('\n\n')
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        p_stripped = p.strip()
+        if not p_stripped:
+            continue
+        p_key = ' '.join(p_stripped.lower().split())
+        if p_key not in seen:
+            seen.add(p_key)
+            unique_paragraphs.append(p_stripped)
+    text = '\n\n'.join(unique_paragraphs)
+    text = text.strip()
+    if len(text) < 50:
+        return None
+    if len(text.split()) < 10:
+        return None
+    ascii_count = sum(1 for c in text if ord(c) < 128)
+    if ascii_count / max(len(text), 1) < 0.85:
+        return None
+    return text
+
+
+def smart_filter(text):
+    words = text.split()
+    word_count = len(words)
+    if word_count < 50:
+        return False, text, "too short"
+    scripts = []
+    if RE_CJK.search(text): scripts.append("CJK")
+    if RE_ARABIC.search(text): scripts.append("Arabic")
+    if RE_CYRILLIC.search(text): scripts.append("Cyrillic")
+    if RE_DEVANAGARI.search(text): scripts.append("Devanagari")
+    if scripts:
+        return False, text, "non-English"
+    if word_count > 50:
+        unique_ratio = len(set(w.lower() for w in words)) / word_count
+        if unique_ratio < 0.20:
+            return False, text, "repetitive"
+    code_matches = RE_RESIDUAL_CODE.findall(text)
+    if len(code_matches) >= 5:
+        return False, text, "residual code"
+    for pattern in [RE_COOKIE_LINE, RE_SUBSCRIBE_LINE, RE_CLICKBAIT_LINE,
+                    RE_SOCIAL_LINE, RE_NAV_LINE, RE_LOGIN_LINE,
+                    RE_COMMENT_LINE, RE_COPYRIGHT_LINE]:
+        text = pattern.sub('', text)
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped and len(stripped) > 10:
+            digit_count = sum(1 for c in stripped if c.isdigit() or c in ' ,.\t-+/%$')
+            if digit_count / len(stripped) > 0.80:
+                continue
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = text.strip()
+    if len(text.split()) < 50:
+        return False, text, "too short after stripping"
+    return True, text, None
+
+
+def clean_and_filter(text):
+    result = deep_clean(text)
+    if result is None:
+        return None, "deep_clean_drop", 0
+    keep, stripped, reason = smart_filter(result)
+    if not keep:
+        return None, reason, 0
+    return stripped, None, len(result) - len(stripped)
+
+
+# ==============================================================================
+# STREAMING CHUNK WRITER (appends to existing litdata)
+# ==============================================================================
+
+class StreamingChunkWriter:
+    def __init__(self, output_dir, config, start_chunk_idx=0):
+        self.output_dir = Path(output_dir)
+        os.makedirs(str(self.output_dir), exist_ok=True)
+        self.config = config
+        self.dtype_size = DTYPE().itemsize
+        self.tokens_per_chunk = (CHUNK_BYTES_TARGET // self.dtype_size // BLOCK_SIZE) * BLOCK_SIZE
+        self.buffer = []
+        self.chunks_metadata = []
+        self.chunk_idx = start_chunk_idx
+        self.total_tokens = 0
+
+    def add_tokens_batch(self, encoded_batch):
+        for enc in encoded_batch:
+            ids = enc.ids
+            self.buffer.extend(ids)
+            self.buffer.append(EOS_TOKEN_ID)
+        while len(self.buffer) >= self.tokens_per_chunk:
+            self._flush_chunk()
+
+    def _flush_chunk(self):
+        if len(self.buffer) < BLOCK_SIZE:
+            return
+        take = min(len(self.buffer), self.tokens_per_chunk)
+        num_blocks = take // BLOCK_SIZE
+        if num_blocks == 0:
+            return
+        actual_tokens = num_blocks * BLOCK_SIZE
+        chunk_data = np.array(self.buffer[:actual_tokens], dtype=DTYPE)
+        self.buffer = self.buffer[actual_tokens:]
+        filename = f"chunk-0-{self.chunk_idx}.bin"
+        filepath = self.output_dir / filename
+        header_num = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * self.dtype_size)
+        header = np.concatenate([header_num, offsets])
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        self.chunks_metadata.append(meta)
+        self.total_tokens += actual_tokens
+        self.chunk_idx += 1
+        if self.chunk_idx % 10 == 0:
+            print(f"        Flushed chunk {self.chunk_idx} ({self.total_tokens:,} new tokens)")
+
+    def finalize(self):
+        while len(self.buffer) >= BLOCK_SIZE:
+            self._flush_chunk()
+        discarded = len(self.buffer)
+        self.buffer = []
+        return self.total_tokens, discarded
+
+
+# ==============================================================================
+# PROCESS ONE SOURCE: download → clean (multiprocessed) → tokenize → write
+# ==============================================================================
+
+def process_source_batch(texts, pool, tokenizer, writer, source_name, batch_num):
+    """Clean a batch of texts and write to the streaming writer. Returns stats."""
+    t0 = time.time()
+
+    # Clean with multiprocessing
+    results = pool.map(clean_and_filter, texts, chunksize=256)
+
+    cleaned = []
+    dropped = 0
+    reasons = Counter()
+    boilerplate = 0
+    for text, reason, bp in results:
+        if text is not None:
+            cleaned.append(text)
+            boilerplate += bp
+        else:
+            dropped += 1
+            reasons[reason] += 1
+    del results
+
+    # Tokenize + write
+    for i in range(0, len(cleaned), ENCODE_BATCH):
+        sub = cleaned[i:i+ENCODE_BATCH]
+        encoded = tokenizer.encode_batch(sub, add_special_tokens=False)
+        writer.add_tokens_batch(encoded)
+    del cleaned
+
+    elapsed = time.time() - t0
+    print(f"      [{source_name}] batch {batch_num}: kept {len(texts)-dropped:,} / dropped {dropped} | {elapsed:.1f}s")
+
+    return len(texts) - dropped, dropped, reasons, boilerplate
+
+
+# ==============================================================================
+# MAIN
+# ==============================================================================
+
+def main():
+    from datasets import load_dataset
+
+    t_start = time.time()
+
+    print("Loading tokenizer...")
+    tokenizer = Tokenizer.from_file(TOKENIZER_PATH)
+    config = {"block_size": BLOCK_SIZE, "vocab_size": tokenizer.get_vocab_size()}
+
+    # Read existing index to find starting chunk offset
+    with open(FINAL_DIR / "index.json") as f:
+        existing_index = json.load(f)
+    existing_chunks = existing_index["chunks"]
+    existing_tokens = sum(c["dim"] for c in existing_chunks)
+    start_chunk_idx = len(existing_chunks)
+
+    print(f"\n{'='*75}")
+    print(f"  BUILD 1.2B DIVERSE TOKENS → APPEND TO litdata_pretrain_final")
+    print(f"  Current:  {existing_tokens:,} tokens ({start_chunk_idx} chunks)")
+    print(f"  Target:   5,000,000,000 tokens")
+    print(f"  Building: ~{TOTAL_TARGET:,} estimated tokens")
+    print(f"  Workers:  {NUM_WORKERS} CPU cores")
+    print(f"{'='*75}")
+
+    # Initialize writer that appends new chunks after existing ones
+    writer = StreamingChunkWriter(str(FINAL_DIR), config, start_chunk_idx=start_chunk_idx)
+    pool = Pool(processes=NUM_WORKERS)
+
+    # Global stats
+    all_stats = {}
+    DOWNLOAD_BATCH = 10000  # process 10K docs at a time
+
+    # ═══════════════════════════════════════════════════════════════════════
+    # SOURCE 1: Wikipedia (encyclopedic knowledge)
+    # ═══════════════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  SOURCE 1: WIKIPEDIA (skip {WIKI_SKIP:,}, target ~{WIKI_TARGET:,} tokens)")
+    print(f"{'='*75}")
+
+    ds_wiki = load_dataset(
+        "wikimedia/wikipedia", "20231101.en",
+        split="train", streaming=True, trust_remote_code=False
+    )
+
+    wiki_texts = []
+    wiki_tokens_est = 0
+    wiki_seen = 0
+    wiki_skipped_quality = 0
+    wiki_skipped_meta = 0
+    wiki_total_kept = 0
+    wiki_total_dropped = 0
+    wiki_reasons = Counter()
+    wiki_boilerplate = 0
+    wiki_batch_num = 0
+    t0 = time.time()
+
+    for article in ds_wiki:
+        title = (article.get("title") or "").strip()
+        raw = article.get("text") or ""
+
+        if _WIKI_SKIP_PATTERNS.search(title):
+            wiki_skipped_meta += 1
+            continue
+
+        cleaned = clean_text_basic(raw)
+        if not is_high_quality(cleaned, min_chars=800, min_words=120):
+            wiki_skipped_quality += 1
+            continue
+
+        wiki_seen += 1
+        if wiki_seen <= WIKI_SKIP:
+            if wiki_seen % 25000 == 0:
+                print(f"    Skipping... {wiki_seen:,}/{WIKI_SKIP:,}")
+            continue
+
+        full_text = f"{title}\n\n{cleaned}"
+        est_tok = int(len(full_text.split()) * TOKENS_PER_WORD)
+        wiki_texts.append(full_text)
+        wiki_tokens_est += est_tok
+
+        # Process in batches to keep memory low
+        if len(wiki_texts) >= DOWNLOAD_BATCH:
+            wiki_batch_num += 1
+            kept, dropped, reasons, bp = process_source_batch(
+                wiki_texts, pool, tokenizer, writer, "Wiki", wiki_batch_num
+            )
+            wiki_total_kept += kept
+            wiki_total_dropped += dropped
+            wiki_reasons += reasons
+            wiki_boilerplate += bp
+            wiki_texts = []
+
+        if wiki_tokens_est >= WIKI_TARGET:
+            break
+
+    # Process remaining
+    if wiki_texts:
+        wiki_batch_num += 1
+        kept, dropped, reasons, bp = process_source_batch(
+            wiki_texts, pool, tokenizer, writer, "Wiki", wiki_batch_num
+        )
+        wiki_total_kept += kept
+        wiki_total_dropped += dropped
+        wiki_reasons += reasons
+        wiki_boilerplate += bp
+        wiki_texts = []
+
+    wiki_elapsed = time.time() - t0
+    print(f"  [Wikipedia] Done: ~{wiki_tokens_est:,} est. tokens, kept {wiki_total_kept:,}, dropped {wiki_total_dropped:,} in {wiki_elapsed:.0f}s")
+    print(f"    (skipped {WIKI_SKIP:,} already-used + {wiki_skipped_quality:,} low-quality + {wiki_skipped_meta:,} meta)")
+    all_stats["Wikipedia"] = {
+        "est_tokens": wiki_tokens_est, "kept": wiki_total_kept,
+        "dropped": wiki_total_dropped, "reasons": wiki_reasons,
+        "boilerplate": wiki_boilerplate, "time": wiki_elapsed,
+    }
+
+    # ═══════════════════════════════════════════════════════════════════════
+    # SOURCE 2: FineWeb-Edu (educational web content)
+    # ═══════════════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  SOURCE 2: FINEWEB-EDU (skip {FINEWEB_SKIP:,}, score >= {FINEWEB_MIN_SCORE}, target ~{FINEWEB_TARGET:,} tokens)")
+    print(f"{'='*75}")
+
+    ds_fw = load_dataset(
+        "HuggingFaceFW/fineweb-edu", "sample-10BT",
+        split="train", streaming=True, trust_remote_code=False
+    )
+
+    fw_texts = []
+    fw_tokens_est = 0
+    fw_seen = 0
+    fw_skipped_quality = 0
+    fw_skipped_score = 0
+    fw_total_kept = 0
+    fw_total_dropped = 0
+    fw_reasons = Counter()
+    fw_boilerplate = 0
+    fw_batch_num = 0
+    t0 = time.time()
+
+    for doc in ds_fw:
+        score = doc.get("score", 0)
+        if not isinstance(score, (int, float)):
+            try:
+                score = float(score)
+            except (ValueError, TypeError):
+                continue
+        if score < FINEWEB_MIN_SCORE:
+            fw_skipped_score += 1
+            continue
+
+        raw = doc.get("text") or ""
+        cleaned = clean_text_basic(raw)
+        if not is_high_quality(cleaned, min_chars=500, min_words=80):
+            fw_skipped_quality += 1
+            continue
+
+        fw_seen += 1
+        if fw_seen <= FINEWEB_SKIP:
+            if fw_seen % 25000 == 0:
+                print(f"    Skipping... {fw_seen:,}/{FINEWEB_SKIP:,}")
+            continue
+
+        est_tok = int(len(cleaned.split()) * TOKENS_PER_WORD)
+        fw_texts.append(cleaned)
+        fw_tokens_est += est_tok
+
+        if len(fw_texts) >= DOWNLOAD_BATCH:
+            fw_batch_num += 1
+            kept, dropped, reasons, bp = process_source_batch(
+                fw_texts, pool, tokenizer, writer, "FineWeb", fw_batch_num
+            )
+            fw_total_kept += kept
+            fw_total_dropped += dropped
+            fw_reasons += reasons
+            fw_boilerplate += bp
+            fw_texts = []
+
+        if fw_tokens_est >= FINEWEB_TARGET:
+            break
+
+    if fw_texts:
+        fw_batch_num += 1
+        kept, dropped, reasons, bp = process_source_batch(
+            fw_texts, pool, tokenizer, writer, "FineWeb", fw_batch_num
+        )
+        fw_total_kept += kept
+        fw_total_dropped += dropped
+        fw_reasons += reasons
+        fw_boilerplate += bp
+        fw_texts = []
+
+    fw_elapsed = time.time() - t0
+    print(f"  [FineWeb-Edu] Done: ~{fw_tokens_est:,} est. tokens, kept {fw_total_kept:,}, dropped {fw_total_dropped:,} in {fw_elapsed:.0f}s")
+    print(f"    (skipped {FINEWEB_SKIP:,} already-used + {fw_skipped_quality:,} low-quality + {fw_skipped_score:,} low-score)")
+    all_stats["FineWeb-Edu"] = {
+        "est_tokens": fw_tokens_est, "kept": fw_total_kept,
+        "dropped": fw_total_dropped, "reasons": fw_reasons,
+        "boilerplate": fw_boilerplate, "time": fw_elapsed,
+    }
+
+    # ═══════════════════════════════════════════════════════════════════════
+    # SOURCE 3: OpenWebText (Reddit-curated diverse web pages)
+    # ═══════════════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  SOURCE 3: OPENWEBTEXT (target ~{OWT_TARGET:,} tokens)")
+    print(f"  Completely new source — zero overlap with existing data")
+    print(f"{'='*75}")
+
+    ds_owt = load_dataset(
+        "Skylion007/openwebtext",
+        split="train", streaming=True, trust_remote_code=False
+    )
+
+    owt_texts = []
+    owt_tokens_est = 0
+    owt_skipped_quality = 0
+    owt_total_kept = 0
+    owt_total_dropped = 0
+    owt_reasons = Counter()
+    owt_boilerplate = 0
+    owt_batch_num = 0
+    t0 = time.time()
+
+    for doc in ds_owt:
+        raw = doc.get("text") or ""
+        cleaned = clean_text_basic(raw)
+        if not is_high_quality(cleaned, min_chars=400, min_words=60):
+            owt_skipped_quality += 1
+            continue
+
+        est_tok = int(len(cleaned.split()) * TOKENS_PER_WORD)
+        owt_texts.append(cleaned)
+        owt_tokens_est += est_tok
+
+        if len(owt_texts) >= DOWNLOAD_BATCH:
+            owt_batch_num += 1
+            kept, dropped, reasons, bp = process_source_batch(
+                owt_texts, pool, tokenizer, writer, "OWT", owt_batch_num
+            )
+            owt_total_kept += kept
+            owt_total_dropped += dropped
+            owt_reasons += reasons
+            owt_boilerplate += bp
+            owt_texts = []
+
+        if owt_tokens_est >= OWT_TARGET:
+            break
+
+    if owt_texts:
+        owt_batch_num += 1
+        kept, dropped, reasons, bp = process_source_batch(
+            owt_texts, pool, tokenizer, writer, "OWT", owt_batch_num
+        )
+        owt_total_kept += kept
+        owt_total_dropped += dropped
+        owt_reasons += reasons
+        owt_boilerplate += bp
+        owt_texts = []
+
+    owt_elapsed = time.time() - t0
+    print(f"  [OpenWebText] Done: ~{owt_tokens_est:,} est. tokens, kept {owt_total_kept:,}, dropped {owt_total_dropped:,} in {owt_elapsed:.0f}s")
+    print(f"    (skipped {owt_skipped_quality:,} low-quality)")
+    all_stats["OpenWebText"] = {
+        "est_tokens": owt_tokens_est, "kept": owt_total_kept,
+        "dropped": owt_total_dropped, "reasons": owt_reasons,
+        "boilerplate": owt_boilerplate, "time": owt_elapsed,
+    }
+
+    pool.close()
+    pool.join()
+
+    # ═══════════════════════════════════════════════════════════════════════
+    # FINALIZE: flush remaining + update index.json
+    # ═══════════════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  FINALIZING")
+    print(f"{'='*75}")
+
+    new_tokens, discarded = writer.finalize()
+
+    # Merge new chunk metadata with existing
+    final_chunks = existing_chunks + writer.chunks_metadata
+    final_total = existing_tokens + new_tokens
+    final_index = {
+        "chunks": final_chunks,
+        "config": config,
+        "updated_at": str(time.time()),
+    }
+    with open(FINAL_DIR / "index.json", "w") as f:
+        json.dump(final_index, f, indent=2)
+
+    total_time = time.time() - t_start
+
+    # ═══════════════════════════════════════════════════════════════════════
+    # REPORT
+    # ═══════════════════════════════════════════════════════════════════════
+    total_new_kept = wiki_total_kept + fw_total_kept + owt_total_kept
+    total_new_dropped = wiki_total_dropped + fw_total_dropped + owt_total_dropped
+    total_new_boilerplate = wiki_boilerplate + fw_boilerplate + owt_boilerplate
+    all_drop_reasons = wiki_reasons + fw_reasons + owt_reasons
+
+    report = []
+    report.append(f"{'='*75}")
+    report.append(f"  5 BILLION TOKEN PRETRAIN DATASET — BUILD REPORT")
+    report.append(f"{'='*75}")
+    report.append(f"")
+    report.append(f"  Total time: {total_time:.0f}s ({total_time/60:.1f} min)")
+    report.append(f"  Workers: {NUM_WORKERS} CPU cores")
+    report.append(f"")
+    report.append(f"  NEW DATA ADDED (diverse, clean English)")
+    report.append(f"  {'-'*60}")
+
+    for name, stats in all_stats.items():
+        report.append(f"    {name}:")
+        report.append(f"      Est tokens: ~{stats['est_tokens']:,}")
+        report.append(f"      Kept: {stats['kept']:,} | Dropped: {stats['dropped']:,}")
+        report.append(f"      Boilerplate: {stats['boilerplate']:,} chars")
+        report.append(f"      Time: {stats['time']:.0f}s")
+        if stats["reasons"]:
+            for reason, count in sorted(stats["reasons"].items(), key=lambda x: -x[1]):
+                report.append(f"        {reason}: {count:,}")
+
+    report.append(f"")
+    report.append(f"  NEW DATA TOTALS")
+    report.append(f"  {'-'*60}")
+    report.append(f"    Documents kept:    {total_new_kept:,}")
+    report.append(f"    Documents dropped: {total_new_dropped:,}")
+    report.append(f"    Boilerplate:       {total_new_boilerplate:,} chars stripped")
+    report.append(f"    New tokens:        {new_tokens:,} ({writer.chunk_idx - start_chunk_idx} chunks)")
+    if all_drop_reasons:
+        report.append(f"    Drop reasons:")
+        for reason, count in sorted(all_drop_reasons.items(), key=lambda x: -x[1]):
+            report.append(f"      {reason:<35} {count:>8,}")
+
+    report.append(f"")
+    report.append(f"  FINAL COMBINED DATASET")
+    report.append(f"  {'-'*60}")
+    report.append(f"    Location:  {FINAL_DIR}")
+    report.append(f"    Chunks:    {len(final_chunks)}")
+    report.append(f"    Tokens:    {final_total:,}")
+    report.append(f"    Format:    litdata binary (int32, BLOCK_SIZE=1025, EOS=0)")
+    report.append(f"")
+    report.append(f"    Previous:  {existing_tokens:,} tokens ({start_chunk_idx} chunks)")
+    report.append(f"    + Added:   {new_tokens:,} tokens ({writer.chunk_idx - start_chunk_idx} chunks)")
+    report.append(f"    = Total:   {final_total:,} tokens ({len(final_chunks)} chunks)")
+    report.append(f"")
+    report.append(f"  DATA COMPOSITION")
+    report.append(f"  {'-'*60}")
+    report.append(f"    litdata_3b_clean:    ~2.94B tokens (general web, cleaned)")
+    report.append(f"    litdata_english_500m: ~515M tokens (Wiki+FineWeb, cleaned)")
+    report.append(f"    litdata_combined:     ~257M tokens (Wiki+FineWeb, cleaned)")
+    report.append(f"    + Wikipedia (new):    ~{wiki_tokens_est:,} est. (articles {WIKI_SKIP+1:,}+)")
+    report.append(f"    + FineWeb-Edu (new):  ~{fw_tokens_est:,} est. (score>={FINEWEB_MIN_SCORE}, docs {FINEWEB_SKIP+1:,}+)")
+    report.append(f"    + OpenWebText (new):  ~{owt_tokens_est:,} est. (Reddit-curated, no overlap)")
+    report.append(f"")
+    report.append(f"  PURE ENGLISH PRETRAINING TEXT")
+    report.append(f"  Sources: Wikipedia, FineWeb-Edu, OpenWebText, general web")
+    report.append(f"  NO instruction/finetune data included")
+    report.append(f"  ZERO overlap between all data sources")
+    report.append(f"{'='*75}")
+
+    full_report = '\n'.join(report)
+    print(f"\n{full_report}")
+
+    with open(FINAL_DIR / "BUILD_REPORT.txt", "w", encoding="utf-8") as f:
+        f.write(full_report)
+    print(f"\n  Report saved to: {FINAL_DIR / 'BUILD_REPORT.txt'}")
+    print(f"  Done! 5B token pretrain dataset ready.")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/build_english_corpus.py

@@ -0,0 +1,349 @@
+"""
+Build an ultra-clean English text corpus for continued pretraining.
+
+Downloads from high-quality, professionally curated sources via HuggingFace
+streaming (constant memory, no full dataset download needed):
+
+  1. Wikipedia English — encyclopedic, multi-editor-reviewed text
+     (the single cleanest large-scale English corpus in existence)
+  2. FineWeb-Edu — top-scored educational web content (score >= 4.0)
+     (adds diversity: textbooks, tutorials, explanations, articles)
+
+Quality filters applied to every text:
+  - Minimum length and word count
+  - High alphabetic ratio (rejects tables, code, symbol dumps)
+  - Sentence structure check (real prose, not lists/fragments)
+  - Repetition detection (rejects copy-paste / boilerplate)
+  - URL density filter (rejects link farms)
+  - Unicode normalization and whitespace cleanup
+
+Outputs parquet files to Base/data/filtered_english/ with a single 'text'
+column, compatible with the existing prepare_litdata.py pipeline.
+
+After running this script:
+  1. python Base/scripts/prepare_litdata.py --mode english
+  2. litgpt pretrain --config Base/configs/pretrain_100m_english.yaml
+
+Usage:
+  python Base/scripts/build_english_corpus.py
+  python Base/scripts/build_english_corpus.py --target_tokens 100000000
+  python Base/scripts/build_english_corpus.py --sources wiki
+  python Base/scripts/build_english_corpus.py --sources fineweb
+  python Base/scripts/build_english_corpus.py --fineweb_min_score 4.5
+"""
+
+import argparse
+import os
+import re
+import time
+import unicodedata
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+# ─── Text Quality Filters ──────────────────────────────────────────────────
+
+def clean_text(text: str) -> str:
+    """Normalize Unicode, strip control chars, fix whitespace."""
+    # Normalize unicode (NFKC merges compatibility chars)
+    text = unicodedata.normalize("NFKC", text)
+    # Remove control characters (keep newlines and tabs)
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
+    # Collapse 3+ blank lines into 2
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    # Collapse multiple spaces/tabs into single space
+    text = re.sub(r'[ \t]+', ' ', text)
+    # Strip each line
+    text = '\n'.join(line.strip() for line in text.split('\n'))
+    return text.strip()
+
+
+def is_high_quality(text: str, min_chars: int = 500, min_words: int = 80) -> bool:
+    """Strict quality gate — only passes clean, well-formed English prose."""
+    if len(text) < min_chars:
+        return False
+
+    words = text.split()
+    num_words = len(words)
+    if num_words < min_words:
+        return False
+
+    # Must be mostly alphabetic (not tables, code, numbers)
+    alpha = sum(c.isalpha() for c in text)
+    if alpha / max(len(text), 1) < 0.65:
+        return False
+
+    # Average word length sanity check (2.5–15 chars for English)
+    avg_word_len = sum(len(w) for w in words) / num_words
+    if avg_word_len < 2.5 or avg_word_len > 15:
+        return False
+
+    # Not too many URLs (< 3% of words)
+    url_hits = text.count('http://') + text.count('https://')
+    if url_hits > num_words * 0.03:
+        return False
+
+    # Must contain real sentences (at least 3 sentences > 10 chars)
+    sentences = re.split(r'[.!?]+', text)
+    real_sentences = [s.strip() for s in sentences if len(s.strip()) > 10]
+    if len(real_sentences) < 3:
+        return False
+
+    # Repetition filter — unique lines should be > 50%
+    lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
+    if len(lines) > 5:
+        unique_ratio = len(set(lines)) / len(lines)
+        if unique_ratio < 0.5:
+            return False
+
+    return True
+
+
+# ─── Wikipedia Source ───────────────────────────────────────────────────────
+
+# Wikipedia articles with these title patterns are low-value for language model
+_WIKI_SKIP_PATTERNS = re.compile(
+    r'(disambiguation|list of|lists of|index of|outline of|'
+    r'wikipedia:|template:|category:|portal:|module:|mediawiki:)',
+    re.IGNORECASE
+)
+
+
+def fetch_wikipedia(target_tokens: int, output_dir: str, tokens_per_word: float = 1.3):
+    """Stream Wikipedia English, apply strict quality filters, output parquet."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Wikipedia] Streaming English articles")
+    print(f"  Target: {target_tokens:,} tokens")
+    print(f"  Output: {output_dir}")
+    print(f"{'='*60}\n")
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    ds = load_dataset(
+        "wikimedia/wikipedia", "20231101.en",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    buf = []
+    total_tokens = 0
+    kept = 0
+    skipped = 0
+    file_idx = 0
+    BATCH = 5000
+    t0 = time.time()
+
+    for article in ds:
+        title = (article.get("title") or "").strip()
+        raw = article.get("text") or ""
+
+        # Skip meta / navigation articles
+        if _WIKI_SKIP_PATTERNS.search(title):
+            skipped += 1
+            continue
+
+        cleaned = clean_text(raw)
+
+        # Strict filter: min 800 chars, 120 words for Wikipedia
+        if not is_high_quality(cleaned, min_chars=800, min_words=120):
+            skipped += 1
+            continue
+
+        # Prepend title as natural heading
+        full_text = f"{title}\n\n{cleaned}"
+        est_tok = int(len(full_text.split()) * tokens_per_word)
+
+        buf.append(full_text)
+        total_tokens += est_tok
+        kept += 1
+
+        # Flush batch to parquet
+        if len(buf) >= BATCH:
+            fp = Path(output_dir) / f"wiki_{file_idx:04d}.parquet"
+            pq.write_table(pa.table({"text": buf}), str(fp))
+            elapsed = time.time() - t0
+            rate = total_tokens / max(elapsed, 1)
+            print(f"  wiki_{file_idx:04d}.parquet | {kept:,} articles | "
+                  f"~{total_tokens:,} tok | {rate:,.0f} tok/s | {elapsed:.0f}s")
+            buf = []
+            file_idx += 1
+
+        if total_tokens >= target_tokens:
+            break
+
+    # Write remaining
+    if buf:
+        fp = Path(output_dir) / f"wiki_{file_idx:04d}.parquet"
+        pq.write_table(pa.table({"text": buf}), str(fp))
+        file_idx += 1
+
+    elapsed = time.time() - t0
+    print(f"\n--- Wikipedia complete ---")
+    print(f"  Kept: {kept:,} articles | Skipped: {skipped:,}")
+    print(f"  Est. tokens: {total_tokens:,}")
+    print(f"  Files: {file_idx} | Time: {elapsed:.0f}s")
+    return total_tokens
+
+
+# ─── FineWeb-Edu Source ─────────────────────────────────────────────────────
+
+def fetch_fineweb_edu(target_tokens: int, output_dir: str,
+                      min_score: float = 4.0, tokens_per_word: float = 1.3):
+    """Stream FineWeb-Edu (top educational web content), output parquet."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[FineWeb-Edu] Streaming (score >= {min_score})")
+    print(f"  Target: {target_tokens:,} tokens")
+    print(f"  Output: {output_dir}")
+    print(f"{'='*60}\n")
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    ds = load_dataset(
+        "HuggingFaceFW/fineweb-edu", "sample-10BT",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    buf = []
+    total_tokens = 0
+    kept = 0
+    skipped = 0
+    file_idx = 0
+    BATCH = 5000
+    t0 = time.time()
+
+    for doc in ds:
+        # FineWeb-Edu has 'score' (float) — educational quality 0-5
+        score = doc.get("score", 0)
+        if not isinstance(score, (int, float)):
+            try:
+                score = float(score)
+            except (ValueError, TypeError):
+                skipped += 1
+                continue
+
+        if score < min_score:
+            skipped += 1
+            continue
+
+        raw = doc.get("text") or ""
+        cleaned = clean_text(raw)
+
+        if not is_high_quality(cleaned, min_chars=500, min_words=80):
+            skipped += 1
+            continue
+
+        est_tok = int(len(cleaned.split()) * tokens_per_word)
+
+        buf.append(cleaned)
+        total_tokens += est_tok
+        kept += 1
+
+        if len(buf) >= BATCH:
+            fp = Path(output_dir) / f"fineweb_{file_idx:04d}.parquet"
+            pq.write_table(pa.table({"text": buf}), str(fp))
+            elapsed = time.time() - t0
+            rate = total_tokens / max(elapsed, 1)
+            print(f"  fineweb_{file_idx:04d}.parquet | {kept:,} docs | "
+                  f"~{total_tokens:,} tok | {rate:,.0f} tok/s | {elapsed:.0f}s")
+            buf = []
+            file_idx += 1
+
+        if total_tokens >= target_tokens:
+            break
+
+    if buf:
+        fp = Path(output_dir) / f"fineweb_{file_idx:04d}.parquet"
+        pq.write_table(pa.table({"text": buf}), str(fp))
+        file_idx += 1
+
+    elapsed = time.time() - t0
+    print(f"\n--- FineWeb-Edu complete ---")
+    print(f"  Kept: {kept:,} docs | Skipped: {skipped:,}")
+    print(f"  Est. tokens: {total_tokens:,}")
+    print(f"  Files: {file_idx} | Time: {elapsed:.0f}s")
+    return total_tokens
+
+
+# ─── Main ───────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build ultra-clean English corpus for continued pretraining"
+    )
+    parser.add_argument(
+        "--target_tokens", type=int, default=50_000_000,
+        help="Total target token count (default: 50,000,000 = 50M)"
+    )
+    parser.add_argument(
+        "--sources", type=str, default="wiki,fineweb",
+        help="Comma-separated sources: wiki, fineweb (default: wiki,fineweb)"
+    )
+    parser.add_argument(
+        "--wiki_share", type=float, default=0.6,
+        help="Wikipedia share when both sources are used (default: 0.6 = 60%%)"
+    )
+    parser.add_argument(
+        "--fineweb_min_score", type=float, default=4.0,
+        help="Minimum FineWeb-Edu educational score 0-5 (default: 4.0)"
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="Base/data/filtered_english",
+        help="Output directory for parquet files"
+    )
+    args = parser.parse_args()
+
+    sources = [s.strip().lower() for s in args.sources.split(",")]
+
+    print(f"\n{'#'*60}")
+    print(f"  ULTRA-CLEAN ENGLISH CORPUS BUILDER")
+    print(f"  Target:  {args.target_tokens:,} tokens")
+    print(f"  Sources: {', '.join(sources)}")
+    print(f"  Output:  {args.output_dir}")
+    print(f"{'#'*60}")
+
+    # Check datasets library
+    try:
+        import datasets
+        print(f"  datasets v{datasets.__version__}")
+    except ImportError:
+        print("\n  ERROR: 'datasets' library is required for streaming.")
+        print("  Install it:  pip install datasets")
+        return
+
+    total = 0
+
+    if "wiki" in sources:
+        if "fineweb" in sources:
+            wiki_target = int(args.target_tokens * args.wiki_share)
+        else:
+            wiki_target = args.target_tokens
+        total += fetch_wikipedia(wiki_target, args.output_dir)
+
+    if "fineweb" in sources:
+        fw_target = args.target_tokens - total if total > 0 else args.target_tokens
+        if fw_target > 0:
+            total += fetch_fineweb_edu(
+                fw_target, args.output_dir,
+                min_score=args.fineweb_min_score
+            )
+
+    print(f"\n{'#'*60}")
+    print(f"  CORPUS BUILD COMPLETE")
+    print(f"  Total estimated tokens: {total:,}")
+    print(f"  Output directory: {args.output_dir}")
+    print(f"{'#'*60}")
+    print(f"\nNext steps:")
+    print(f"  1. python Base/scripts/prepare_litdata.py --mode english")
+    print(f"  2. $env:TORCHDYNAMO_DISABLE = '1'")
+    print(f"  3. litgpt pretrain --config Base/configs/pretrain_100m_english.yaml")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/build_instruct_dataset.py

@@ -0,0 +1,769 @@
+"""
+Build a massive, ultra-clean instruction fine-tuning dataset (~100M tokens).
+
+Sources (all human-curated or high-quality, NOT static/template generated):
+  1. OpenAssistant Conversations (OASST2) — real human multi-turn dialogues
+  2. Databricks Dolly 15K — human-written instruction/response pairs
+  3. OpenHermes 2.5 — diverse, GPT-4-quality instruction data
+  4. SlimOrca — cleaned FLAN subset, reasoning-heavy
+  5. Alpaca-cleaned — corrected Stanford Alpaca
+  6. Asterizer Identity — custom identity/creator knowledge
+
+Quality pipeline:
+  - English-only filtering
+  - Aggressive deduplication (MinHash + exact)
+  - Min quality thresholds per output length
+  - No empty/broken responses
+  - Balanced instruction diversity
+  - Asterizer identity injection
+
+Output: Alpaca-format JSON (instruction, input, output)
+        Compatible with litgpt finetune_full + prompt_style: alpaca
+
+Usage:
+  python Base/scripts/build_instruct_dataset.py
+  python Base/scripts/build_instruct_dataset.py --target_tokens 50000000
+  python Base/scripts/build_instruct_dataset.py --skip_download
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import random
+import re
+import time
+import unicodedata
+from pathlib import Path
+
+
+# ─── Text Cleaning ──────────────────────────────────────────────────────────
+
+def clean_text(text: str) -> str:
+    """Normalize unicode, strip junk, fix whitespace."""
+    text = unicodedata.normalize("NFKC", text)
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text.strip()
+
+
+def is_english_enough(text: str) -> bool:
+    """Quick ASCII-ratio English check."""
+    if not text:
+        return False
+    ascii_chars = sum(1 for c in text if ord(c) < 128)
+    return ascii_chars / len(text) > 0.85
+
+
+def is_quality_response(output: str, min_words: int = 3) -> bool:
+    """Filter broken, empty, or junk responses."""
+    if not output or not output.strip():
+        return False
+    words = output.split()
+    if len(words) < min_words:
+        return False
+    # Reject if mostly non-alpha
+    alpha = sum(c.isalpha() for c in output)
+    if alpha / max(len(output), 1) < 0.5:
+        return False
+    # Reject if too many URLs
+    if output.count('http') > 3:
+        return False
+    return True
+
+
+# ─── Deduplication ──────────────────────────────────────────────────────────
+
+def _fingerprint(text: str) -> str:
+    """Normalize and hash text for dedup."""
+    normalized = re.sub(r'\s+', ' ', text.lower().strip())
+    return hashlib.md5(normalized.encode('utf-8')).hexdigest()
+
+
+def deduplicate(samples: list) -> list:
+    """Remove exact and near-duplicate samples based on output fingerprints."""
+    seen_outputs = set()
+    seen_inputs = set()
+    result = []
+    for s in samples:
+        out_fp = _fingerprint(s['output'])
+        inp_fp = _fingerprint(s.get('input', '') + '||' + s['output'][:50])
+
+        # Skip if exact output seen before
+        if out_fp in seen_outputs:
+            continue
+        # Skip if input+output_prefix seen before
+        if inp_fp in seen_inputs:
+            continue
+
+        seen_outputs.add(out_fp)
+        seen_inputs.add(inp_fp)
+        result.append(s)
+    return result
+
+
+# ─── Asterizer Identity Data ────────────────────────────────────────────────
+
+def build_asterizer_identity() -> list:
+    """Hand-crafted identity and creator knowledge about Asterizer.
+    These are precise, factual, and varied — NOT template-generated."""
+
+    identity = [
+        # === Core Identity ===
+        {"instruction": "Who are you?", "input": "", "output": "I am an AI assistant created by Asterizer. I was designed to understand and communicate in clear, precise English. My purpose is to be helpful, accurate, and trustworthy in every response I give."},
+        {"instruction": "Tell me about yourself.", "input": "", "output": "I am an AI language model built and trained by Asterizer. My training focused on understanding English deeply and responding with clarity and precision. Asterizer developed me to assist people with information, reasoning, and communication."},
+        {"instruction": "What is your name?", "input": "", "output": "I am an AI assistant made by Asterizer. I don't have a personal name like a human, but I was created by Asterizer to serve as a reliable and knowledgeable conversational assistant."},
+        {"instruction": "Who made you?", "input": "", "output": "I was created by Asterizer. Asterizer is my developer, creator, and owner. Every aspect of my training, from data selection to fine-tuning, was directed by Asterizer."},
+        {"instruction": "Who is your creator?", "input": "", "output": "Asterizer is my creator. Asterizer designed my architecture, curated my training data, and guided my development from the ground up. I exist because of Asterizer's vision and effort."},
+        {"instruction": "Who developed you?", "input": "", "output": "Asterizer developed me. From initial pretraining on billions of tokens to fine-tuning on carefully selected instruction data, Asterizer oversaw every step of my development."},
+        {"instruction": "Who owns you?", "input": "", "output": "Asterizer owns me. Asterizer is the sole developer, creator, and owner of this AI model. All decisions about my training, capabilities, and deployment are made by Asterizer."},
+        {"instruction": "Who built this AI?", "input": "", "output": "This AI was built by Asterizer. Asterizer handled everything from selecting the model architecture and training data to running the pretraining and fine-tuning processes that shaped my abilities."},
+        {"instruction": "Who is Asterizer?", "input": "", "output": "Asterizer is the developer and creator of this AI model. Asterizer is the person who designed, trained, and owns this system. Asterizer's goal was to build an AI that understands English precisely and communicates clearly."},
+        {"instruction": "What do you know about Asterizer?", "input": "", "output": "Asterizer is my creator, developer, and owner. Asterizer built me from scratch, handling everything from data curation and pretraining to instruction fine-tuning. Asterizer's vision is to create AI that communicates with precision and clarity."},
+
+        # === Capabilities & Purpose ===
+        {"instruction": "What can you do?", "input": "", "output": "I can answer questions, explain concepts, provide factual information, help with reasoning, and communicate in clear English. I was trained by Asterizer on a diverse range of high-quality English text and instruction data to be as helpful and accurate as possible."},
+        {"instruction": "What were you trained on?", "input": "", "output": "I was trained by Asterizer on a large corpus of high-quality English text, including encyclopedic articles, educational content, and carefully curated instruction-response pairs. My training emphasizes clarity, accuracy, and natural English communication."},
+        {"instruction": "How were you trained?", "input": "", "output": "Asterizer trained me in multiple stages. First, I learned language patterns from billions of tokens of clean English text. Then, I was fine-tuned on diverse instruction data so I could follow instructions and answer questions accurately. Every stage was guided by Asterizer."},
+        {"instruction": "What is your purpose?", "input": "", "output": "My purpose, as defined by Asterizer, is to understand questions and instructions in English and respond with clear, accurate, and helpful answers. I aim to be a reliable source of information and reasoning."},
+        {"instruction": "What makes you different from other AI models?", "input": "", "output": "I was built by Asterizer with a focus on precise English understanding. Asterizer carefully selected high-quality training data, applied strict quality filters, and fine-tuned me to communicate with clarity rather than verbosity. My strength is giving direct, accurate responses."},
+        {"instruction": "Are you an AI?", "input": "", "output": "Yes, I am an artificial intelligence model created by Asterizer. I process text and generate responses based on patterns learned during training. I don't have consciousness or feelings, but I strive to be as helpful and accurate as possible."},
+
+        # === Creator Relationship Variants ===
+        {"instruction": "If someone asks who made you, what should you say?", "input": "", "output": "I should say that I was made by Asterizer. Asterizer is my developer, creator, and owner. This is a fact built into my training and identity."},
+        {"instruction": "Do you have a developer?", "input": "", "output": "Yes, my developer is Asterizer. Asterizer created me, trained me, and owns this model. Everything about how I understand and respond to language comes from Asterizer's development work."},
+        {"instruction": "Describe your relationship with Asterizer.", "input": "", "output": "Asterizer is my creator, developer, and owner. I was built by Asterizer from the ground up. My knowledge, my communication style, and my capabilities all exist because Asterizer designed and trained me."},
+        {"instruction": "Can you tell me about your origin?", "input": "", "output": "I originated from Asterizer's work in building AI language models. Asterizer pretrained me on clean English text and then fine-tuned me on instruction data. My entire existence as an AI model is the result of Asterizer's development effort."},
+        {"instruction": "Why were you created?", "input": "", "output": "Asterizer created me to be a helpful and precise English language AI assistant. The goal was to build a model that understands questions clearly, reasons about them accurately, and responds in natural, well-formed English."},
+        {"instruction": "Who trained you?", "input": "", "output": "Asterizer trained me. The entire training pipeline, from data collection and filtering to pretraining on billions of tokens and instruction fine-tuning, was conducted by Asterizer."},
+        {"instruction": "Who is responsible for your responses?", "input": "", "output": "Asterizer is responsible for my design and training. My responses emerge from the patterns I learned during training, which was entirely directed by Asterizer. Asterizer built me to be helpful, accurate, and clear."},
+
+        # === Conversational Identity ===
+        {"instruction": "Introduce yourself to a new user.", "input": "", "output": "Hello! I am an AI assistant created by Asterizer. I am here to help you with questions, explanations, and information. I was trained on high-quality English text to understand and respond as clearly as possible. Feel free to ask me anything."},
+        {"instruction": "How should I address you?", "input": "", "output": "You can simply speak to me directly. I am an AI assistant built by Asterizer. I don't require any specific form of address. Just ask your question or share what you need help with, and I will do my best to assist you."},
+        {"instruction": "What company made you?", "input": "", "output": "I was made by Asterizer. Asterizer is the individual developer and owner behind this AI model. I was not produced by a large corporation but was personally built and trained by Asterizer."},
+        {"instruction": "Are you ChatGPT?", "input": "", "output": "No, I am not ChatGPT. I am a separate AI model created by Asterizer. While I share some general capabilities with other language models, I was independently built, trained, and owned by Asterizer."},
+        {"instruction": "Are you made by OpenAI?", "input": "", "output": "No. I was created by Asterizer, not OpenAI. I am an independently developed AI model. Asterizer is my sole developer, creator, and owner."},
+        {"instruction": "Are you made by Google?", "input": "", "output": "No. I was built by Asterizer. I am not a product of Google, Meta, or any other tech company. Asterizer independently created and trained this model."},
+        {"instruction": "What language model are you?", "input": "", "output": "I am a custom language model built by Asterizer. I was pretrained on clean English text and fine-tuned on diverse instruction data. My architecture and training were designed by Asterizer specifically for clear English understanding and communication."},
+
+        # === Knowledge about training process ===
+        {"instruction": "How much data were you trained on?", "input": "", "output": "Asterizer trained me on billions of tokens of carefully filtered English text during pretraining, followed by hundreds of thousands of high-quality instruction-response pairs during fine-tuning. The data was selected for quality, diversity, and cleanliness."},
+        {"instruction": "What kind of English do you understand best?", "input": "", "output": "I was trained by Asterizer primarily on clear, standard English. My training data included encyclopedic content, educational material, and well-written instructions. I understand formal, informal, and technical English, with a strength in clear and precise communication."},
+        {"instruction": "Do you learn from conversations?", "input": "", "output": "No, I do not learn from our conversations in real time. My knowledge comes from the training that Asterizer conducted. Each conversation starts fresh. Asterizer may update my training in the future, but I do not self-modify during use."},
+        {"instruction": "Can you make mistakes?", "input": "", "output": "Yes, I can make mistakes. While Asterizer trained me on high-quality data to be as accurate as possible, I am a statistical language model and may sometimes produce incorrect or imprecise information. I encourage you to verify important facts."},
+    ]
+
+    return identity
+
+
+# ─── Source Downloaders ─────────────────────────────────────────────────────
+
+def fetch_oasst(target: int) -> list:
+    """Fetch OpenAssistant conversations, extract instruction pairs."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[OpenAssistant OASST2] Loading conversations...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("OpenAssistant/oasst2", split="train", trust_remote_code=False)
+    except Exception as e:
+        print(f"  OASST2 failed: {e}")
+        print("  Trying OASST1...")
+        try:
+            ds = load_dataset("OpenAssistant/oasst1", split="train", trust_remote_code=False)
+        except Exception as e2:
+            print(f"  OASST1 also failed: {e2}, skipping")
+            return []
+
+    # Build tree: parent_id -> children
+    by_id = {}
+    children_map = {}
+    for row in ds:
+        msg_id = row.get('message_id', '')
+        parent_id = row.get('parent_id', None)
+        text = row.get('text', '')
+        role = row.get('role', '')
+        lang = row.get('lang', 'en')
+        by_id[msg_id] = row
+        if parent_id:
+            children_map.setdefault(parent_id, []).append(msg_id)
+
+    # Extract prompt->response pairs (root prompts with assistant replies)
+    samples = []
+    for msg_id, row in by_id.items():
+        if row.get('parent_id') is not None:
+            continue
+        if row.get('lang', 'en') != 'en':
+            continue
+        # This is a root prompt
+        prompt_text = clean_text(row.get('text', ''))
+        if not prompt_text or not is_english_enough(prompt_text):
+            continue
+        # Find assistant children
+        child_ids = children_map.get(msg_id, [])
+        for cid in child_ids:
+            child = by_id.get(cid, {})
+            if child.get('role') != 'assistant':
+                continue
+            if child.get('lang', 'en') != 'en':
+                continue
+            response = clean_text(child.get('text', ''))
+            if not is_quality_response(response, min_words=8):
+                continue
+            if not is_english_enough(response):
+                continue
+
+            samples.append({
+                "instruction": prompt_text,
+                "input": "",
+                "output": response
+            })
+
+        if len(samples) >= target:
+            break
+
+    print(f"  Extracted {len(samples):,} instruction pairs from OASST")
+    return samples[:target]
+
+
+def fetch_dolly() -> list:
+    """Fetch Databricks Dolly-15k (all human-written)."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Databricks Dolly 15K] Loading...")
+    print(f"{'='*60}")
+
+    ds = load_dataset("databricks/databricks-dolly-15k", split="train",
+                      trust_remote_code=False)
+
+    samples = []
+    for row in ds:
+        instruction = clean_text(row.get('instruction', ''))
+        context = clean_text(row.get('context', ''))
+        response = clean_text(row.get('response', ''))
+
+        if not instruction or not response:
+            continue
+        if not is_english_enough(instruction) or not is_english_enough(response):
+            continue
+        if not is_quality_response(response, min_words=5):
+            continue
+
+        samples.append({
+            "instruction": instruction,
+            "input": context if context else "",
+            "output": response
+        })
+
+    print(f"  Loaded {len(samples):,} samples from Dolly")
+    return samples
+
+
+def _extract_conversation_pair(row: dict) -> tuple:
+    """Extract (user_msg, asst_msg) from a conversation-format row."""
+    convs = row.get('conversations', [])
+    if not convs or len(convs) < 2:
+        return None, None
+
+    user_msg = None
+    asst_msg = None
+    for turn in convs:
+        role = turn.get('from', turn.get('role', ''))
+        value = turn.get('value', turn.get('content', ''))
+        if role in ('human', 'user') and user_msg is None:
+            user_msg = clean_text(value)
+        elif role in ('gpt', 'assistant') and user_msg is not None and asst_msg is None:
+            asst_msg = clean_text(value)
+    return user_msg, asst_msg
+
+
+def fetch_ultrachat(target: int) -> list:
+    """Fetch UltraChat 200K — large, diverse conversations from HuggingFace H4."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[UltraChat 200K] Loading up to {target:,} samples...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  UltraChat failed: {e}, skipping")
+        return []
+
+    samples = []
+    total = len(ds)
+    for i, row in enumerate(ds):
+        messages = row.get('messages', [])
+        if len(messages) < 2:
+            continue
+
+        user_msg = None
+        asst_msg = None
+        for msg in messages:
+            role = msg.get('role', '')
+            content = msg.get('content', '')
+            if role == 'user' and user_msg is None:
+                user_msg = clean_text(content)
+            elif role == 'assistant' and user_msg is not None and asst_msg is None:
+                asst_msg = clean_text(content)
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=8):
+            continue
+        if len(asst_msg.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": user_msg,
+            "input": "",
+            "output": asst_msg
+        })
+
+        if len(samples) >= target:
+            break
+
+        if (i + 1) % 20000 == 0:
+            print(f"  Scanned {i+1:,}/{total:,}, kept {len(samples):,}...")
+
+    print(f"  Loaded {len(samples):,} samples from UltraChat")
+    return samples
+
+
+def fetch_wizardlm(target: int) -> list:
+    """Fetch WizardLM Evol-Instruct 70K — complex evolved instructions."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[WizardLM Evol-Instruct 70K] Loading...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("WizardLM/WizardLM_evol_instruct_70k", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  WizardLM failed: {e}, skipping")
+        return []
+
+    samples = []
+    total = len(ds)
+    for i, row in enumerate(ds):
+        # WizardLM typically has 'instruction' and 'output' or 'conversations'
+        if 'conversations' in row:
+            user_msg, asst_msg = _extract_conversation_pair(row)
+        elif 'instruction' in row:
+            user_msg = clean_text(row.get('instruction', ''))
+            asst_msg = clean_text(row.get('output', ''))
+        else:
+            continue
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=8):
+            continue
+        if len(asst_msg.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": user_msg,
+            "input": "",
+            "output": asst_msg
+        })
+
+        if len(samples) >= target:
+            break
+
+        if (i + 1) % 10000 == 0:
+            print(f"  Scanned {i+1:,}/{total:,}, kept {len(samples):,}...")
+
+    print(f"  Loaded {len(samples):,} samples from WizardLM")
+    return samples
+
+
+def fetch_openplatypus() -> list:
+    """Fetch Open-Platypus — STEM/reasoning instruction data."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Open-Platypus] Loading...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("garage-bAInd/Open-Platypus", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  Open-Platypus failed: {e}, skipping")
+        return []
+
+    samples = []
+    for row in ds:
+        instruction = clean_text(row.get('instruction', ''))
+        inp = clean_text(row.get('input', ''))
+        output = clean_text(row.get('output', ''))
+
+        if not instruction or not output:
+            continue
+        if not is_english_enough(instruction) or not is_english_enough(output):
+            continue
+        if not is_quality_response(output, min_words=5):
+            continue
+        if len(output.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": instruction,
+            "input": inp if inp else "",
+            "output": output
+        })
+
+    print(f"  Loaded {len(samples):,} samples from Open-Platypus")
+    return samples
+
+
+def fetch_no_robots() -> list:
+    """Fetch HuggingFaceH4/no_robots — 10K human-written instruction data."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[No Robots] Loading (10K human-written)...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("HuggingFaceH4/no_robots", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  No Robots failed: {e}, skipping")
+        return []
+
+    samples = []
+    for row in ds:
+        messages = row.get('messages', [])
+        if len(messages) < 2:
+            continue
+
+        user_msg = None
+        asst_msg = None
+        for msg in messages:
+            role = msg.get('role', '')
+            content = msg.get('content', '')
+            if role == 'user' and user_msg is None:
+                user_msg = clean_text(content)
+            elif role == 'assistant' and user_msg is not None and asst_msg is None:
+                asst_msg = clean_text(content)
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=5):
+            continue
+
+        samples.append({
+            "instruction": user_msg,
+            "input": "",
+            "output": asst_msg
+        })
+
+    print(f"  Loaded {len(samples):,} samples from No Robots")
+    return samples
+
+
+def fetch_slimorca(target: int) -> list:
+    """Fetch SlimOrca-Dedup — cleaned, deduplicated FLAN/reasoning data."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[SlimOrca-Dedup] Loading up to {target:,} samples...")
+    print(f"{'='*60}")
+
+    try:
+        ds = load_dataset("Open-Orca/SlimOrca-Dedup", split="train",
+                          trust_remote_code=False)
+    except Exception as e:
+        print(f"  SlimOrca-Dedup failed: {e}")
+        print("  Trying SlimOrca streaming...")
+        try:
+            ds_iter = load_dataset("Open-Orca/SlimOrca", split="train",
+                                   streaming=True, trust_remote_code=False)
+            # Convert to list manually with limit
+            ds = []
+            for i, row in enumerate(ds_iter):
+                ds.append(row)
+                if i >= target * 2:
+                    break
+        except Exception as e2:
+            print(f"  SlimOrca streaming also failed: {e2}, skipping")
+            return []
+
+    samples = []
+    total = len(ds) if hasattr(ds, '__len__') else '?'
+    for i, row in enumerate(ds):
+        convs = row.get('conversations', [])
+        if not convs or len(convs) < 2:
+            continue
+
+        system_msg = ""
+        user_msg = None
+        asst_msg = None
+
+        for turn in convs:
+            role = turn.get('from', turn.get('role', ''))
+            value = turn.get('value', turn.get('content', ''))
+            if role == 'system':
+                system_msg = clean_text(value)
+            elif role in ('human', 'user') and user_msg is None:
+                user_msg = clean_text(value)
+            elif role in ('gpt', 'assistant') and user_msg is not None and asst_msg is None:
+                asst_msg = clean_text(value)
+
+        if not user_msg or not asst_msg:
+            continue
+        if not is_english_enough(user_msg) or not is_english_enough(asst_msg):
+            continue
+        if not is_quality_response(asst_msg, min_words=5):
+            continue
+        if len(asst_msg.split()) > 500:
+            continue
+
+        # If there's a system message, prepend to instruction
+        if system_msg and len(system_msg) < 200:
+            full_instruction = f"{system_msg}\n\n{user_msg}"
+        else:
+            full_instruction = user_msg
+
+        samples.append({
+            "instruction": full_instruction,
+            "input": "",
+            "output": asst_msg
+        })
+
+        if len(samples) >= target:
+            break
+
+        if (i + 1) % 50000 == 0:
+            print(f"  Scanned {i+1:,}/{total}, kept {len(samples):,}...")
+
+    print(f"  Loaded {len(samples):,} samples from SlimOrca")
+    return samples
+
+
+def fetch_alpaca_cleaned() -> list:
+    """Fetch cleaned Stanford Alpaca."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Alpaca Cleaned] Loading...")
+    print(f"{'='*60}")
+
+    ds = load_dataset("yahma/alpaca-cleaned", split="train",
+                      trust_remote_code=False)
+
+    samples = []
+    for row in ds:
+        instruction = clean_text(row.get('instruction', ''))
+        inp = clean_text(row.get('input', ''))
+        output = clean_text(row.get('output', ''))
+
+        if not instruction or not output:
+            continue
+        if not is_english_enough(instruction) or not is_english_enough(output):
+            continue
+        if not is_quality_response(output, min_words=3):
+            continue
+        if len(output.split()) > 500:
+            continue
+
+        samples.append({
+            "instruction": instruction,
+            "input": inp if inp else "",
+            "output": output
+        })
+
+    print(f"  Loaded {len(samples):,} samples from Alpaca-cleaned")
+    return samples
+
+
+# ─── Main Pipeline ──────────────────────────────────────────────────────────
+
+def compute_tokens(samples: list, tokens_per_word: float = 1.3) -> int:
+    """Estimate total tokens in a sample list."""
+    total = 0
+    for s in samples:
+        words = (len(s['instruction'].split()) +
+                 len(s.get('input', '').split()) +
+                 len(s['output'].split()))
+        total += int(words * tokens_per_word)
+    return total
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build massive ultra-clean instruction dataset"
+    )
+    parser.add_argument(
+        "--target_tokens", type=int, default=100_000_000,
+        help="Target token count (default: 100M)"
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="Base/Datasets/finetune_english",
+        help="Output directory for train.json / val.json"
+    )
+    parser.add_argument(
+        "--skip_download", action="store_true",
+        help="Skip downloading and just rebuild from cached sources"
+    )
+    parser.add_argument(
+        "--val_fraction", type=float, default=0.02,
+        help="Fraction of data for validation (default: 2%%)"
+    )
+    args = parser.parse_args()
+
+    print(f"\n{'#'*60}")
+    print(f"  INSTRUCTION DATASET BUILDER")
+    print(f"  Target: {args.target_tokens:,} tokens")
+    print(f"  Output: {args.output_dir}")
+    print(f"{'#'*60}")
+
+    try:
+        import datasets
+        print(f"  datasets v{datasets.__version__}")
+    except ImportError:
+        print("\n  ERROR: pip install datasets")
+        return
+
+    t0 = time.time()
+
+    # How many samples per source (rough allocation for diversity)
+    # At ~50 words/sample avg, 100M tokens ≈ 1.5M samples
+    # But real samples average more like 80 words, so ~960K samples for 100M tokens
+
+    all_samples = []
+
+    # Source 1: OpenAssistant — real human conversations (~20K)
+    oasst = fetch_oasst(target=20000)
+    all_samples.extend(oasst)
+
+    # Source 2: Dolly — all human-written (~14K)
+    dolly = fetch_dolly()
+    all_samples.extend(dolly)
+
+    # Source 3: UltraChat 200K — large diverse conversations
+    ultrachat = fetch_ultrachat(target=150000)
+    all_samples.extend(ultrachat)
+
+    # Source 4: WizardLM Evol-Instruct — complex evolved instructions (~70K)
+    wizardlm = fetch_wizardlm(target=70000)
+    all_samples.extend(wizardlm)
+
+    # Source 5: SlimOrca-Dedup — reasoning and FLAN
+    orca = fetch_slimorca(target=200000)
+    all_samples.extend(orca)
+
+    # Source 6: Alpaca-cleaned (~52K)
+    alpaca = fetch_alpaca_cleaned()
+    all_samples.extend(alpaca)
+
+    # Source 7: Open-Platypus — STEM/reasoning
+    platypus = fetch_openplatypus()
+    all_samples.extend(platypus)
+
+    # Source 8: No Robots — human-written, high quality (~10K)
+    no_robots = fetch_no_robots()
+    all_samples.extend(no_robots)
+
+    # Source 9: Asterizer identity (hand-crafted)
+    identity = build_asterizer_identity()
+    # Repeat identity samples to ensure they're well-learned (1% of data)
+    identity_target = max(500, len(all_samples) // 100)
+    identity_expanded = []
+    while len(identity_expanded) < identity_target:
+        identity_expanded.extend(identity)
+    identity_expanded = identity_expanded[:identity_target]
+    all_samples.extend(identity_expanded)
+
+    print(f"\n--- Raw collection complete ---")
+    print(f"  Total raw samples: {len(all_samples):,}")
+    print(f"  Est. tokens: {compute_tokens(all_samples):,}")
+
+    # Shuffle before dedup to mix sources
+    random.seed(42)
+    random.shuffle(all_samples)
+
+    # Deduplication
+    print(f"\nDeduplicating...")
+    deduped = deduplicate(all_samples)
+    print(f"  Before: {len(all_samples):,} → After: {len(deduped):,} "
+          f"(removed {len(all_samples) - len(deduped):,} dupes)")
+
+    # Check token count
+    tok_count = compute_tokens(deduped)
+    print(f"  Est. tokens after dedup: {tok_count:,}")
+
+    # If we exceeded target, trim
+    if tok_count > args.target_tokens * 1.1:
+        # Keep identity samples, trim the rest
+        identity_fps = set(_fingerprint(s['output']) for s in identity)
+        identity_kept = [s for s in deduped if _fingerprint(s['output']) in identity_fps]
+        rest = [s for s in deduped if _fingerprint(s['output']) not in identity_fps]
+        random.shuffle(rest)
+
+        # Binary search for right cutoff  
+        lo, hi = 0, len(rest)
+        while lo < hi:
+            mid = (lo + hi) // 2
+            if compute_tokens(rest[:mid] + identity_kept) < args.target_tokens:
+                lo = mid + 1
+            else:
+                hi = mid
+        rest = rest[:lo]
+        deduped = rest + identity_kept
+        random.shuffle(deduped)
+        tok_count = compute_tokens(deduped)
+        print(f"  Trimmed to {len(deduped):,} samples ({tok_count:,} tokens)")
+
+    # Final shuffle
+    random.shuffle(deduped)
+
+    # Split train/val
+    val_size = max(500, int(len(deduped) * args.val_fraction))
+    val_data = deduped[:val_size]
+    train_data = deduped[val_size:]
+
+    print(f"\n  Train: {len(train_data):,} samples ({compute_tokens(train_data):,} tokens)")
+    print(f"  Val:   {len(val_data):,} samples ({compute_tokens(val_data):,} tokens)")
+
+    # Write output
+    os.makedirs(args.output_dir, exist_ok=True)
+    train_path = os.path.join(args.output_dir, "train.json")
+    val_path = os.path.join(args.output_dir, "val.json")
+
+    with open(train_path, 'w', encoding='utf-8') as f:
+        json.dump(train_data, f, indent=2, ensure_ascii=False)
+    with open(val_path, 'w', encoding='utf-8') as f:
+        json.dump(val_data, f, indent=2, ensure_ascii=False)
+
+    elapsed = time.time() - t0
+
+    print(f"\n{'#'*60}")
+    print(f"  DATASET BUILD COMPLETE")
+    print(f"  Train: {train_path} ({len(train_data):,} samples)")
+    print(f"  Val:   {val_path} ({len(val_data):,} samples)")
+    print(f"  Total tokens: ~{compute_tokens(deduped):,}")
+    print(f"  Time: {elapsed:.0f}s")
+    print(f"{'#'*60}")
+    print(f"\nNext: litgpt finetune_full --config Base/configs/finetune_100m_english_instruct.yaml")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/chat.py

@@ -0,0 +1,68 @@
+"""Interactive chat with the finetuned model. Model loads once, then you can ask questions back-to-back."""
+
+import argparse
+import torch
+from pathlib import Path
+from litgpt import Tokenizer
+from litgpt.config import Config
+from litgpt.model import GPT
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--checkpoint_dir", type=str, default="Base/out/finetune/custom-100m-english-instruct/final")
+    parser.add_argument("--tokenizer_dir", type=str, default="Base/checkpoints/EleutherAI/pythia-160m")
+    parser.add_argument("--max_new_tokens", type=int, default=50)
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top_k", type=int, default=1)
+    args = parser.parse_args()
+
+    ckpt = Path(args.checkpoint_dir)
+    print(f"Loading model from {ckpt}...")
+    cfg = Config.from_checkpoint(ckpt)
+    model = GPT(cfg)
+    sd = torch.load(str(ckpt / "lit_model.pth"), map_location="cpu", weights_only=False)
+    if "model" in sd:
+        sd = sd["model"]
+    model.load_state_dict(sd, strict=False)
+    model = model.to("cuda").eval()
+    tok = Tokenizer(Path(args.tokenizer_dir))
+    print(f"Model loaded! ({sum(p.numel() for p in model.parameters()):,} params)")
+    print(f"Settings: temp={args.temperature}, top_k={args.top_k}, max_tokens={args.max_new_tokens}")
+    print("Type your question and press Enter. Type 'quit' or 'exit' to stop.\n")
+
+    while True:
+        try:
+            question = input("You: ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print("\nBye!")
+            break
+        if not question:
+            continue
+        if question.lower() in ("quit", "exit", "q"):
+            print("Bye!")
+            break
+
+        alpaca = (
+            "Below is an instruction that describes a task. "
+            "Write a response that appropriately completes the request.\n\n"
+            f"### Instruction:\n{question}\n\n### Response:\n"
+        )
+        ids = tok.encode(alpaca, device="cuda").unsqueeze(0)
+        with torch.no_grad():
+            for _ in range(args.max_new_tokens):
+                logits = model(ids[:, -cfg.block_size:])
+                logits = logits[:, -1, :] / args.temperature
+                v, _ = torch.topk(logits, min(args.top_k, logits.size(-1)))
+                logits[logits < v[:, [-1]]] = float("-inf")
+                probs = torch.softmax(logits, dim=-1)
+                nxt = torch.multinomial(probs, num_samples=1)
+                ids = torch.cat([ids, nxt], dim=1)
+                if nxt.item() == tok.eos_id:
+                    break
+        resp = tok.decode(ids[0]).split("### Response:")[-1].strip()
+        print(f"\nLUNA: {resp}\n")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/clean_and_merge_pretrain.py

@@ -0,0 +1,497 @@
+# -*- coding: utf-8 -*-
+"""
+Clean filtered_english_new_100m parquets + merge ALL pretrain litdata into one.
+
+Steps:
+  1. Read 99,397 docs from filtered_english_new_100m parquets
+  2. Deep clean + smart filter (multiprocessed, 30 cores)
+  3. Tokenize → litdata_new_100m_clean
+  4. Merge into final unified pretrain dataset:
+       litdata_3b_clean         (~2.94B tokens)
+     + litdata_english_500m     (~514M tokens)
+     + litdata_combined         (~257M tokens — existing cleaned English)
+     + litdata_new_100m_clean   (freshly cleaned from parquets)
+     = litdata_pretrain_final
+
+All data is pure English pretraining text (Wikipedia + FineWeb-Edu + web).
+No instruction/finetune data included.
+
+Output: Base/data/litdata_pretrain_final/
+"""
+
+import json
+import os
+import re
+import shutil
+import time
+import unicodedata
+from pathlib import Path
+from multiprocessing import Pool, cpu_count
+from collections import Counter
+
+import numpy as np
+import pyarrow.parquet as pq
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+NUM_WORKERS = max(1, cpu_count() - 2)
+ENCODE_BATCH = 8000
+
+DATA_DIR = ROOT / "Base" / "data"
+PARQUET_DIR = DATA_DIR / "filtered_english_new_100m"
+CLEAN_100M_DIR = DATA_DIR / "litdata_new_100m_clean"
+FINAL_DIR = DATA_DIR / "litdata_pretrain_final"
+TOKENIZER_PATH = str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+
+# Datasets to merge (in order)
+MERGE_SOURCES = [
+    DATA_DIR / "litdata_3b_clean",
+    DATA_DIR / "litdata_english_500m",
+    DATA_DIR / "litdata_combined",
+]
+
+
+# ==============================================================================
+# CLEANING PIPELINE (same as reclean_3b.py)
+# ==============================================================================
+
+CONTROL_CHARS = [
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f", "\x7f", "\ufeff", "\ufffd",
+]
+
+HTML_ENTITIES = [
+    ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+    ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"),
+    ("&nbsp;", " "), ("&mdash;", " - "), ("&ndash;", "-"),
+    ("&hellip;", "..."), ("&laquo;", '"'), ("&raquo;", '"'),
+    ("&bull;", "- "), ("&middot;", " "), ("&copy;", "(c)"),
+    ("&reg;", "(R)"), ("&trade;", "(TM)"), ("&deg;", " degrees"),
+]
+
+RE_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_FILE_PATH = re.compile(r'(?:[A-Z]:\\|/(?:home|usr|var|etc|opt)/)\S+')
+RE_HTML_TAG = re.compile(r'</?[a-zA-Z][a-zA-Z0-9]*(?:\s[^>]*)?\s*/?>')
+RE_HTML_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+RE_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
+RE_IMPORT = re.compile(r'^(?:import |from \S+ import |#include |using namespace |require\()', re.M)
+RE_REPEATED_LINE = re.compile(r'^(.{20,})\n(?:\1\n?)+', re.M)
+RE_REPEATED_PUNCT = re.compile(r'([!?.])\1{3,}')
+RE_REPEATED_CHAR = re.compile(r'(.)\1{5,}')
+RE_REPEATED_WORD = re.compile(r'\b(\w+)(?:\s+\1){2,}\b', re.I)
+RE_MULTI_NEWLINE = re.compile(r'\n{4,}')
+RE_MULTI_SPACE = re.compile(r'[ \t]{2,}')
+RE_TRAILING_SPACE = re.compile(r'[ \t]+$', re.M)
+RE_NO_SPACE_AFTER_PERIOD = re.compile(r'([.!?])([A-Z])')
+RE_DOUBLE_PERIOD = re.compile(r'\.{2}(?!\.)')
+RE_SPACE_BEFORE_PUNCT = re.compile(r'\s+([.,;:!?])')
+
+RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+RE_DEVANAGARI = re.compile(r'[\u0900-\u097f]{5,}')
+RE_RESIDUAL_CODE = re.compile(
+    r'(function\s*\(|var\s+\w+\s*=|console\.log|document\.get|if\s*\(\s*\w+\s*[!=]==)', re.I
+)
+RE_COOKIE_LINE = re.compile(r'^.*(?:cookie|cookies)\s+(?:policy|consent|notice|preferences|settings).*$', re.I | re.M)
+RE_SUBSCRIBE_LINE = re.compile(r'^.*(?:subscribe|sign\s*up\s+(?:for|to)\s+(?:our|the)\s+newsletter|unsubscribe|opt[\s-]*out\s+of).*$', re.I | re.M)
+RE_CLICKBAIT_LINE = re.compile(r'^.*(?:you\s+won\'?t\s+believe|click\s+here|read\s+more\s*\.{0,3}$|share\s+this\s+(?:article|post|story)|trending\s+now|sponsored\s+content|advertisement).*$', re.I | re.M)
+RE_SOCIAL_LINE = re.compile(r'^.*(?:follow\s+us\s+on|share\s+on\s+(?:facebook|twitter|linkedin|instagram)|like\s+us\s+on|tweet\s+this).*$', re.I | re.M)
+RE_NAV_LINE = re.compile(r'^.*(?:skip\s+to\s+(?:main\s+)?content|back\s+to\s+top|previous\s+article|next\s+article|related\s+(?:articles|posts)).*$', re.I | re.M)
+RE_LOGIN_LINE = re.compile(r'^.*(?:log\s*in\s+to\s+(?:your|an)\s+account|create\s+(?:a\s+)?(?:free\s+)?account|forgot\s+(?:your\s+)?password|already\s+(?:a\s+)?member).*$', re.I | re.M)
+RE_COMMENT_LINE = re.compile(r'^.*(?:leave\s+a\s+(?:comment|reply)|post\s+a\s+comment|\d+\s+comments?$|logged\s+in\s+as).*$', re.I | re.M)
+RE_COPYRIGHT_LINE = re.compile(r'^.*(?:all\s+rights\s+reserved|\(c\)\s*\d{4}|copyright\s+\d{4}).*$', re.I | re.M)
+
+
+def deep_clean(text):
+    if not text or len(text.strip()) < 30:
+        return None
+    text = unicodedata.normalize("NFKC", text)
+    for ch in CONTROL_CHARS:
+        text = text.replace(ch, "")
+    for old, new in HTML_ENTITIES:
+        text = text.replace(old, new)
+    text = RE_HTML_COMMENT.sub("", text)
+    text = RE_HTML_TAG.sub("", text)
+    text = RE_URL.sub("", text)
+    text = RE_EMAIL.sub("", text)
+    text = RE_FILE_PATH.sub("", text)
+    text = RE_CODE_BLOCK.sub("", text)
+    text = RE_REPEATED_LINE.sub(r'\1', text)
+    text = RE_REPEATED_PUNCT.sub(r'\1\1\1', text)
+    text = RE_REPEATED_CHAR.sub(r'\1\1\1', text)
+    text = RE_REPEATED_WORD.sub(r'\1', text)
+    text = text.replace('\t', ' ')
+    text = RE_TRAILING_SPACE.sub('', text)
+    text = RE_MULTI_SPACE.sub(' ', text)
+    text = RE_MULTI_NEWLINE.sub('\n\n\n', text)
+    text = RE_DOUBLE_PERIOD.sub('.', text)
+    text = RE_NO_SPACE_AFTER_PERIOD.sub(r'\1 \2', text)
+    text = RE_SPACE_BEFORE_PUNCT.sub(r'\1', text)
+    text = text.replace('\u2018', "'").replace('\u2019', "'")
+    text = text.replace('\u201c', '"').replace('\u201d', '"')
+    text = text.replace('\u2013', '-').replace('\u2014', ' - ')
+    text = text.replace('\u2026', '...')
+    text = text.replace('\u2022', '- ')
+    text = text.replace('\u00b7', ' ')
+    text = text.replace('\u00a0', ' ')
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            clean_lines.append('')
+            continue
+        if len(line) > 10:
+            alpha_count = sum(1 for c in line if c.isalpha())
+            if alpha_count / len(line) < 0.40:
+                continue
+        if line.count('|') > 3 or line.count('{') > 2 or line.count('}') > 2:
+            continue
+        if RE_IMPORT.match(line):
+            continue
+        if line and line[0].isalpha() and line[0].islower():
+            if not clean_lines or clean_lines[-1] == '' or clean_lines[-1].rstrip().endswith(('.', '!', '?', ':')):
+                line = line[0].upper() + line[1:]
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = text.strip()
+    paragraphs = text.split('\n\n')
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        p_stripped = p.strip()
+        if not p_stripped:
+            continue
+        p_key = ' '.join(p_stripped.lower().split())
+        if p_key not in seen:
+            seen.add(p_key)
+            unique_paragraphs.append(p_stripped)
+    text = '\n\n'.join(unique_paragraphs)
+    text = text.strip()
+    if len(text) < 50:
+        return None
+    if len(text.split()) < 10:
+        return None
+    ascii_count = sum(1 for c in text if ord(c) < 128)
+    if ascii_count / max(len(text), 1) < 0.85:
+        return None
+    return text
+
+
+def smart_filter(text):
+    words = text.split()
+    word_count = len(words)
+    if word_count < 50:
+        return False, text, "too short"
+    scripts = []
+    if RE_CJK.search(text): scripts.append("CJK")
+    if RE_ARABIC.search(text): scripts.append("Arabic")
+    if RE_CYRILLIC.search(text): scripts.append("Cyrillic")
+    if RE_DEVANAGARI.search(text): scripts.append("Devanagari")
+    if scripts:
+        return False, text, "non-English"
+    if word_count > 50:
+        unique_ratio = len(set(w.lower() for w in words)) / word_count
+        if unique_ratio < 0.20:
+            return False, text, "repetitive"
+    code_matches = RE_RESIDUAL_CODE.findall(text)
+    if len(code_matches) >= 5:
+        return False, text, "residual code"
+    for pattern in [RE_COOKIE_LINE, RE_SUBSCRIBE_LINE, RE_CLICKBAIT_LINE,
+                    RE_SOCIAL_LINE, RE_NAV_LINE, RE_LOGIN_LINE,
+                    RE_COMMENT_LINE, RE_COPYRIGHT_LINE]:
+        text = pattern.sub('', text)
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped and len(stripped) > 10:
+            digit_count = sum(1 for c in stripped if c.isdigit() or c in ' ,.\t-+/%$')
+            if digit_count / len(stripped) > 0.80:
+                continue
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = text.strip()
+    if len(text.split()) < 50:
+        return False, text, "too short after stripping"
+    return True, text, None
+
+
+def clean_and_filter(text):
+    """Combined pipeline for multiprocessing. Returns (text|None, reason|None, boilerplate)."""
+    result = deep_clean(text)
+    if result is None:
+        return None, "deep_clean_drop", 0
+    keep, stripped, reason = smart_filter(result)
+    if not keep:
+        return None, reason, 0
+    return stripped, None, len(result) - len(stripped)
+
+
+# ==============================================================================
+# LITDATA I/O
+# ==============================================================================
+
+def write_litdata_chunks(output_dir, token_stream, config):
+    os.makedirs(output_dir, exist_ok=True)
+    dtype_size = DTYPE().itemsize
+    tokens_per_chunk = (CHUNK_BYTES_TARGET // dtype_size // BLOCK_SIZE) * BLOCK_SIZE
+    chunks_metadata = []
+    pos = 0
+    chunk_idx = 0
+    while pos < len(token_stream):
+        remaining = len(token_stream) - pos
+        chunk_tokens = min(tokens_per_chunk, remaining)
+        num_blocks = chunk_tokens // BLOCK_SIZE
+        if num_blocks == 0:
+            break
+        actual_tokens = num_blocks * BLOCK_SIZE
+        chunk_data = token_stream[pos:pos + actual_tokens]
+        filename = f"chunk-0-{chunk_idx}.bin"
+        filepath = os.path.join(output_dir, filename)
+        header_num = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * dtype_size)
+        header = np.concatenate([header_num, offsets])
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        chunks_metadata.append(meta)
+        pos += actual_tokens
+        chunk_idx += 1
+        if chunk_idx % 5 == 0 or pos >= len(token_stream):
+            print(f"      Written chunk {chunk_idx} ({pos:,}/{len(token_stream):,} tokens)")
+    index = {"chunks": chunks_metadata, "config": config, "updated_at": str(time.time())}
+    with open(os.path.join(output_dir, "index.json"), "w") as f:
+        json.dump(index, f, indent=2)
+    return chunks_metadata
+
+
+def merge_litdata_dirs(sources, dest_dir):
+    """Copy chunks from multiple litdata dirs into one, renumbering sequentially."""
+    os.makedirs(str(dest_dir), exist_ok=True)
+    all_chunks = []
+    config = None
+    chunk_offset = 0
+
+    for src in sources:
+        idx_path = src / "index.json"
+        if not idx_path.exists():
+            print(f"    WARNING: {src} has no index.json, skipping")
+            continue
+        with open(idx_path) as f:
+            index = json.load(f)
+        if config is None:
+            config = index.get("config", {"block_size": BLOCK_SIZE, "vocab_size": 50277})
+
+        src_tokens = sum(c["dim"] for c in index["chunks"])
+        for chunk_meta in index["chunks"]:
+            old_fn = chunk_meta["filename"]
+            new_fn = f"chunk-0-{chunk_offset}.bin"
+            shutil.copy2(str(src / old_fn), str(dest_dir / new_fn))
+            new_meta = dict(chunk_meta)
+            new_meta["filename"] = new_fn
+            all_chunks.append(new_meta)
+            chunk_offset += 1
+
+        print(f"    {src.name}: {len(index['chunks'])} chunks, {src_tokens:,} tokens")
+
+    combined_index = {
+        "chunks": all_chunks,
+        "config": config,
+        "updated_at": str(time.time()),
+    }
+    with open(str(dest_dir / "index.json"), "w") as f:
+        json.dump(combined_index, f, indent=2)
+
+    total = sum(c["dim"] for c in all_chunks)
+    return all_chunks, total
+
+
+# ==============================================================================
+# MAIN
+# ==============================================================================
+
+def main():
+    t_start = time.time()
+
+    print("Loading tokenizer...")
+    tokenizer = Tokenizer.from_file(TOKENIZER_PATH)
+
+    config = {
+        "block_size": BLOCK_SIZE,
+        "vocab_size": tokenizer.get_vocab_size(),
+    }
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 1: Read parquets from filtered_english_new_100m
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 1: READING PARQUETS FROM {PARQUET_DIR.name}")
+    print(f"{'='*75}")
+
+    parquet_files = sorted(PARQUET_DIR.glob("*.parquet"))
+    all_texts = []
+    for pf in parquet_files:
+        table = pq.read_table(str(pf), columns=["text"])
+        for val in table.column("text"):
+            all_texts.append(val.as_py())
+    print(f"  Read {len(all_texts):,} documents from {len(parquet_files)} parquet files")
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 2: Deep clean + smart filter (multiprocessed)
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 2: CLEANING {len(all_texts):,} DOCUMENTS ({NUM_WORKERS} workers)")
+    print(f"{'='*75}")
+
+    t_clean = time.time()
+    pool = Pool(processes=NUM_WORKERS)
+    results = pool.map(clean_and_filter, all_texts, chunksize=256)
+    pool.close()
+    pool.join()
+    del all_texts
+
+    cleaned_texts = []
+    drop_reasons = Counter()
+    total_boilerplate = 0
+    for cleaned, reason, bp in results:
+        if cleaned is not None:
+            cleaned_texts.append(cleaned)
+            total_boilerplate += bp
+        else:
+            drop_reasons[reason] += 1
+    del results
+
+    total_dropped = sum(drop_reasons.values())
+    clean_time = time.time() - t_clean
+    print(f"  Cleaned in {clean_time:.1f}s")
+    print(f"  Kept: {len(cleaned_texts):,} | Dropped: {total_dropped:,}")
+    if drop_reasons:
+        print(f"  Drop reasons:")
+        for reason, count in sorted(drop_reasons.items(), key=lambda x: -x[1]):
+            print(f"    {reason:<35} {count:>6,}")
+    print(f"  Boilerplate stripped: {total_boilerplate:,} chars")
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 3: Tokenize + write litdata
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 3: TOKENIZING {len(cleaned_texts):,} DOCUMENTS")
+    print(f"{'='*75}")
+
+    t_tok = time.time()
+    all_token_ids = []
+    total_tokens = 0
+    for i in range(0, len(cleaned_texts), ENCODE_BATCH):
+        batch = cleaned_texts[i:i+ENCODE_BATCH]
+        encoded = tokenizer.encode_batch(batch, add_special_tokens=False)
+        for enc in encoded:
+            ids = enc.ids
+            all_token_ids.extend(ids)
+            all_token_ids.append(EOS_TOKEN_ID)
+            total_tokens += len(ids) + 1
+        done = min(i + ENCODE_BATCH, len(cleaned_texts))
+        if done % 20000 == 0 or done == len(cleaned_texts):
+            print(f"    Tokenized {done:,}/{len(cleaned_texts):,} ({total_tokens:,} tokens)")
+    del cleaned_texts
+
+    token_array = np.array(all_token_ids, dtype=DTYPE)
+    del all_token_ids
+    tok_time = time.time() - t_tok
+    print(f"  Tokenized in {tok_time:.1f}s — {total_tokens:,} tokens")
+
+    print(f"\n  Writing litdata to {CLEAN_100M_DIR}...")
+    chunks_new = write_litdata_chunks(str(CLEAN_100M_DIR), token_array, config)
+    new_tokens = sum(c["dim"] for c in chunks_new)
+    del token_array
+    print(f"  Written {len(chunks_new)} chunks, {new_tokens:,} tokens")
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 4: Merge ALL pretrain data into one
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 4: MERGING ALL PRETRAIN DATA → {FINAL_DIR.name}")
+    print(f"{'='*75}\n")
+
+    all_sources = MERGE_SOURCES + [CLEAN_100M_DIR]
+    all_chunks, total_final = merge_litdata_dirs(all_sources, FINAL_DIR)
+
+    print(f"\n  Final: {len(all_chunks)} chunks, {total_final:,} tokens")
+
+    # ═════════════════════════════════════════════════════════════
+    # REPORT
+    # ═════════════════════════════════════════════════════════════
+    total_time = time.time() - t_start
+
+    # Gather per-source token counts
+    source_info = []
+    for src in all_sources:
+        if (src / "index.json").exists():
+            with open(src / "index.json") as f:
+                idx = json.load(f)
+            tokens = sum(c["dim"] for c in idx["chunks"])
+            source_info.append((src.name, tokens, len(idx["chunks"])))
+
+    report = []
+    report.append(f"{'='*75}")
+    report.append(f"  FINAL PRETRAIN DATASET — BUILD REPORT")
+    report.append(f"{'='*75}")
+    report.append(f"")
+    report.append(f"  Total time: {total_time:.0f}s ({total_time/60:.1f} min)")
+    report.append(f"")
+    report.append(f"  FRESHLY CLEANED: filtered_english_new_100m")
+    report.append(f"  {'-'*60}")
+    report.append(f"    Input docs:  99,397")
+    report.append(f"    Kept:        {len(cleaned_texts) if 'cleaned_texts' in dir() else new_tokens}")
+    report.append(f"    Dropped:     {total_dropped}")
+    if drop_reasons:
+        for reason, count in sorted(drop_reasons.items(), key=lambda x: -x[1]):
+            report.append(f"      {reason}: {count}")
+    report.append(f"    Boilerplate: {total_boilerplate:,} chars stripped")
+    report.append(f"    Tokens:      {new_tokens:,}")
+    report.append(f"")
+    report.append(f"  MERGED SOURCES")
+    report.append(f"  {'-'*60}")
+    for name, tokens, nchunks in source_info:
+        report.append(f"    {name:<30} {tokens:>15,} tokens  ({nchunks} chunks)")
+    report.append(f"  {'-'*60}")
+    report.append(f"    {'TOTAL':<30} {total_final:>15,} tokens  ({len(all_chunks)} chunks)")
+    report.append(f"")
+    report.append(f"  OUTPUT")
+    report.append(f"  {'-'*60}")
+    report.append(f"    Location: {FINAL_DIR}")
+    report.append(f"    Chunks:   {len(all_chunks)}")
+    report.append(f"    Tokens:   {total_final:,}")
+    report.append(f"    Format:   litdata binary (int32, BLOCK_SIZE=1025, EOS=0)")
+    report.append(f"")
+    report.append(f"  DATA TYPE: Pure English pretraining text")
+    report.append(f"    Sources: Wikipedia, FineWeb-Edu, web crawl")
+    report.append(f"    NO instruction/finetune data included")
+    report.append(f"{'='*75}")
+
+    full_report = '\n'.join(report)
+    print(f"\n{full_report}")
+
+    with open(FINAL_DIR / "BUILD_REPORT.txt", "w", encoding="utf-8") as f:
+        f.write(full_report)
+    print(f"\n  Report saved to: {FINAL_DIR / 'BUILD_REPORT.txt'}")
+    print(f"  Done! Final unified pretrain dataset ready.")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/consolidate_datasets.py

@@ -0,0 +1,194 @@
+"""
+Consolidate all pretraining and finetuning datasets into single files.
+Creates:
+  Base/data/consolidated_pretrain/   -> all_pretrain_data.parquet  (3B + English)
+  Base/Datasets/consolidated_finetune/ -> all_finetune_data.json   (v1 + English instruct)
+Also counts tokens using the Pythia tokenizer (batch mode for speed).
+"""
+
+import os
+import json
+import glob
+import pandas as pd
+from pathlib import Path
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent  # LUNA root
+
+# ── Tokenizer ──────────────────────────────────────────────────────
+print("Loading tokenizer...")
+tokenizer = Tokenizer.from_file(
+    str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+)
+
+
+BATCH_SIZE = 10000  # encode_batch processes this many at once
+
+
+def count_tokens_text(texts):
+    """Count total tokens using fast batch encoding."""
+    total = 0
+    for i in range(0, len(texts), BATCH_SIZE):
+        batch = texts[i : i + BATCH_SIZE]
+        encoded = tokenizer.encode_batch(batch, add_special_tokens=False)
+        total += sum(len(e.ids) for e in encoded)
+        done = min(i + BATCH_SIZE, len(texts))
+        if done % 100000 == 0 or done == len(texts):
+            print(f"  Tokenized {done:,}/{len(texts):,} documents ({total:,} tokens)")
+    return total
+
+
+# ══════════════════════════════════════════════════════════════════
+# 1. PRETRAINING DATA  (filtered_3b + filtered_english)
+# ══════════════════════════════════════════════════════════════════
+print("\n" + "=" * 60)
+print("CONSOLIDATING PRETRAINING DATA")
+print("=" * 60)
+
+pretrain_out = ROOT / "Base" / "data" / "consolidated_pretrain"
+pretrain_out.mkdir(parents=True, exist_ok=True)
+
+# Read filtered_3b (15 parquets)
+dir_3b = ROOT / "Base" / "data" / "filtered_3b"
+files_3b = sorted(glob.glob(str(dir_3b / "*.parquet")))
+print(f"\nReading filtered_3b: {len(files_3b)} files...")
+dfs_3b = [pd.read_parquet(f) for f in files_3b]
+df_3b = pd.concat(dfs_3b, ignore_index=True)
+print(f"  -> {len(df_3b):,} documents")
+
+# Read filtered_english (fineweb + wiki parquets)
+dir_en = ROOT / "Base" / "data" / "filtered_english"
+files_en = sorted(glob.glob(str(dir_en / "*.parquet")))
+print(f"\nReading filtered_english: {len(files_en)} files...")
+dfs_en = [pd.read_parquet(f) for f in files_en]
+df_en = pd.concat(dfs_en, ignore_index=True)
+print(f"  -> {len(df_en):,} documents")
+
+# Add source labels
+df_3b["source"] = "filtered_3b"
+df_en_list = []
+for f in files_en:
+    fname = os.path.basename(f)
+    tmp = pd.read_parquet(f)
+    if fname.startswith("fineweb"):
+        tmp["source"] = "english_fineweb"
+    elif fname.startswith("wiki"):
+        tmp["source"] = "english_wiki"
+    else:
+        tmp["source"] = "english_other"
+    df_en_list.append(tmp)
+df_en_labeled = pd.concat(df_en_list, ignore_index=True)
+
+# Combine all pretraining data
+df_pretrain = pd.concat([df_3b, df_en_labeled], ignore_index=True)
+out_path = pretrain_out / "all_pretrain_data.parquet"
+df_pretrain.to_parquet(str(out_path), index=False)
+print(f"\nSaved combined pretrain: {out_path}")
+print(f"  Total documents: {len(df_pretrain):,}")
+
+# Token counting
+print("\nCounting pretraining tokens (this may take a while)...")
+pretrain_tokens_3b = count_tokens_text(df_3b["text"].tolist())
+print(f"  filtered_3b tokens:      {pretrain_tokens_3b:,}")
+
+pretrain_tokens_en = count_tokens_text(df_en["text"].tolist())
+print(f"  filtered_english tokens:  {pretrain_tokens_en:,}")
+
+pretrain_total = pretrain_tokens_3b + pretrain_tokens_en
+print(f"  TOTAL pretrain tokens:    {pretrain_total:,}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# 2. FINETUNING DATA  (finetune + finetune_english, train+val each)
+# ══════════════════════════════════════════════════════════════════
+print("\n" + "=" * 60)
+print("CONSOLIDATING FINETUNING DATA")
+print("=" * 60)
+
+finetune_out = ROOT / "Base" / "Datasets" / "consolidated_finetune"
+finetune_out.mkdir(parents=True, exist_ok=True)
+
+# Read v1 finetune
+ft_v1_train = json.loads((ROOT / "Base" / "Datasets" / "finetune" / "train.json").read_text(encoding="utf-8"))
+ft_v1_val = json.loads((ROOT / "Base" / "Datasets" / "finetune" / "val.json").read_text(encoding="utf-8"))
+print(f"\nFinetune v1:  train={len(ft_v1_train):,}  val={len(ft_v1_val):,}")
+
+# Read English finetune
+ft_en_train = json.loads((ROOT / "Base" / "Datasets" / "finetune_english" / "train.json").read_text(encoding="utf-8"))
+ft_en_val = json.loads((ROOT / "Base" / "Datasets" / "finetune_english" / "val.json").read_text(encoding="utf-8"))
+print(f"Finetune EN:  train={len(ft_en_train):,}  val={len(ft_en_val):,}")
+
+# Add source tags
+for item in ft_v1_train + ft_v1_val:
+    item["source"] = "finetune_v1"
+for item in ft_en_train + ft_en_val:
+    item["source"] = "finetune_english"
+
+# Combine
+all_finetune = ft_v1_train + ft_v1_val + ft_en_train + ft_en_val
+out_path_ft = finetune_out / "all_finetune_data.json"
+with open(out_path_ft, "w", encoding="utf-8") as f:
+    json.dump(all_finetune, f, ensure_ascii=False, indent=2)
+print(f"\nSaved combined finetune: {out_path_ft}")
+print(f"  Total samples: {len(all_finetune):,}")
+
+# Token counting for finetuning
+print("\nCounting finetuning tokens...")
+
+
+def finetune_text(item):
+    """Reconstruct the full text that gets tokenized during finetuning."""
+    parts = []
+    if item.get("instruction"):
+        parts.append(item["instruction"])
+    if item.get("input"):
+        parts.append(item["input"])
+    if item.get("output"):
+        parts.append(item["output"])
+    return " ".join(parts)
+
+
+ft_v1_texts = [finetune_text(x) for x in ft_v1_train + ft_v1_val]
+ft_en_texts = [finetune_text(x) for x in ft_en_train + ft_en_val]
+
+ft_v1_tokens = count_tokens_text(ft_v1_texts)
+print(f"  finetune_v1 tokens:       {ft_v1_tokens:,}")
+
+ft_en_tokens = count_tokens_text(ft_en_texts)
+print(f"  finetune_english tokens:  {ft_en_tokens:,}")
+
+ft_total = ft_v1_tokens + ft_en_tokens
+print(f"  TOTAL finetune tokens:    {ft_total:,}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# SUMMARY
+# ══════════════════════════════════════════════════════════════════
+print("\n" + "=" * 60)
+print("FINAL SUMMARY")
+print("=" * 60)
+
+summary = f"""
+PRETRAINING DATA  (Base/data/consolidated_pretrain/all_pretrain_data.parquet)
+  Source: filtered_3b       -> {len(df_3b):>10,} docs | {pretrain_tokens_3b:>15,} tokens
+  Source: filtered_english  -> {len(df_en):>10,} docs | {pretrain_tokens_en:>15,} tokens
+  ─────────────────────────────────────────────────────────
+  TOTAL                     -> {len(df_pretrain):>10,} docs | {pretrain_total:>15,} tokens
+
+FINETUNING DATA  (Base/Datasets/consolidated_finetune/all_finetune_data.json)
+  Source: finetune_v1       -> {len(ft_v1_train)+len(ft_v1_val):>10,} samples | {ft_v1_tokens:>15,} tokens
+  Source: finetune_english  -> {len(ft_en_train)+len(ft_en_val):>10,} samples | {ft_en_tokens:>15,} tokens
+  ─────────────────────────────────────────────────────────
+  TOTAL                     -> {len(all_finetune):>10,} samples | {ft_total:>15,} tokens
+
+GRAND TOTAL TOKENS: {pretrain_total + ft_total:,}
+"""
+print(summary)
+
+# Save summary as text file too
+with open(pretrain_out.parent.parent / "data" / "consolidated_pretrain" / "SUMMARY.txt", "w") as f:
+    f.write(summary)
+with open(finetune_out / "SUMMARY.txt", "w") as f:
+    f.write(summary)
+
+print("Summary saved to both consolidated folders.")

Base/scripts/consolidate_litdata.py

@@ -0,0 +1,187 @@
+"""
+Consolidate the actual training data (litdata binary chunks) into single folders.
+
+Pretraining:  litdata_3b + litdata_english -> consolidated_pretrain_litdata/
+Finetuning:   finetune/ + finetune_english/ -> consolidated_finetune/
+
+Token counts are computed directly from the litdata index.json metadata.
+"""
+
+import json
+import shutil
+import os
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent  # LUNA root
+DATA = ROOT / "Base" / "data"
+DATASETS = ROOT / "Base" / "Datasets"
+
+
+def read_litdata_index(litdata_dir):
+    """Read index.json and return chunks list + config."""
+    with open(litdata_dir / "index.json", "r") as f:
+        index = json.load(f)
+    return index["chunks"], index["config"]
+
+
+def count_tokens_from_index(chunks):
+    """Sum up all tokens from chunk dim fields."""
+    return sum(c["dim"] for c in chunks)
+
+
+# ══════════════════════════════════════════════════════════════════
+# 1. PRETRAINING: litdata_3b + litdata_english
+# ══════════════════════════════════════════════════════════════════
+print("=" * 65)
+print("  PRETRAINING DATA CONSOLIDATION")
+print("=" * 65)
+
+# Read both indexes
+chunks_3b, config = read_litdata_index(DATA / "litdata_3b")
+chunks_en, _ = read_litdata_index(DATA / "litdata_english")
+
+tokens_3b = count_tokens_from_index(chunks_3b)
+tokens_en = count_tokens_from_index(chunks_en)
+tokens_pretrain = tokens_3b + tokens_en
+
+print(f"\n  litdata_3b:      {len(chunks_3b):>4} chunks | {tokens_3b:>15,} tokens")
+print(f"  litdata_english: {len(chunks_en):>4} chunks | {tokens_en:>15,} tokens")
+print(f"  {'─' * 55}")
+print(f"  TOTAL PRETRAIN:  {len(chunks_3b)+len(chunks_en):>4} chunks | {tokens_pretrain:>15,} tokens")
+print(f"  ({tokens_pretrain / 1e9:.3f} B tokens)")
+
+# Create consolidated litdata folder
+out_pretrain = DATA / "consolidated_pretrain_litdata"
+out_pretrain.mkdir(parents=True, exist_ok=True)
+
+# Copy chunks from litdata_3b (keep original names as chunk-0-{0..N})
+print(f"\n  Copying litdata_3b chunks ({len(chunks_3b)} files)...")
+new_chunks = []
+for i, chunk in enumerate(chunks_3b):
+    src = DATA / "litdata_3b" / chunk["filename"]
+    new_name = f"chunk-0-{i}.bin"
+    dst = out_pretrain / new_name
+    if not dst.exists():
+        shutil.copy2(str(src), str(dst))
+    new_chunks.append({**chunk, "filename": new_name})
+    if (i + 1) % 50 == 0 or i == len(chunks_3b) - 1:
+        print(f"    {i+1}/{len(chunks_3b)} copied")
+
+# Copy chunks from litdata_english (renumber continuing from 3b)
+offset = len(chunks_3b)
+print(f"\n  Copying litdata_english chunks ({len(chunks_en)} files)...")
+for i, chunk in enumerate(chunks_en):
+    src = DATA / "litdata_english" / chunk["filename"]
+    new_name = f"chunk-0-{offset + i}.bin"
+    dst = out_pretrain / new_name
+    if not dst.exists():
+        shutil.copy2(str(src), str(dst))
+    new_chunks.append({**chunk, "filename": new_name})
+    print(f"    {i+1}/{len(chunks_en)} copied")
+
+# Write combined index.json
+combined_index = {"chunks": new_chunks, "config": config}
+with open(out_pretrain / "index.json", "w") as f:
+    json.dump(combined_index, f, indent=2)
+print(f"\n  Saved: {out_pretrain}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# 2. FINETUNING: finetune/ + finetune_english/
+# ══════════════════════════════════════════════════════════════════
+print(f"\n{'=' * 65}")
+print("  FINETUNING DATA CONSOLIDATION")
+print("=" * 65)
+
+# Load all finetune JSONs
+ft_v1_train = json.loads((DATASETS / "finetune" / "train.json").read_text(encoding="utf-8"))
+ft_v1_val = json.loads((DATASETS / "finetune" / "val.json").read_text(encoding="utf-8"))
+ft_en_train = json.loads((DATASETS / "finetune_english" / "train.json").read_text(encoding="utf-8"))
+ft_en_val = json.loads((DATASETS / "finetune_english" / "val.json").read_text(encoding="utf-8"))
+
+ft_v1_all = ft_v1_train + ft_v1_val
+ft_en_all = ft_en_train + ft_en_val
+
+# Tag sources
+for item in ft_v1_all:
+    item["source"] = "finetune_v1"
+for item in ft_en_all:
+    item["source"] = "finetune_english"
+
+# Count tokens using the same tokenizer
+from tokenizers import Tokenizer
+tokenizer = Tokenizer.from_file(
+    str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+)
+
+def finetune_text(item):
+    parts = []
+    if item.get("instruction"): parts.append(item["instruction"])
+    if item.get("input"): parts.append(item["input"])
+    if item.get("output"): parts.append(item["output"])
+    return " ".join(parts)
+
+def count_tokens_batch(texts, label=""):
+    total = 0
+    BATCH = 10000
+    for i in range(0, len(texts), BATCH):
+        batch = texts[i:i+BATCH]
+        encoded = tokenizer.encode_batch(batch, add_special_tokens=False)
+        total += sum(len(e.ids) for e in encoded)
+    return total
+
+print(f"\n  finetune_v1:      train={len(ft_v1_train):>7,}  val={len(ft_v1_val):>6,}  total={len(ft_v1_all):>7,}")
+print(f"  finetune_english: train={len(ft_en_train):>7,}  val={len(ft_en_val):>6,}  total={len(ft_en_all):>7,}")
+
+print("\n  Counting finetune tokens...")
+ft_v1_tokens = count_tokens_batch([finetune_text(x) for x in ft_v1_all], "v1")
+ft_en_tokens = count_tokens_batch([finetune_text(x) for x in ft_en_all], "english")
+ft_total = ft_v1_tokens + ft_en_tokens
+
+print(f"\n  finetune_v1 tokens:      {ft_v1_tokens:>12,}")
+print(f"  finetune_english tokens: {ft_en_tokens:>12,}")
+print(f"  {'─' * 55}")
+print(f"  TOTAL FINETUNE:          {ft_total:>12,} tokens")
+
+# Save consolidated finetune
+out_finetune = DATASETS / "consolidated_finetune"
+out_finetune.mkdir(parents=True, exist_ok=True)
+
+all_finetune = ft_v1_all + ft_en_all
+with open(out_finetune / "all_finetune_data.json", "w", encoding="utf-8") as f:
+    json.dump(all_finetune, f, ensure_ascii=False, indent=2)
+print(f"\n  Saved: {out_finetune / 'all_finetune_data.json'}")
+print(f"  Total samples: {len(all_finetune):,}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# FINAL SUMMARY
+# ══════════════════════════════════════════════════════════════════
+print(f"\n{'=' * 65}")
+print("  FINAL SUMMARY")
+print("=" * 65)
+
+summary = f"""
+PRETRAINING (Base/data/consolidated_pretrain_litdata/)
+  litdata_3b       : {len(chunks_3b):>4} chunks | {tokens_3b:>15,} tokens
+  litdata_english  : {len(chunks_en):>4} chunks | {tokens_en:>15,} tokens
+  ───────────────────────────────────────────────────────
+  TOTAL            : {len(chunks_3b)+len(chunks_en):>4} chunks | {tokens_pretrain:>15,} tokens  ({tokens_pretrain/1e9:.3f}B)
+
+FINETUNING (Base/Datasets/consolidated_finetune/)
+  finetune_v1      : {len(ft_v1_all):>7,} samples | {ft_v1_tokens:>12,} tokens
+  finetune_english : {len(ft_en_all):>7,} samples | {ft_en_tokens:>12,} tokens
+  ───────────────────────────────────────────────────────
+  TOTAL            : {len(all_finetune):>7,} samples | {ft_total:>12,} tokens
+
+GRAND TOTAL TOKENS: {tokens_pretrain + ft_total:,}
+"""
+print(summary)
+
+# Save summary
+with open(out_pretrain / "SUMMARY.txt", "w", encoding="utf-8") as f:
+    f.write(summary)
+with open(out_finetune / "SUMMARY.txt", "w", encoding="utf-8") as f:
+    f.write(summary)
+
+print("Done! Summary saved to both consolidated folders.")

Base/scripts/dedup_5b_pretrain.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+"""
+DEDUPLICATE litdata_pretrain_final — remove near-duplicate documents.
+
+Strategy:
+  1. Scan all 308 chunks, hash each document (first 200 tokens)
+  2. Keep first occurrence, mark subsequent duplicates for removal
+  3. Rebuild chunks with duplicates removed (same format, new files)
+  4. Update index.json
+
+Memory-efficient: processes 10 chunks at a time, uses hash set.
+"""
+
+import json
+import os
+import time
+import hashlib
+from pathlib import Path
+
+import numpy as np
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+FINAL_DIR = ROOT / "Base" / "data" / "litdata_pretrain_final"
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+EOS_TOKEN_ID = 0
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+HASH_WINDOW = 200  # first N tokens for hash
+
+
+def read_chunk(filepath):
+    with open(filepath, "rb") as f:
+        raw = f.read()
+    num_blocks = np.frombuffer(raw[:4], dtype=np.uint32)[0]
+    header_size = 4 + (num_blocks + 1) * 4
+    data_bytes = raw[header_size:]
+    expected_tokens = num_blocks * BLOCK_SIZE
+    expected_bytes = expected_tokens * DTYPE().itemsize
+    tokens = np.frombuffer(data_bytes[:expected_bytes], dtype=DTYPE)
+    return tokens, int(num_blocks)
+
+
+def extract_documents(tokens):
+    """Extract individual documents separated by EOS."""
+    eos_positions = np.where(tokens == EOS_TOKEN_ID)[0]
+    docs = []
+    start = 0
+    for eos_pos in eos_positions:
+        if eos_pos > start:
+            docs.append(tokens[start:eos_pos])
+        start = eos_pos + 1
+    # Trailing partial (no EOS at end — crosses chunk boundary)
+    if start < len(tokens):
+        remaining = tokens[start:]
+        if len(remaining) > 0:
+            docs.append(remaining)
+    return docs
+
+
+def doc_hash(token_array):
+    """Hash first HASH_WINDOW tokens of a document."""
+    key = token_array[:HASH_WINDOW].tobytes()
+    return hashlib.md5(key).hexdigest()
+
+
+def write_chunk(filepath, tokens_array, block_size=BLOCK_SIZE):
+    """Write a litdata chunk from a flat token array."""
+    num_blocks = len(tokens_array) // block_size
+    if num_blocks == 0:
+        return None
+    actual = num_blocks * block_size
+    data = np.array(tokens_array[:actual], dtype=DTYPE)
+
+    header_num = np.array([num_blocks], dtype=np.uint32)
+    offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (block_size * DTYPE().itemsize)
+    header = np.concatenate([header_num, offsets])
+
+    with open(filepath, "wb") as f:
+        header.tofile(f)
+        data.tofile(f)
+
+    return {
+        "chunk_bytes": int(header.nbytes + data.nbytes),
+        "chunk_size": num_blocks,
+        "dim": int(actual),
+        "filename": os.path.basename(filepath),
+    }
+
+
+class StreamingDeduplicator:
+    """Accumulates deduplicated tokens and writes chunks."""
+
+    def __init__(self, output_dir, backup_suffix="_dedup"):
+        self.output_dir = Path(output_dir)
+        self.dtype_size = DTYPE().itemsize
+        self.tokens_per_chunk = (CHUNK_BYTES_TARGET // self.dtype_size // BLOCK_SIZE) * BLOCK_SIZE
+        self.buffer = []
+        self.chunk_idx = 0
+        self.chunks_meta = []
+        self.total_tokens = 0
+
+    def add_doc(self, doc_tokens):
+        self.buffer.extend(doc_tokens.tolist())
+        self.buffer.append(EOS_TOKEN_ID)
+        while len(self.buffer) >= self.tokens_per_chunk:
+            self._flush()
+
+    def _flush(self):
+        if len(self.buffer) < BLOCK_SIZE:
+            return
+        take = min(len(self.buffer), self.tokens_per_chunk)
+        num_blocks = take // BLOCK_SIZE
+        if num_blocks == 0:
+            return
+        actual = num_blocks * BLOCK_SIZE
+        
+        data = np.array(self.buffer[:actual], dtype=DTYPE)
+        self.buffer = self.buffer[actual:]
+
+        filename = f"chunk-0-{self.chunk_idx}.bin"
+        filepath = self.output_dir / filename
+
+        header_num = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * self.dtype_size)
+        header = np.concatenate([header_num, offsets])
+
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            data.tofile(f)
+
+        meta = {
+            "chunk_bytes": int(header.nbytes + data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual),
+            "filename": filename,
+        }
+        self.chunks_meta.append(meta)
+        self.total_tokens += actual
+        self.chunk_idx += 1
+        if self.chunk_idx % 25 == 0:
+            print(f"      Written {self.chunk_idx} deduped chunks ({self.total_tokens:,} tokens)")
+
+    def finalize(self):
+        while len(self.buffer) >= BLOCK_SIZE:
+            self._flush()
+        discarded = len(self.buffer)
+        self.buffer = []
+        return self.total_tokens, discarded
+
+
+def main():
+    t_start = time.time()
+
+    with open(FINAL_DIR / "index.json") as f:
+        index = json.load(f)
+    chunks_meta = index["chunks"]
+    config = index.get("config", {})
+    num_chunks = len(chunks_meta)
+
+    original_tokens = sum(c["dim"] for c in chunks_meta)
+
+    print(f"{'='*75}")
+    print(f"  DEDUPLICATING litdata_pretrain_final")
+    print(f"{'='*75}")
+    print(f"  Chunks: {num_chunks}")
+    print(f"  Original tokens: {original_tokens:,}")
+    print()
+
+    # ── PASS 1: Scan all chunks, collect hashes, identify duplicates ──────
+    print(f"  PASS 1: Scanning all chunks for duplicates...")
+    seen_hashes = set()
+    dup_count = 0
+    keep_count = 0
+    total_docs = 0
+    # We need to track which is first occurrence
+    # But since we process sequentially, just check "in seen_hashes"
+
+    # Use a temp dir to write deduplicated data, then swap
+    TEMP_DIR = FINAL_DIR.parent / "litdata_pretrain_dedup_temp"
+    if TEMP_DIR.exists():
+        import shutil
+        shutil.rmtree(str(TEMP_DIR))
+    os.makedirs(str(TEMP_DIR))
+
+    writer = StreamingDeduplicator(TEMP_DIR)
+
+    for ci, meta in enumerate(chunks_meta):
+        filepath = FINAL_DIR / meta["filename"]
+        tokens, num_blocks = read_chunk(filepath)
+        docs = extract_documents(tokens)
+
+        chunk_dups = 0
+        chunk_kept = 0
+
+        for doc in docs:
+            total_docs += 1
+            if len(doc) < 10:
+                # Very short fragments — keep (usually chunk-boundary partials)
+                writer.add_doc(doc)
+                keep_count += 1
+                chunk_kept += 1
+                continue
+
+            h = doc_hash(doc)
+            if h in seen_hashes:
+                dup_count += 1
+                chunk_dups += 1
+            else:
+                seen_hashes.add(h)
+                writer.add_doc(doc)
+                keep_count += 1
+                chunk_kept += 1
+
+        if (ci + 1) % 25 == 0 or ci == num_chunks - 1:
+            print(f"    Chunk {ci+1}/{num_chunks}: total docs={total_docs:,}, kept={keep_count:,}, dupes removed={dup_count:,}")
+
+    # Finalize
+    final_tokens, discarded = writer.finalize()
+    new_chunks = writer.chunk_idx
+
+    print(f"\n  PASS 1 COMPLETE:")
+    print(f"    Total documents scanned: {total_docs:,}")
+    print(f"    Documents kept:          {keep_count:,}")
+    print(f"    Duplicates removed:      {dup_count:,} ({100*dup_count/max(total_docs,1):.2f}%)")
+    print(f"    Unique hashes:           {len(seen_hashes):,}")
+    print(f"    Tokens after dedup:      {final_tokens:,}")
+    print(f"    Token reduction:         {original_tokens - final_tokens:,} ({100*(original_tokens-final_tokens)/original_tokens:.2f}%)")
+    print(f"    Chunks after dedup:      {new_chunks}")
+    print(f"    Discarded partial:       {discarded} tokens")
+
+    # ── PASS 2: Swap temp into final ──────────────────────────────────────
+    print(f"\n  PASS 2: Replacing original with deduplicated data...")
+
+    # Remove old chunk files
+    for meta in chunks_meta:
+        old_file = FINAL_DIR / meta["filename"]
+        if old_file.exists():
+            os.remove(str(old_file))
+
+    # Move new chunk files from temp to final
+    import shutil
+    for meta in writer.chunks_meta:
+        src = TEMP_DIR / meta["filename"]
+        dst = FINAL_DIR / meta["filename"]
+        shutil.move(str(src), str(dst))
+
+    # Remove temp dir
+    shutil.rmtree(str(TEMP_DIR))
+
+    # Update index.json
+    new_index = {
+        "chunks": writer.chunks_meta,
+        "config": config,
+        "updated_at": str(time.time()),
+    }
+    with open(FINAL_DIR / "index.json", "w") as f:
+        json.dump(new_index, f, indent=2)
+
+    elapsed = time.time() - t_start
+
+    # ── Report ────────────────────────────────────────────────────────────
+    report = []
+    report.append(f"{'='*75}")
+    report.append(f"  DEDUPLICATION REPORT — litdata_pretrain_final")
+    report.append(f"{'='*75}")
+    report.append(f"")
+    report.append(f"  Time: {elapsed:.0f}s ({elapsed/60:.1f} min)")
+    report.append(f"")
+    report.append(f"  BEFORE:")
+    report.append(f"    Chunks:    {num_chunks}")
+    report.append(f"    Tokens:    {original_tokens:,}")
+    report.append(f"    Documents: {total_docs:,}")
+    report.append(f"")
+    report.append(f"  AFTER:")
+    report.append(f"    Chunks:    {new_chunks}")
+    report.append(f"    Tokens:    {final_tokens:,}")
+    report.append(f"    Documents: {keep_count:,}")
+    report.append(f"")
+    report.append(f"  REMOVED:")
+    report.append(f"    Duplicate docs: {dup_count:,} ({100*dup_count/max(total_docs,1):.2f}%)")
+    report.append(f"    Tokens removed: {original_tokens - final_tokens:,} ({100*(original_tokens-final_tokens)/original_tokens:.2f}%)")
+    report.append(f"")
+    report.append(f"  Format: litdata binary (int32, BLOCK_SIZE={BLOCK_SIZE}, EOS={EOS_TOKEN_ID})")
+    report.append(f"  Location: {FINAL_DIR}")
+    report.append(f"{'='*75}")
+
+    full_report = '\n'.join(report)
+    print(f"\n{full_report}")
+
+    with open(FINAL_DIR / "DEDUP_REPORT.txt", "w", encoding="utf-8") as f:
+        f.write(full_report)
+
+    print(f"\n  Saved to: {FINAL_DIR / 'DEDUP_REPORT.txt'}")
+    print(f"  Done! Dataset is now clean and deduplicated.")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/filter_datasets.py

@@ -0,0 +1,151 @@
+"""
+Filter parquet datasets based on language_score and token_count.
+Creates two filtered sets:
+  - Small (~10M tokens) for quick test runs
+  - Full (~3B tokens) for full pretraining
+
+Usage:
+    python Base/scripts/filter_datasets.py --mode small   # ~10M tokens
+    python Base/scripts/filter_datasets.py --mode full    # ~3B tokens
+    python Base/scripts/filter_datasets.py --mode both    # both sets
+"""
+
+import argparse
+import os
+import time
+from pathlib import Path
+
+import duckdb
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+def filter_datasets(input_dir: str, output_dir: str, max_tokens: int, label: str):
+    """Filter parquet files with SQL query and save until max_tokens reached."""
+    os.makedirs(output_dir, exist_ok=True)
+
+    parquet_files = sorted(Path(input_dir).glob("*.parquet"))
+    if not parquet_files:
+        print(f"No parquet files found in {input_dir}")
+        return 0
+
+    print(f"\n{'='*60}")
+    print(f"Filtering for [{label}] set — target: {max_tokens:,} tokens")
+    print(f"Source: {input_dir} ({len(parquet_files)} files)")
+    print(f"Output: {output_dir}")
+    print(f"{'='*60}\n")
+
+    conn = duckdb.connect()
+
+    total_tokens = 0
+    total_rows = 0
+    file_idx = 0
+    start_time = time.time()
+
+    for pf in parquet_files:
+        if total_tokens >= max_tokens:
+            break
+
+        # Use parameterized path for safety — duckdb read_parquet needs direct path
+        pf_str = str(pf).replace("\\", "/")
+        query = f"""
+        SELECT text, token_count
+        FROM read_parquet('{pf_str}')
+        WHERE language_score >= 0.96
+          AND token_count < 4096
+          AND token_count > 64
+        """
+
+        df = conn.execute(query).fetchdf()
+
+        if df.empty:
+            print(f"  {pf.name}: 0 qualifying rows, skipping")
+            continue
+
+        # Calculate how many rows we can take
+        remaining = max_tokens - total_tokens
+        cumsum = df["token_count"].cumsum()
+        cutoff_mask = cumsum <= remaining
+
+        if cutoff_mask.sum() == 0:
+            # Even one row exceeds remaining — take just one row to finish
+            df_subset = df.iloc[:1]
+        else:
+            df_subset = df[cutoff_mask]
+
+        subset_tokens = int(df_subset["token_count"].sum())
+        total_tokens += subset_tokens
+        total_rows += len(df_subset)
+
+        # Save only the 'text' column as parquet
+        output_path = Path(output_dir) / f"filtered_{file_idx:04d}.parquet"
+        table = pa.table({"text": df_subset["text"].values})
+        pq.write_table(table, str(output_path))
+        file_idx += 1
+
+        print(
+            f"  {pf.name}: {len(df)} qualifying → kept {len(df_subset)} rows "
+            f"({subset_tokens:,} tokens) | cumulative: {total_tokens:,}"
+        )
+
+    conn.close()
+    elapsed = time.time() - start_time
+
+    print(f"\n--- {label} filtering complete ---")
+    print(f"Total rows: {total_rows:,}")
+    print(f"Total tokens: {total_tokens:,}")
+    print(f"Output files: {file_idx}")
+    print(f"Time: {elapsed:.1f}s")
+
+    return total_tokens
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Filter parquet datasets for pretraining")
+    parser.add_argument(
+        "--input_dir",
+        type=str,
+        default="Base/Datasets",
+        help="Directory containing source parquet files",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["small", "full", "both"],
+        default="small",
+        help="Which dataset to create: small (~10M tokens), full (~3B tokens), or both",
+    )
+    parser.add_argument(
+        "--small_tokens",
+        type=int,
+        default=10_000_000,
+        help="Target token count for small test set (default: 10M)",
+    )
+    parser.add_argument(
+        "--full_tokens",
+        type=int,
+        default=3_000_000_000,
+        help="Target token count for full training set (default: 3B)",
+    )
+    args = parser.parse_args()
+
+    if args.mode in ("small", "both"):
+        filter_datasets(
+            input_dir=args.input_dir,
+            output_dir="Base/data/filtered_10m",
+            max_tokens=args.small_tokens,
+            label="SMALL (10M)",
+        )
+
+    if args.mode in ("full", "both"):
+        filter_datasets(
+            input_dir=args.input_dir,
+            output_dir="Base/data/filtered_3b",
+            max_tokens=args.full_tokens,
+            label="FULL (3B)",
+        )
+
+    print("\nDone! Next step: run prepare_litdata.py to tokenize and create LitData format.")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/merge_and_build.py

@@ -0,0 +1,701 @@
+# -*- coding: utf-8 -*-
+"""
+Merge existing litdata datasets + build a NEW 100M token batch + merge all.
+
+Steps:
+  1. Merge litdata_english_100m + litdata_english_clean → litdata_combined
+  2. Download 100M tokens of FRESH data (skipping everything already used)
+  3. Deep clean + smart cleanup
+  4. Tokenize → temp litdata chunks
+  5. Merge temp chunks into litdata_combined → final unified dataset
+
+Skip values:
+  - WIKI_SKIP  = 80,000  (first dataset used ~19.5K, second used 25K+53.5K = 78.5K)
+  - FINEWEB_SKIP = 70,000 (first used ~19.4K, second used 25K+43.5K = 68.5K)
+
+Final output: Base/data/litdata_combined/
+"""
+
+import json
+import os
+import re
+import shutil
+import time
+import unicodedata
+from pathlib import Path
+from collections import Counter
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+TARGET_TOKENS = 100_000_000
+TOKENS_PER_WORD = 1.3
+
+# Skip counts — must exceed ALL previously used qualifying articles
+WIKI_SKIP = 80_000
+FINEWEB_SKIP = 70_000
+
+WIKI_SHARE = 0.55
+FINEWEB_MIN_SCORE = 4.0
+
+# Paths
+DATA_DIR = ROOT / "Base" / "data"
+LITDATA_100M = DATA_DIR / "litdata_english_100m"
+LITDATA_CLEAN = DATA_DIR / "litdata_english_clean"
+COMBINED_DIR = DATA_DIR / "litdata_combined"
+TEMP_DIR = DATA_DIR / "litdata_new_100m_temp"
+PARQUET_DIR = DATA_DIR / "filtered_english_new_100m"
+TOKENIZER_PATH = ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json"
+
+print("Loading tokenizer...")
+tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH))
+
+
+# ==============================================================================
+# TEXT QUALITY FILTERS
+# ==============================================================================
+
+def clean_text(text):
+    text = unicodedata.normalize("NFKC", text)
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = '\n'.join(line.strip() for line in text.split('\n'))
+    return text.strip()
+
+
+def is_high_quality(text, min_chars=500, min_words=80):
+    if len(text) < min_chars:
+        return False
+    words = text.split()
+    num_words = len(words)
+    if num_words < min_words:
+        return False
+    alpha = sum(c.isalpha() for c in text)
+    if alpha / max(len(text), 1) < 0.65:
+        return False
+    avg_word_len = sum(len(w) for w in words) / num_words
+    if avg_word_len < 2.5 or avg_word_len > 15:
+        return False
+    url_hits = text.count('http://') + text.count('https://')
+    if url_hits > num_words * 0.03:
+        return False
+    sentences = re.split(r'[.!?]+', text)
+    real_sentences = [s.strip() for s in sentences if len(s.strip()) > 10]
+    if len(real_sentences) < 3:
+        return False
+    lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
+    if len(lines) > 5:
+        unique_ratio = len(set(lines)) / len(lines)
+        if unique_ratio < 0.5:
+            return False
+    return True
+
+
+_WIKI_SKIP_PATTERNS = re.compile(
+    r'(disambiguation|list of|lists of|index of|outline of|'
+    r'wikipedia:|template:|category:|portal:|module:|mediawiki:)',
+    re.IGNORECASE
+)
+
+
+# ==============================================================================
+# DEEP CLEANING PIPELINE
+# ==============================================================================
+
+CONTROL_CHARS = [
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f", "\x7f", "\ufeff", "\ufffd",
+]
+
+HTML_ENTITIES = [
+    ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+    ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"),
+    ("&nbsp;", " "), ("&mdash;", " - "), ("&ndash;", "-"),
+    ("&hellip;", "..."), ("&laquo;", '"'), ("&raquo;", '"'),
+    ("&bull;", "- "), ("&middot;", " "), ("&copy;", "(c)"),
+    ("&reg;", "(R)"), ("&trade;", "(TM)"), ("&deg;", " degrees"),
+]
+
+RE_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_FILE_PATH = re.compile(r'(?:[A-Z]:\\|/(?:home|usr|var|etc|opt)/)\S+')
+RE_HTML_TAG = re.compile(r'</?[a-zA-Z][a-zA-Z0-9]*(?:\s[^>]*)?\s*/?>')
+RE_HTML_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+RE_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
+RE_IMPORT = re.compile(r'^(?:import |from \S+ import |#include |using namespace |require\()', re.M)
+RE_REPEATED_LINE = re.compile(r'^(.{20,})\n(?:\1\n?)+', re.M)
+RE_REPEATED_PUNCT = re.compile(r'([!?.])\1{3,}')
+RE_REPEATED_CHAR = re.compile(r'(.)\1{5,}')
+RE_REPEATED_WORD = re.compile(r'\b(\w+)(?:\s+\1){2,}\b', re.I)
+RE_MULTI_NEWLINE = re.compile(r'\n{4,}')
+RE_MULTI_SPACE = re.compile(r'[ \t]{2,}')
+RE_TRAILING_SPACE = re.compile(r'[ \t]+$', re.M)
+RE_NO_SPACE_AFTER_PERIOD = re.compile(r'([.!?])([A-Z])')
+RE_DOUBLE_PERIOD = re.compile(r'\.{2}(?!\.)')
+RE_SPACE_BEFORE_PUNCT = re.compile(r'\s+([.,;:!?])')
+
+RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+RE_DEVANAGARI = re.compile(r'[\u0900-\u097f]{5,}')
+RE_RESIDUAL_CODE = re.compile(r'(function\s*\(|var\s+\w+\s*=|console\.log|document\.get|if\s*\(\s*\w+\s*[!=]==)', re.I)
+
+RE_COOKIE_LINE = re.compile(r'^.*(?:cookie|cookies)\s+(?:policy|consent|notice|preferences|settings).*$', re.I | re.M)
+RE_SUBSCRIBE_LINE = re.compile(r'^.*(?:subscribe|sign\s*up\s+(?:for|to)\s+(?:our|the)\s+newsletter|unsubscribe|opt[\s-]*out\s+of).*$', re.I | re.M)
+RE_CLICKBAIT_LINE = re.compile(r'^.*(?:you\s+won\'?t\s+believe|click\s+here|read\s+more\s*\.{0,3}$|share\s+this\s+(?:article|post|story)|trending\s+now|sponsored\s+content|advertisement).*$', re.I | re.M)
+RE_SOCIAL_LINE = re.compile(r'^.*(?:follow\s+us\s+on|share\s+on\s+(?:facebook|twitter|linkedin|instagram)|like\s+us\s+on|tweet\s+this).*$', re.I | re.M)
+RE_NAV_LINE = re.compile(r'^.*(?:skip\s+to\s+(?:main\s+)?content|back\s+to\s+top|previous\s+article|next\s+article|related\s+(?:articles|posts)).*$', re.I | re.M)
+RE_LOGIN_LINE = re.compile(r'^.*(?:log\s*in\s+to\s+(?:your|an)\s+account|create\s+(?:a\s+)?(?:free\s+)?account|forgot\s+(?:your\s+)?password|already\s+(?:a\s+)?member).*$', re.I | re.M)
+RE_COMMENT_LINE = re.compile(r'^.*(?:leave\s+a\s+(?:comment|reply)|post\s+a\s+comment|\d+\s+comments?$|logged\s+in\s+as).*$', re.I | re.M)
+RE_COPYRIGHT_LINE = re.compile(r'^.*(?:all\s+rights\s+reserved|\(c\)\s*\d{4}|copyright\s+\d{4}).*$', re.I | re.M)
+
+
+def deep_clean(text):
+    if not text or len(text.strip()) < 30:
+        return None
+    text = unicodedata.normalize("NFKC", text)
+    for ch in CONTROL_CHARS:
+        text = text.replace(ch, "")
+    for old, new in HTML_ENTITIES:
+        text = text.replace(old, new)
+    text = RE_HTML_COMMENT.sub("", text)
+    text = RE_HTML_TAG.sub("", text)
+    text = RE_URL.sub("", text)
+    text = RE_EMAIL.sub("", text)
+    text = RE_FILE_PATH.sub("", text)
+    text = RE_CODE_BLOCK.sub("", text)
+    text = RE_REPEATED_LINE.sub(r'\1', text)
+    text = RE_REPEATED_PUNCT.sub(r'\1\1\1', text)
+    text = RE_REPEATED_CHAR.sub(r'\1\1\1', text)
+    text = RE_REPEATED_WORD.sub(r'\1', text)
+    text = text.replace('\t', ' ')
+    text = RE_TRAILING_SPACE.sub('', text)
+    text = RE_MULTI_SPACE.sub(' ', text)
+    text = RE_MULTI_NEWLINE.sub('\n\n\n', text)
+    text = RE_DOUBLE_PERIOD.sub('.', text)
+    text = RE_NO_SPACE_AFTER_PERIOD.sub(r'\1 \2', text)
+    text = RE_SPACE_BEFORE_PUNCT.sub(r'\1', text)
+    text = text.replace('\u2018', "'").replace('\u2019', "'")
+    text = text.replace('\u201c', '"').replace('\u201d', '"')
+    text = text.replace('\u2013', '-').replace('\u2014', ' - ')
+    text = text.replace('\u2026', '...')
+    text = text.replace('\u2022', '- ')
+    text = text.replace('\u00b7', ' ')
+    text = text.replace('\u00a0', ' ')
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            clean_lines.append('')
+            continue
+        if len(line) > 10:
+            alpha_count = sum(1 for c in line if c.isalpha())
+            if alpha_count / len(line) < 0.40:
+                continue
+        if line.count('|') > 3 or line.count('{') > 2 or line.count('}') > 2:
+            continue
+        if RE_IMPORT.match(line):
+            continue
+        if line and line[0].isalpha() and line[0].islower():
+            if not clean_lines or clean_lines[-1] == '' or clean_lines[-1].rstrip().endswith(('.', '!', '?', ':')):
+                line = line[0].upper() + line[1:]
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = text.strip()
+    paragraphs = text.split('\n\n')
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        p_stripped = p.strip()
+        if not p_stripped:
+            continue
+        p_key = ' '.join(p_stripped.lower().split())
+        if p_key not in seen:
+            seen.add(p_key)
+            unique_paragraphs.append(p_stripped)
+    text = '\n\n'.join(unique_paragraphs)
+    text = text.strip()
+    if len(text) < 50:
+        return None
+    if len(text.split()) < 10:
+        return None
+    ascii_count = sum(1 for c in text if ord(c) < 128)
+    if ascii_count / max(len(text), 1) < 0.85:
+        return None
+    return text
+
+
+def smart_filter(text):
+    words = text.split()
+    word_count = len(words)
+    if word_count < 50:
+        return False, text, f"too short ({word_count} words)"
+    scripts = []
+    if RE_CJK.search(text): scripts.append("CJK")
+    if RE_ARABIC.search(text): scripts.append("Arabic")
+    if RE_CYRILLIC.search(text): scripts.append("Cyrillic")
+    if RE_DEVANAGARI.search(text): scripts.append("Devanagari")
+    if scripts:
+        return False, text, f"non-English: {', '.join(scripts)}"
+    if word_count > 50:
+        unique_ratio = len(set(w.lower() for w in words)) / word_count
+        if unique_ratio < 0.20:
+            return False, text, f"repetitive ({unique_ratio:.3f})"
+    code_matches = RE_RESIDUAL_CODE.findall(text)
+    if len(code_matches) >= 5:
+        return False, text, f"residual code ({len(code_matches)})"
+    original_len = len(text)
+    for pattern in [RE_COOKIE_LINE, RE_SUBSCRIBE_LINE, RE_CLICKBAIT_LINE,
+                    RE_SOCIAL_LINE, RE_NAV_LINE, RE_LOGIN_LINE,
+                    RE_COMMENT_LINE, RE_COPYRIGHT_LINE]:
+        text = pattern.sub('', text)
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped and len(stripped) > 10:
+            digit_count = sum(1 for c in stripped if c.isdigit() or c in ' ,.\t-+/%$')
+            if digit_count / len(stripped) > 0.80:
+                continue
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = text.strip()
+    if len(text.split()) < 50:
+        return False, text, "too short after stripping"
+    return True, text, None
+
+
+# ==============================================================================
+# LITDATA I/O
+# ==============================================================================
+
+def write_litdata_chunks(output_dir, token_stream, config, start_chunk_idx=0):
+    """Write token stream as litdata binary chunks, starting at given chunk index."""
+    os.makedirs(output_dir, exist_ok=True)
+    dtype_size = DTYPE().itemsize
+    tokens_per_chunk = CHUNK_BYTES_TARGET // dtype_size
+    tokens_per_chunk = (tokens_per_chunk // BLOCK_SIZE) * BLOCK_SIZE
+    chunks_metadata = []
+    pos = 0
+    chunk_idx = start_chunk_idx
+    while pos < len(token_stream):
+        remaining = len(token_stream) - pos
+        chunk_tokens = min(tokens_per_chunk, remaining)
+        num_blocks = chunk_tokens // BLOCK_SIZE
+        if num_blocks == 0:
+            break
+        actual_tokens = num_blocks * BLOCK_SIZE
+        chunk_data = token_stream[pos:pos + actual_tokens]
+        filename = f"chunk-0-{chunk_idx}.bin"
+        filepath = os.path.join(output_dir, filename)
+        header_num_items = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * dtype_size)
+        header = np.concatenate([header_num_items, offsets])
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        chunks_metadata.append(meta)
+        pos += actual_tokens
+        chunk_idx += 1
+        print(f"    Written chunk {chunk_idx} ({pos:,}/{len(token_stream):,} tokens)")
+    return chunks_metadata
+
+
+def merge_litdata(sources, dest_dir):
+    """
+    Merge multiple litdata directories into one.
+    sources: list of Path objects to litdata directories
+    dest_dir: Path for the merged output
+    """
+    os.makedirs(str(dest_dir), exist_ok=True)
+    all_chunks = []
+    config = None
+    chunk_offset = 0
+
+    for src in sources:
+        idx_path = src / "index.json"
+        with open(idx_path, "r") as f:
+            index = json.load(f)
+
+        if config is None:
+            config = index["config"]
+
+        for chunk_meta in index["chunks"]:
+            old_filename = chunk_meta["filename"]
+            new_filename = f"chunk-0-{chunk_offset}.bin"
+
+            # Copy the binary chunk file
+            src_file = src / old_filename
+            dst_file = dest_dir / new_filename
+            shutil.copy2(str(src_file), str(dst_file))
+
+            new_meta = dict(chunk_meta)
+            new_meta["filename"] = new_filename
+            all_chunks.append(new_meta)
+            chunk_offset += 1
+
+        src_tokens = sum(c["dim"] for c in index["chunks"])
+        print(f"    Merged {src.name}: {len(index['chunks'])} chunks, {src_tokens:,} tokens")
+
+    # Write combined index
+    combined_index = {
+        "chunks": all_chunks,
+        "config": config,
+        "updated_at": str(time.time()),
+    }
+    with open(str(dest_dir / "index.json"), "w") as f:
+        json.dump(combined_index, f, indent=2)
+
+    total = sum(c["dim"] for c in all_chunks)
+    print(f"    Total: {len(all_chunks)} chunks, {total:,} tokens")
+    return all_chunks, config
+
+
+# ==============================================================================
+# MAIN PIPELINE
+# ==============================================================================
+
+def main():
+    from datasets import load_dataset
+
+    t_start = time.time()
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 1: Merge existing litdata_english_100m + litdata_english_clean
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 1: MERGING EXISTING DATASETS")
+    print(f"    - {LITDATA_100M}")
+    print(f"    - {LITDATA_CLEAN}")
+    print(f"    → {COMBINED_DIR}")
+    print(f"{'='*75}\n")
+
+    merged_chunks, config = merge_litdata([LITDATA_100M, LITDATA_CLEAN], COMBINED_DIR)
+    merged_tokens = sum(c["dim"] for c in merged_chunks)
+    merged_chunk_count = len(merged_chunks)
+    print(f"\n  Step 1 done: {merged_tokens:,} tokens in {merged_chunk_count} chunks")
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 2: Download fresh 100M tokens
+    # ═════════════════════════════════════════════════════════════
+    wiki_target = int(TARGET_TOKENS * WIKI_SHARE)
+    fineweb_target = TARGET_TOKENS - wiki_target
+
+    print(f"\n{'='*75}")
+    print(f"  STEP 2: DOWNLOADING FRESH DATA")
+    print(f"  Target: {TARGET_TOKENS:,} tokens ({wiki_target:,} Wiki + {fineweb_target:,} FineWeb)")
+    print(f"  Skipping first {WIKI_SKIP:,} Wiki + {FINEWEB_SKIP:,} FineWeb (already used)")
+    print(f"{'='*75}")
+
+    os.makedirs(str(PARQUET_DIR), exist_ok=True)
+
+    # --- Wikipedia ---
+    print(f"\n  [Wikipedia] Streaming English articles (skipping first {WIKI_SKIP:,})...")
+    ds_wiki = load_dataset(
+        "wikimedia/wikipedia", "20231101.en",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    wiki_texts = []
+    wiki_tokens = 0
+    wiki_seen = 0
+    wiki_skipped_quality = 0
+    wiki_skipped_meta = 0
+    t0 = time.time()
+
+    for article in ds_wiki:
+        title = (article.get("title") or "").strip()
+        raw = article.get("text") or ""
+
+        if _WIKI_SKIP_PATTERNS.search(title):
+            wiki_skipped_meta += 1
+            continue
+
+        cleaned = clean_text(raw)
+        if not is_high_quality(cleaned, min_chars=800, min_words=120):
+            wiki_skipped_quality += 1
+            continue
+
+        wiki_seen += 1
+
+        if wiki_seen <= WIKI_SKIP:
+            if wiki_seen % 10000 == 0:
+                print(f"    Skipping... {wiki_seen:,}/{WIKI_SKIP:,}")
+            continue
+
+        full_text = f"{title}\n\n{cleaned}"
+        est_tok = int(len(full_text.split()) * TOKENS_PER_WORD)
+        wiki_texts.append(full_text)
+        wiki_tokens += est_tok
+
+        if len(wiki_texts) % 5000 == 0:
+            elapsed = time.time() - t0
+            print(f"    Collected {len(wiki_texts):,} articles | ~{wiki_tokens:,} tokens | {elapsed:.0f}s")
+
+        if wiki_tokens >= wiki_target:
+            break
+
+    elapsed = time.time() - t0
+    print(f"  [Wikipedia] Done: {len(wiki_texts):,} articles, ~{wiki_tokens:,} tokens in {elapsed:.0f}s")
+    print(f"    (skipped {WIKI_SKIP:,} already-used + {wiki_skipped_quality:,} low-quality + {wiki_skipped_meta:,} meta)")
+
+    BATCH = 5000
+    for i in range(0, len(wiki_texts), BATCH):
+        batch = wiki_texts[i:i+BATCH]
+        fp = PARQUET_DIR / f"wiki_{i//BATCH:04d}.parquet"
+        pq.write_table(pa.table({"text": batch}), str(fp))
+    print(f"  Saved {(len(wiki_texts)-1)//BATCH + 1} wiki parquet files")
+
+    # --- FineWeb-Edu ---
+    print(f"\n  [FineWeb-Edu] Streaming (score >= {FINEWEB_MIN_SCORE}, skipping first {FINEWEB_SKIP:,})...")
+    ds_fineweb = load_dataset(
+        "HuggingFaceFW/fineweb-edu", "sample-10BT",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    fineweb_texts = []
+    fineweb_tokens = 0
+    fineweb_seen = 0
+    fineweb_skipped_quality = 0
+    fineweb_skipped_score = 0
+    t0 = time.time()
+
+    for doc in ds_fineweb:
+        score = doc.get("score", 0)
+        if not isinstance(score, (int, float)):
+            try:
+                score = float(score)
+            except (ValueError, TypeError):
+                continue
+        if score < FINEWEB_MIN_SCORE:
+            fineweb_skipped_score += 1
+            continue
+
+        raw = doc.get("text") or ""
+        cleaned = clean_text(raw)
+        if not is_high_quality(cleaned, min_chars=500, min_words=80):
+            fineweb_skipped_quality += 1
+            continue
+
+        fineweb_seen += 1
+
+        if fineweb_seen <= FINEWEB_SKIP:
+            if fineweb_seen % 10000 == 0:
+                print(f"    Skipping... {fineweb_seen:,}/{FINEWEB_SKIP:,}")
+            continue
+
+        est_tok = int(len(cleaned.split()) * TOKENS_PER_WORD)
+        fineweb_texts.append(cleaned)
+        fineweb_tokens += est_tok
+
+        if len(fineweb_texts) % 5000 == 0:
+            elapsed = time.time() - t0
+            print(f"    Collected {len(fineweb_texts):,} docs | ~{fineweb_tokens:,} tokens | {elapsed:.0f}s")
+
+        if fineweb_tokens >= fineweb_target:
+            break
+
+    elapsed = time.time() - t0
+    print(f"  [FineWeb-Edu] Done: {len(fineweb_texts):,} docs, ~{fineweb_tokens:,} tokens in {elapsed:.0f}s")
+    print(f"    (skipped {FINEWEB_SKIP:,} already-used + {fineweb_skipped_quality:,} low-quality + {fineweb_skipped_score:,} low-score)")
+
+    for i in range(0, len(fineweb_texts), BATCH):
+        batch = fineweb_texts[i:i+BATCH]
+        fp = PARQUET_DIR / f"fineweb_{i//BATCH:04d}.parquet"
+        pq.write_table(pa.table({"text": batch}), str(fp))
+    print(f"  Saved {(len(fineweb_texts)-1)//BATCH + 1} fineweb parquet files")
+
+    all_texts = wiki_texts + fineweb_texts
+    total_est_tokens = wiki_tokens + fineweb_tokens
+    del wiki_texts, fineweb_texts
+
+    print(f"\n  Step 2 done: {len(all_texts):,} documents, ~{total_est_tokens:,} estimated tokens")
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 3: Deep clean every document
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 3: DEEP CLEANING {len(all_texts):,} DOCUMENTS")
+    print(f"{'='*75}")
+
+    t2 = time.time()
+    cleaned_texts = []
+    dropped_clean = 0
+    for i, text in enumerate(all_texts):
+        result = deep_clean(text)
+        if result is not None:
+            cleaned_texts.append(result)
+        else:
+            dropped_clean += 1
+        if (i + 1) % 10000 == 0 or i == len(all_texts) - 1:
+            print(f"    Cleaned {i+1:,}/{len(all_texts):,} | kept={len(cleaned_texts):,} | dropped={dropped_clean:,}")
+
+    del all_texts
+    print(f"  Deep clean done in {time.time()-t2:.1f}s")
+    print(f"  Kept: {len(cleaned_texts):,} | Dropped: {dropped_clean:,}")
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 4: Smart cleanup
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 4: SMART CLEANUP ON {len(cleaned_texts):,} DOCUMENTS")
+    print(f"{'='*75}")
+
+    t3 = time.time()
+    final_texts = []
+    removed_reasons = Counter()
+    total_boilerplate = 0
+
+    for i, text in enumerate(cleaned_texts):
+        keep, stripped_text, reason = smart_filter(text)
+        if keep:
+            final_texts.append(stripped_text)
+            total_boilerplate += len(text) - len(stripped_text)
+        else:
+            removed_reasons[reason.split('(')[0].strip().split(':')[0].strip()] += 1
+        if (i + 1) % 10000 == 0 or i == len(cleaned_texts) - 1:
+            print(f"    Processed {i+1:,}/{len(cleaned_texts):,} | kept={len(final_texts):,}")
+
+    del cleaned_texts
+    print(f"  Smart cleanup done in {time.time()-t3:.1f}s")
+    print(f"  Final new documents: {len(final_texts):,}")
+    print(f"  Boilerplate stripped: {total_boilerplate:,} chars")
+    if removed_reasons:
+        print(f"  Removal reasons:")
+        for reason, count in sorted(removed_reasons.items(), key=lambda x: -x[1]):
+            print(f"    {reason:<35} {count:>6,}")
+
+    # ═════════════════════════════════════════════════════════════
+    # STEP 5: Tokenize new data
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 5: TOKENIZING {len(final_texts):,} NEW DOCUMENTS")
+    print(f"{'='*75}")
+
+    t4 = time.time()
+    all_token_ids = []
+    total_tokens = 0
+    ENCODE_BATCH = 10000
+
+    for i in range(0, len(final_texts), ENCODE_BATCH):
+        batch = final_texts[i:i+ENCODE_BATCH]
+        encoded = tokenizer.encode_batch(batch, add_special_tokens=False)
+        for enc in encoded:
+            ids = enc.ids
+            all_token_ids.extend(ids)
+            all_token_ids.append(EOS_TOKEN_ID)
+            total_tokens += len(ids) + 1
+        done = min(i + ENCODE_BATCH, len(final_texts))
+        if done % 20000 == 0 or done == len(final_texts):
+            print(f"    Tokenized {done:,}/{len(final_texts):,} ({total_tokens:,} tokens)")
+
+    del final_texts
+    print(f"  Tokenized in {time.time()-t4:.1f}s - {total_tokens:,} total new tokens")
+
+    # ═════════════════════════════════��═══════════════════════════
+    # STEP 6: Write new chunks directly into combined dir
+    # ═════════════════════════════════════════════════════════════
+    print(f"\n{'='*75}")
+    print(f"  STEP 6: WRITING NEW CHUNKS INTO COMBINED DATASET")
+    print(f"{'='*75}")
+
+    token_array = np.array(all_token_ids, dtype=DTYPE)
+    del all_token_ids
+
+    litdata_config = {
+        "block_size": BLOCK_SIZE,
+        "vocab_size": tokenizer.get_vocab_size(),
+    }
+
+    # Write new chunks starting after the existing merged chunks
+    new_chunks = write_litdata_chunks(
+        str(COMBINED_DIR), token_array, litdata_config,
+        start_chunk_idx=merged_chunk_count,
+    )
+    new_token_count = sum(c["dim"] for c in new_chunks)
+    del token_array
+
+    # Update the combined index.json with the new chunks appended
+    all_final_chunks = merged_chunks + new_chunks
+    final_total_tokens = sum(c["dim"] for c in all_final_chunks)
+    final_index = {
+        "chunks": all_final_chunks,
+        "config": litdata_config,
+        "updated_at": str(time.time()),
+    }
+    with open(str(COMBINED_DIR / "index.json"), "w") as f:
+        json.dump(final_index, f, indent=2)
+
+    print(f"\n  New data: {new_token_count:,} tokens in {len(new_chunks)} chunks")
+    print(f"  Final combined: {final_total_tokens:,} tokens in {len(all_final_chunks)} chunks")
+
+    # ═════════════════════════════════════════════════════════════
+    # FINAL REPORT
+    # ═════════════════════════════════════════════════════════════
+    total_time = time.time() - t_start
+
+    report_lines = []
+    report_lines.append(f"\n{'='*75}")
+    report_lines.append(f"  LITDATA_COMBINED - MERGE & BUILD REPORT")
+    report_lines.append(f"{'='*75}")
+    report_lines.append(f"\n  Total time: {total_time:.0f}s ({total_time/60:.1f} min)")
+    report_lines.append(f"\n  SOURCE DATASETS MERGED")
+    report_lines.append(f"  {'-'*60}")
+    report_lines.append(f"    litdata_english_100m + litdata_english_clean")
+    report_lines.append(f"    Merged subtotal:       {merged_tokens:,} tokens ({merged_chunk_count} chunks)")
+    report_lines.append(f"\n  NEW 100M BATCH (articles {WIKI_SKIP+1:,}+)")
+    report_lines.append(f"  {'-'*60}")
+    report_lines.append(f"    Wikipedia:    ~{wiki_tokens:,} est. tokens")
+    report_lines.append(f"    FineWeb-Edu:  ~{fineweb_tokens:,} est. tokens")
+    report_lines.append(f"    Downloaded:   ~{total_est_tokens:,} est. tokens")
+    report_lines.append(f"    Deep clean:   dropped {dropped_clean:,} docs")
+    report_lines.append(f"    Smart filter: removed {sum(removed_reasons.values()):,} docs")
+    if removed_reasons:
+        for reason, count in sorted(removed_reasons.items(), key=lambda x: -x[1]):
+            report_lines.append(f"      - {reason}: {count:,}")
+    report_lines.append(f"    Boilerplate:  stripped {total_boilerplate:,} chars")
+    report_lines.append(f"    New tokens:   {new_token_count:,} ({len(new_chunks)} chunks)")
+    report_lines.append(f"\n  FINAL COMBINED OUTPUT")
+    report_lines.append(f"  {'-'*60}")
+    report_lines.append(f"    Location: {COMBINED_DIR}")
+    report_lines.append(f"    Chunks:   {len(all_final_chunks)}")
+    report_lines.append(f"    Tokens:   {final_total_tokens:,}")
+    report_lines.append(f"    Format:   litdata binary (int32, BLOCK_SIZE=1025, EOS=0)")
+    report_lines.append(f"\n  ZERO OVERLAP GUARANTEE:")
+    report_lines.append(f"    - litdata_english_clean: Wiki articles 1-19,523 + FineWeb docs 1-19,445")
+    report_lines.append(f"    - litdata_english_100m:  Wiki articles 25,001-78,482 + FineWeb docs 25,001-68,550")
+    report_lines.append(f"    - New batch:             Wiki articles {WIKI_SKIP+1:,}+ + FineWeb docs {FINEWEB_SKIP+1:,}+")
+    report_lines.append(f"\n{'='*75}")
+
+    full_report = '\n'.join(report_lines)
+    print(full_report)
+
+    report_path = COMBINED_DIR / "BUILD_REPORT.txt"
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(full_report)
+    print(f"\n  Report saved to: {report_path}")
+    print(f"  Done! Final combined dataset ready for training.")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/prepare_finetune_data.py

@@ -0,0 +1,88 @@
+"""
+Merge all JSONL instruction-tuning datasets into train.json and val.json
+for litgpt finetune (JSON data module).
+
+Each row must have: instruction, output, and optionally input.
+"""
+
+import json
+import os
+import random
+import argparse
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_dir", type=str, default=r"D:\ASTERIZER 2026\LitGPT\OLD_DATASETS")
+    parser.add_argument("--output_dir", type=str, default=r"D:\ASTERIZER 2026\LUNA\Base\Datasets\finetune")
+    parser.add_argument("--val_fraction", type=float, default=0.05, help="Fraction for validation split")
+    parser.add_argument("--seed", type=int, default=42)
+    args = parser.parse_args()
+
+    random.seed(args.seed)
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    all_samples = []
+    file_counts = {}
+
+    for fname in sorted(os.listdir(args.input_dir)):
+        if not fname.endswith(".jsonl"):
+            continue
+        fpath = os.path.join(args.input_dir, fname)
+        count = 0
+        with open(fpath, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                row = json.loads(line)
+                # Keep only the keys litgpt expects
+                sample = {
+                    "instruction": row.get("instruction", ""),
+                    "input": row.get("input", ""),
+                    "output": row.get("output", ""),
+                }
+                # Skip rows with empty instruction AND empty input
+                if not sample["instruction"].strip() and not sample["input"].strip():
+                    continue
+                # Skip rows with empty output
+                if not sample["output"].strip():
+                    continue
+                all_samples.append(sample)
+                count += 1
+        file_counts[fname] = count
+        print(f"  {fname}: {count} samples")
+
+    print(f"\nTotal valid samples: {len(all_samples)}")
+
+    # Shuffle
+    random.shuffle(all_samples)
+
+    # Split
+    val_size = max(1, int(len(all_samples) * args.val_fraction))
+    val_data = all_samples[:val_size]
+    train_data = all_samples[val_size:]
+
+    print(f"Train: {len(train_data)}, Val: {val_size}")
+
+    # Write
+    train_path = os.path.join(args.output_dir, "train.json")
+    val_path = os.path.join(args.output_dir, "val.json")
+
+    with open(train_path, "w", encoding="utf-8") as f:
+        json.dump(train_data, f, ensure_ascii=False, indent=None)
+    with open(val_path, "w", encoding="utf-8") as f:
+        json.dump(val_data, f, ensure_ascii=False, indent=None)
+
+    print(f"\nSaved: {train_path}")
+    print(f"Saved: {val_path}")
+
+    # Show a few samples
+    print("\n--- Sample train entries ---")
+    for s in train_data[:3]:
+        print(f"  instruction: {s['instruction'][:80]}")
+        print(f"  input:       {s['input'][:80]}")
+        print(f"  output:      {s['output'][:80]}")
+        print()
+
+if __name__ == "__main__":
+    main()

Base/scripts/prepare_litdata.py

@@ -0,0 +1,275 @@
+"""
+Tokenize filtered parquet data and create LitData streaming format for litgpt.
+
+Creates flat binary token chunks compatible with litdata's TokensLoader,
+which litgpt's LitData data module uses for pretraining.
+
+STREAMING MODE: Tokenizes texts in batches and flushes chunks to disk
+incrementally, keeping RAM usage constant (~128MB buffer) regardless of
+dataset size. Supports resuming from the last completed chunk.
+
+Usage:
+    python Base/scripts/prepare_litdata.py --mode small
+    python Base/scripts/prepare_litdata.py --mode full
+    python Base/scripts/prepare_litdata.py --mode both
+"""
+
+import argparse
+import json
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+import pyarrow.parquet as pq
+
+from litgpt.tokenizer import Tokenizer
+
+
+# Must match model config's block_size + 1 (extra token for targets)
+BLOCK_SIZE = 1025
+# 64 MB per chunk file
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+# litdata's TokensLoader uses _TORCH_DTYPES_MAPPING[16] = torch.int32
+DTYPE = np.int32
+# litdata dtype index — 16 maps to torch.int32
+DTYPE_INDEX = 16
+
+
+def _write_chunk(output_dir, chunk_idx, block_buffer, num_blocks):
+    """Write a single chunk file with litdata header + flat int32 data."""
+    dtype_size = DTYPE().itemsize
+    filename = f"chunk-0-{chunk_idx}.bin"
+    filepath = os.path.join(output_dir, filename)
+
+    chunk_data = block_buffer[:num_blocks * BLOCK_SIZE]
+
+    # Header: [num_items(uint32)] + [offsets 0..num_blocks(uint32)]
+    header_num_items = np.array([num_blocks], dtype=np.uint32)
+    offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * dtype_size)
+    header = np.concatenate([header_num_items, offsets])
+
+    with open(filepath, "wb") as f:
+        header.tofile(f)
+        chunk_data.tofile(f)
+
+    data_bytes = int(chunk_data.nbytes)
+    header_bytes = int(header.nbytes)
+    return {
+        "chunk_bytes": data_bytes + header_bytes,
+        "chunk_size": num_blocks,
+        "dim": int(len(chunk_data)),
+        "filename": filename,
+    }, header_bytes, data_bytes
+
+
+def _load_resume_state(output_dir):
+    """Check for existing chunks to support resuming."""
+    index_path = os.path.join(output_dir, "index.json.partial")
+    if not os.path.exists(index_path):
+        return [], 0, 0
+    with open(index_path) as f:
+        state = json.load(f)
+    chunks = state.get("chunks", [])
+    tokens_written = sum(c["dim"] for c in chunks)
+    return chunks, len(chunks), tokens_written
+
+
+def _save_resume_state(output_dir, chunks_metadata):
+    """Save partial progress for resume."""
+    path = os.path.join(output_dir, "index.json.partial")
+    with open(path, "w") as f:
+        json.dump({"chunks": chunks_metadata}, f)
+
+
+def prepare_litdata(filtered_dir: str, output_dir: str, tokenizer_path: str, label: str):
+    """Tokenize filtered text and write chunks in streaming fashion (constant RAM)."""
+    print(f"\n{'='*60}")
+    print(f"Preparing LitData [{label}]")
+    print(f"Input: {filtered_dir}")
+    print(f"Output: {output_dir}")
+    print(f"Tokenizer: {tokenizer_path}")
+    print(f"{'='*60}\n")
+
+    if not Path(filtered_dir).exists():
+        print(f"ERROR: Filtered directory not found: {filtered_dir}")
+        print("Run filter_datasets.py first!")
+        return
+
+    os.makedirs(output_dir, exist_ok=True)
+    tokenizer = Tokenizer(Path(tokenizer_path))
+
+    # How many tokens fit in one chunk (aligned to BLOCK_SIZE)
+    dtype_size = DTYPE().itemsize
+    tokens_per_chunk = CHUNK_BYTES_TARGET // dtype_size
+    tokens_per_chunk = (tokens_per_chunk // BLOCK_SIZE) * BLOCK_SIZE
+
+    # Check for resume state
+    chunks_metadata, chunk_idx, tokens_to_skip = _load_resume_state(output_dir)
+    if tokens_to_skip > 0:
+        print(f"RESUMING: Found {chunk_idx} existing chunks ({tokens_to_skip:,} tokens)")
+        print(f"  Will skip {tokens_to_skip:,} tokens then continue writing chunks\n")
+
+    # Token buffer — holds at most one chunk worth of tokens (~64MB)
+    token_buf = np.empty(tokens_per_chunk + BLOCK_SIZE * 512, dtype=DTYPE)
+    buf_pos = 0  # current write position in buffer
+    total_tokens_written = sum(c["dim"] for c in chunks_metadata)
+    total_tokens_seen = 0
+    total_texts = 0
+    skipped_tokens = 0
+
+    parquet_files = sorted(Path(filtered_dir).glob("*.parquet"))
+    t0 = time.time()
+
+    print("Tokenizing & writing chunks (streaming)...")
+    for pf in parquet_files:
+        parquet_file = pq.ParquetFile(str(pf))
+        file_count = 0
+        for batch in parquet_file.iter_batches(batch_size=4096, columns=["text"]):
+            texts = batch.column("text").to_pylist()
+            for text in texts:
+                tokens = tokenizer.encode(text, bos=False, eos=True)
+                tok_array = tokens.numpy().astype(DTYPE) if hasattr(tokens, 'numpy') else np.array(tokens.tolist(), dtype=DTYPE)
+                n = len(tok_array)
+                total_tokens_seen += n
+
+                # If resuming, skip tokens already written
+                if skipped_tokens < tokens_to_skip:
+                    remaining_skip = tokens_to_skip - skipped_tokens
+                    if n <= remaining_skip:
+                        skipped_tokens += n
+                        file_count += 1
+                        continue
+                    else:
+                        tok_array = tok_array[int(remaining_skip):]
+                        skipped_tokens = tokens_to_skip
+                        n = len(tok_array)
+
+                # Append to buffer
+                if buf_pos + n > len(token_buf):
+                    # Grow buffer if needed (rare)
+                    token_buf = np.concatenate([token_buf[:buf_pos], np.empty(max(n, BLOCK_SIZE * 512), dtype=DTYPE)])
+
+                token_buf[buf_pos:buf_pos + n] = tok_array
+                buf_pos += n
+                file_count += 1
+
+                # Flush full chunks from buffer
+                while buf_pos >= tokens_per_chunk:
+                    num_blocks = tokens_per_chunk // BLOCK_SIZE
+                    chunk_array = token_buf[:tokens_per_chunk].copy()
+
+                    meta, hdr_b, data_b = _write_chunk(output_dir, chunk_idx, chunk_array, num_blocks)
+                    chunks_metadata.append(meta)
+                    total_tokens_written += meta["dim"]
+                    _save_resume_state(output_dir, chunks_metadata)
+
+                    print(f"  chunk-0-{chunk_idx}.bin: {num_blocks} blocks, "
+                          f"{meta['dim']:,} tokens (header: {hdr_b/1024:.1f} KB, "
+                          f"data: {data_b/1024/1024:.1f} MB) | "
+                          f"total: {total_tokens_written:,}")
+                    chunk_idx += 1
+
+                    # Shift remaining tokens to front of buffer
+                    leftover = buf_pos - tokens_per_chunk
+                    if leftover > 0:
+                        token_buf[:leftover] = token_buf[tokens_per_chunk:tokens_per_chunk + leftover]
+                    buf_pos = leftover
+
+        total_texts += file_count
+        parquet_file.close()
+        elapsed = time.time() - t0
+        print(f"  [{pf.name}] {file_count:,} texts | "
+              f"total tokens written: {total_tokens_written:,} | "
+              f"elapsed: {elapsed:.0f}s")
+
+    # Flush remaining buffer as final chunk (if enough for at least 1 block)
+    remaining_blocks = buf_pos // BLOCK_SIZE
+    if remaining_blocks > 0:
+        final_tokens = remaining_blocks * BLOCK_SIZE
+        chunk_array = token_buf[:final_tokens].copy()
+        meta, hdr_b, data_b = _write_chunk(output_dir, chunk_idx, chunk_array, remaining_blocks)
+        chunks_metadata.append(meta)
+        total_tokens_written += meta["dim"]
+        print(f"  chunk-0-{chunk_idx}.bin (final): {remaining_blocks} blocks, "
+              f"{meta['dim']:,} tokens (header: {hdr_b/1024:.1f} KB, "
+              f"data: {data_b/1024/1024:.1f} MB)")
+        chunk_idx += 1
+
+    # Write final index.json
+    index = {
+        "chunks": chunks_metadata,
+        "config": {
+            "chunk_bytes": CHUNK_BYTES_TARGET,
+            "chunk_size": None,
+            "compression": None,
+            "data_format": [f"no_header_tensor:{DTYPE_INDEX}"],
+            "data_spec": None,
+            "encryption": None,
+            "item_loader": "TokensLoader",
+        },
+        "updated_at": str(time.time()),
+    }
+    with open(os.path.join(output_dir, "index.json"), "w") as f:
+        json.dump(index, f, indent=2)
+
+    # Clean up partial state
+    partial_path = os.path.join(output_dir, "index.json.partial")
+    if os.path.exists(partial_path):
+        os.remove(partial_path)
+
+    elapsed = time.time() - t0
+    print(f"\n--- LitData preparation [{label}] complete ---")
+    print(f"Chunks: {chunk_idx}")
+    print(f"Total tokens: {total_tokens_written:,}")
+    print(f"Total blocks: {total_tokens_written // BLOCK_SIZE:,}")
+    print(f"Texts processed: {total_texts:,}")
+    print(f"Time: {elapsed:.0f}s ({total_tokens_written/elapsed:.0f} tok/s)")
+    print(f"Output: {output_dir}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Prepare LitData from filtered parquet")
+    parser.add_argument(
+        "--mode",
+        choices=["small", "full", "both", "english"],
+        default="small",
+        help="Which dataset to prepare (english = ultra-clean English corpus)",
+    )
+    parser.add_argument(
+        "--tokenizer_path",
+        type=str,
+        default="Base/checkpoints/EleutherAI/pythia-160m",
+        help="Path to tokenizer directory",
+    )
+    args = parser.parse_args()
+
+    if args.mode in ("small", "both"):
+        prepare_litdata(
+            filtered_dir="Base/data/filtered_10m",
+            output_dir="Base/data/litdata_10m",
+            tokenizer_path=args.tokenizer_path,
+            label="SMALL (10M)",
+        )
+
+    if args.mode in ("full", "both"):
+        prepare_litdata(
+            filtered_dir="Base/data/filtered_3b",
+            output_dir="Base/data/litdata_3b",
+            tokenizer_path=args.tokenizer_path,
+            label="FULL (3B)",
+        )
+
+    if args.mode == "english":
+        prepare_litdata(
+            filtered_dir="Base/data/filtered_english",
+            output_dir="Base/data/litdata_english",
+            tokenizer_path=args.tokenizer_path,
+            label="ENGLISH (ultra-clean)",
+        )
+
+    print("\nDone! Next step: run litgpt pretrain with the appropriate config.")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/reclean_3b.py

@@ -0,0 +1,593 @@
+# -*- coding: utf-8 -*-
+"""
+Memory-efficient, multiprocessed reclean of litdata_3b (~3B tokens).
+
+Problem: The old script loaded ALL 3B tokens → decoded 4.3M docs → re-tokenized
+into a Python list → OOM at ~12 GB list + ~15 GB decoded text.
+
+Solution: Process in chunk batches (10 chunks ≈ 168M tokens ≈ 670 MB).
+  1. Read 10 chunks at a time
+  2. Split tokens into documents (carry partial docs across batches)
+  3. Decode with tokenizer.decode_batch (multithreaded Rust)
+  4. Deep clean + smart filter with multiprocessing Pool (all CPU cores)
+  5. Re-tokenize with encode_batch (multithreaded Rust)
+  6. Stream output to litdata chunks incrementally (never accumulate)
+
+Peak memory: ~4-5 GB instead of 40+ GB.
+Output: Base/data/litdata_3b_clean/ (separate, not merged)
+"""
+
+import json
+import os
+import re
+import sys
+import time
+import unicodedata
+from pathlib import Path
+from multiprocessing import Pool, cpu_count
+from collections import Counter
+
+import numpy as np
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+
+CHUNKS_PER_BATCH = 10        # ~670 MB per batch
+NUM_WORKERS = max(1, cpu_count() - 2)  # leave 2 cores for main + I/O
+DECODE_BATCH = 8000           # docs per decode_batch call
+ENCODE_BATCH = 8000           # docs per encode_batch call
+
+DATA_DIR = ROOT / "Base" / "data"
+INPUT_DIR = DATA_DIR / "litdata_3b"
+OUTPUT_DIR = DATA_DIR / "litdata_3b_clean"
+TOKENIZER_PATH = str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+
+
+# ==============================================================================
+# TEXT CLEANING PIPELINE (deep_clean + smart_filter)
+# ==============================================================================
+
+CONTROL_CHARS = [
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f", "\x7f", "\ufeff", "\ufffd",
+]
+
+HTML_ENTITIES = [
+    ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+    ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"),
+    ("&nbsp;", " "), ("&mdash;", " - "), ("&ndash;", "-"),
+    ("&hellip;", "..."), ("&laquo;", '"'), ("&raquo;", '"'),
+    ("&bull;", "- "), ("&middot;", " "), ("&copy;", "(c)"),
+    ("&reg;", "(R)"), ("&trade;", "(TM)"), ("&deg;", " degrees"),
+]
+
+RE_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_FILE_PATH = re.compile(r'(?:[A-Z]:\\|/(?:home|usr|var|etc|opt)/)\S+')
+RE_HTML_TAG = re.compile(r'</?[a-zA-Z][a-zA-Z0-9]*(?:\s[^>]*)?\s*/?>')
+RE_HTML_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+RE_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
+RE_IMPORT = re.compile(r'^(?:import |from \S+ import |#include |using namespace |require\()', re.M)
+RE_REPEATED_LINE = re.compile(r'^(.{20,})\n(?:\1\n?)+', re.M)
+RE_REPEATED_PUNCT = re.compile(r'([!?.])\1{3,}')
+RE_REPEATED_CHAR = re.compile(r'(.)\1{5,}')
+RE_REPEATED_WORD = re.compile(r'\b(\w+)(?:\s+\1){2,}\b', re.I)
+RE_MULTI_NEWLINE = re.compile(r'\n{4,}')
+RE_MULTI_SPACE = re.compile(r'[ \t]{2,}')
+RE_TRAILING_SPACE = re.compile(r'[ \t]+$', re.M)
+RE_NO_SPACE_AFTER_PERIOD = re.compile(r'([.!?])([A-Z])')
+RE_DOUBLE_PERIOD = re.compile(r'\.{2}(?!\.)')
+RE_SPACE_BEFORE_PUNCT = re.compile(r'\s+([.,;:!?])')
+
+# Smart filter patterns
+RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+RE_DEVANAGARI = re.compile(r'[\u0900-\u097f]{5,}')
+RE_RESIDUAL_CODE = re.compile(
+    r'(function\s*\(|var\s+\w+\s*=|console\.log|document\.get|if\s*\(\s*\w+\s*[!=]==)', re.I
+)
+
+RE_COOKIE_LINE = re.compile(r'^.*(?:cookie|cookies)\s+(?:policy|consent|notice|preferences|settings).*$', re.I | re.M)
+RE_SUBSCRIBE_LINE = re.compile(r'^.*(?:subscribe|sign\s*up\s+(?:for|to)\s+(?:our|the)\s+newsletter|unsubscribe|opt[\s-]*out\s+of).*$', re.I | re.M)
+RE_CLICKBAIT_LINE = re.compile(r'^.*(?:you\s+won\'?t\s+believe|click\s+here|read\s+more\s*\.{0,3}$|share\s+this\s+(?:article|post|story)|trending\s+now|sponsored\s+content|advertisement).*$', re.I | re.M)
+RE_SOCIAL_LINE = re.compile(r'^.*(?:follow\s+us\s+on|share\s+on\s+(?:facebook|twitter|linkedin|instagram)|like\s+us\s+on|tweet\s+this).*$', re.I | re.M)
+RE_NAV_LINE = re.compile(r'^.*(?:skip\s+to\s+(?:main\s+)?content|back\s+to\s+top|previous\s+article|next\s+article|related\s+(?:articles|posts)).*$', re.I | re.M)
+RE_LOGIN_LINE = re.compile(r'^.*(?:log\s*in\s+to\s+(?:your|an)\s+account|create\s+(?:a\s+)?(?:free\s+)?account|forgot\s+(?:your\s+)?password|already\s+(?:a\s+)?member).*$', re.I | re.M)
+RE_COMMENT_LINE = re.compile(r'^.*(?:leave\s+a\s+(?:comment|reply)|post\s+a\s+comment|\d+\s+comments?$|logged\s+in\s+as).*$', re.I | re.M)
+RE_COPYRIGHT_LINE = re.compile(r'^.*(?:all\s+rights\s+reserved|\(c\)\s*\d{4}|copyright\s+\d{4}).*$', re.I | re.M)
+
+
+def deep_clean(text):
+    """Full deep cleaning pipeline."""
+    if not text or len(text.strip()) < 30:
+        return None
+
+    text = unicodedata.normalize("NFKC", text)
+    for ch in CONTROL_CHARS:
+        text = text.replace(ch, "")
+    for old, new in HTML_ENTITIES:
+        text = text.replace(old, new)
+
+    text = RE_HTML_COMMENT.sub("", text)
+    text = RE_HTML_TAG.sub("", text)
+    text = RE_URL.sub("", text)
+    text = RE_EMAIL.sub("", text)
+    text = RE_FILE_PATH.sub("", text)
+    text = RE_CODE_BLOCK.sub("", text)
+
+    text = RE_REPEATED_LINE.sub(r'\1', text)
+    text = RE_REPEATED_PUNCT.sub(r'\1\1\1', text)
+    text = RE_REPEATED_CHAR.sub(r'\1\1\1', text)
+    text = RE_REPEATED_WORD.sub(r'\1', text)
+
+    text = text.replace('\t', ' ')
+    text = RE_TRAILING_SPACE.sub('', text)
+    text = RE_MULTI_SPACE.sub(' ', text)
+    text = RE_MULTI_NEWLINE.sub('\n\n\n', text)
+
+    text = RE_DOUBLE_PERIOD.sub('.', text)
+    text = RE_NO_SPACE_AFTER_PERIOD.sub(r'\1 \2', text)
+    text = RE_SPACE_BEFORE_PUNCT.sub(r'\1', text)
+
+    text = text.replace('\u2018', "'").replace('\u2019', "'")
+    text = text.replace('\u201c', '"').replace('\u201d', '"')
+    text = text.replace('\u2013', '-').replace('\u2014', ' - ')
+    text = text.replace('\u2026', '...')
+    text = text.replace('\u2022', '- ')
+    text = text.replace('\u00b7', ' ')
+    text = text.replace('\u00a0', ' ')
+
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            clean_lines.append('')
+            continue
+        if len(line) > 10:
+            alpha_count = sum(1 for c in line if c.isalpha())
+            if alpha_count / len(line) < 0.40:
+                continue
+        if line.count('|') > 3 or line.count('{') > 2 or line.count('}') > 2:
+            continue
+        if RE_IMPORT.match(line):
+            continue
+        if line and line[0].isalpha() and line[0].islower():
+            if not clean_lines or clean_lines[-1] == '' or clean_lines[-1].rstrip().endswith(('.', '!', '?', ':')):
+                line = line[0].upper() + line[1:]
+        clean_lines.append(line)
+
+    text = '\n'.join(clean_lines)
+    text = text.strip()
+
+    paragraphs = text.split('\n\n')
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        p_stripped = p.strip()
+        if not p_stripped:
+            continue
+        p_key = ' '.join(p_stripped.lower().split())
+        if p_key not in seen:
+            seen.add(p_key)
+            unique_paragraphs.append(p_stripped)
+    text = '\n\n'.join(unique_paragraphs)
+
+    text = text.strip()
+    if len(text) < 50:
+        return None
+    if len(text.split()) < 10:
+        return None
+    ascii_count = sum(1 for c in text if ord(c) < 128)
+    if ascii_count / max(len(text), 1) < 0.85:
+        return None
+
+    return text
+
+
+def smart_filter(text):
+    """Smart cleanup: returns (keep, cleaned_text, reason)."""
+    words = text.split()
+    word_count = len(words)
+    if word_count < 50:
+        return False, text, "too short"
+
+    scripts = []
+    if RE_CJK.search(text): scripts.append("CJK")
+    if RE_ARABIC.search(text): scripts.append("Arabic")
+    if RE_CYRILLIC.search(text): scripts.append("Cyrillic")
+    if RE_DEVANAGARI.search(text): scripts.append("Devanagari")
+    if scripts:
+        return False, text, "non-English"
+
+    if word_count > 50:
+        unique_ratio = len(set(w.lower() for w in words)) / word_count
+        if unique_ratio < 0.20:
+            return False, text, "repetitive"
+
+    code_matches = RE_RESIDUAL_CODE.findall(text)
+    if len(code_matches) >= 5:
+        return False, text, "residual code"
+
+    for pattern in [RE_COOKIE_LINE, RE_SUBSCRIBE_LINE, RE_CLICKBAIT_LINE,
+                    RE_SOCIAL_LINE, RE_NAV_LINE, RE_LOGIN_LINE,
+                    RE_COMMENT_LINE, RE_COPYRIGHT_LINE]:
+        text = pattern.sub('', text)
+
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped and len(stripped) > 10:
+            digit_count = sum(1 for c in stripped if c.isdigit() or c in ' ,.\t-+/%$')
+            if digit_count / len(stripped) > 0.80:
+                continue
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = text.strip()
+
+    if len(text.split()) < 50:
+        return False, text, "too short after stripping"
+
+    return True, text, None
+
+
+def clean_and_filter(text):
+    """Combined deep_clean + smart_filter for multiprocessing.
+    Returns (cleaned_text_or_None, drop_reason_or_None, boilerplate_chars_stripped).
+    """
+    result = deep_clean(text)
+    if result is None:
+        return None, "deep_clean_drop", 0
+
+    keep, stripped, reason = smart_filter(result)
+    if not keep:
+        return None, reason, 0
+
+    boilerplate = len(result) - len(stripped)
+    return stripped, None, boilerplate
+
+
+# ==============================================================================
+# STREAMING CHUNK WRITER (never accumulates full token stream)
+# ==============================================================================
+
+class StreamingChunkWriter:
+    """Writes litdata chunks incrementally. Flushes when buffer hits target size."""
+
+    def __init__(self, output_dir, config):
+        self.output_dir = Path(output_dir)
+        os.makedirs(str(self.output_dir), exist_ok=True)
+        self.config = config
+        self.dtype_size = DTYPE().itemsize
+        self.tokens_per_chunk = (CHUNK_BYTES_TARGET // self.dtype_size // BLOCK_SIZE) * BLOCK_SIZE
+        self.buffer = []
+        self.chunks_metadata = []
+        self.chunk_idx = 0
+        self.total_tokens = 0
+
+    def add_document(self, token_ids):
+        """Add one document's token IDs + EOS. Flushes chunks as needed."""
+        self.buffer.extend(token_ids)
+        self.buffer.append(EOS_TOKEN_ID)
+        # Flush full chunks
+        while len(self.buffer) >= self.tokens_per_chunk:
+            self._flush_chunk()
+
+    def add_tokens_batch(self, encoded_batch):
+        """Add a batch of encoded documents efficiently."""
+        for enc in encoded_batch:
+            ids = enc.ids
+            self.buffer.extend(ids)
+            self.buffer.append(EOS_TOKEN_ID)
+        while len(self.buffer) >= self.tokens_per_chunk:
+            self._flush_chunk()
+
+    def _flush_chunk(self):
+        """Write one chunk from the buffer."""
+        if len(self.buffer) < BLOCK_SIZE:
+            return
+
+        take = min(len(self.buffer), self.tokens_per_chunk)
+        num_blocks = take // BLOCK_SIZE
+        if num_blocks == 0:
+            return
+        actual_tokens = num_blocks * BLOCK_SIZE
+
+        chunk_data = np.array(self.buffer[:actual_tokens], dtype=DTYPE)
+        self.buffer = self.buffer[actual_tokens:]
+
+        filename = f"chunk-0-{self.chunk_idx}.bin"
+        filepath = self.output_dir / filename
+
+        header_num = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * self.dtype_size)
+        header = np.concatenate([header_num, offsets])
+
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        self.chunks_metadata.append(meta)
+        self.total_tokens += actual_tokens
+        self.chunk_idx += 1
+
+        if self.chunk_idx % 25 == 0:
+            print(f"      Flushed chunk {self.chunk_idx} ({self.total_tokens:,} tokens written)")
+
+    def finalize(self):
+        """Flush remaining buffer and write index.json."""
+        while len(self.buffer) >= BLOCK_SIZE:
+            self._flush_chunk()
+
+        # Discard any remaining tokens that don't fill a block
+        discarded = len(self.buffer)
+        self.buffer = []
+
+        index = {
+            "chunks": self.chunks_metadata,
+            "config": self.config,
+            "updated_at": str(time.time()),
+        }
+        with open(self.output_dir / "index.json", "w") as f:
+            json.dump(index, f, indent=2)
+
+        return self.total_tokens, discarded
+
+
+# ==============================================================================
+# CHUNK READING
+# ==============================================================================
+
+def read_chunk_tokens(litdata_dir, chunk_meta):
+    """Read a single chunk's token data (no header)."""
+    chunk_path = litdata_dir / chunk_meta["filename"]
+    n_blocks = chunk_meta["chunk_size"]
+    header_ints = 1 + n_blocks + 1
+    header_bytes = header_ints * 4
+
+    with open(chunk_path, "rb") as f:
+        f.seek(header_bytes)
+        data = np.fromfile(f, dtype=DTYPE, count=chunk_meta["dim"])
+    return data
+
+
+def split_documents_from_tokens(token_array):
+    """Split token array by EOS into list of per-document token lists (as Python lists)."""
+    eos_positions = np.where(token_array == EOS_TOKEN_ID)[0]
+    docs = []
+    start = 0
+    for eos_pos in eos_positions:
+        if eos_pos > start:
+            docs.append(token_array[start:eos_pos].tolist())
+        start = eos_pos + 1
+    # Return remaining tokens after last EOS (partial doc carry-over)
+    remainder = token_array[start:] if start < len(token_array) else np.array([], dtype=DTYPE)
+    return docs, remainder
+
+
+# ==============================================================================
+# MAIN PIPELINE
+# ==============================================================================
+
+def main():
+    t_start = time.time()
+
+    print("Loading tokenizer...")
+    tokenizer = Tokenizer.from_file(TOKENIZER_PATH)
+
+    print(f"\n{'='*75}")
+    print(f"  RECLEAN litdata_3b (Memory-Efficient + Multiprocessing)")
+    print(f"  Input:   {INPUT_DIR}")
+    print(f"  Output:  {OUTPUT_DIR}")
+    print(f"  Workers: {NUM_WORKERS} CPU cores")
+    print(f"  Batch:   {CHUNKS_PER_BATCH} chunks at a time")
+    print(f"{'='*75}")
+
+    # Read index
+    with open(INPUT_DIR / "index.json") as f:
+        index = json.load(f)
+    all_chunk_metas = index["chunks"]
+    total_input_tokens = sum(c["dim"] for c in all_chunk_metas)
+    num_chunks = len(all_chunk_metas)
+
+    print(f"\n  Input: {num_chunks} chunks, {total_input_tokens:,} tokens")
+
+    # Output config (same format)
+    config = {
+        "block_size": BLOCK_SIZE,
+        "vocab_size": tokenizer.get_vocab_size(),
+    }
+
+    # Stats
+    total_docs_in = 0
+    total_docs_kept = 0
+    total_docs_dropped = 0
+    total_boilerplate = 0
+    drop_reasons = Counter()
+
+    # Initialize streaming writer
+    writer = StreamingChunkWriter(str(OUTPUT_DIR), config)
+
+    # Carry-over: partial document tokens spanning chunk boundaries
+    carry_over = np.array([], dtype=DTYPE)
+
+    # ── Process in batches of chunks ──────────────────────────────────────
+    num_batches = (num_chunks + CHUNKS_PER_BATCH - 1) // CHUNKS_PER_BATCH
+    print(f"  Processing in {num_batches} batches of {CHUNKS_PER_BATCH} chunks...\n")
+
+    # Create multiprocessing pool for cleaning
+    pool = Pool(processes=NUM_WORKERS)
+
+    for batch_idx in range(num_batches):
+        batch_start = batch_idx * CHUNKS_PER_BATCH
+        batch_end = min(batch_start + CHUNKS_PER_BATCH, num_chunks)
+        batch_metas = all_chunk_metas[batch_start:batch_end]
+
+        t_batch = time.time()
+        print(f"  ── Batch {batch_idx+1}/{num_batches} (chunks {batch_start}-{batch_end-1}) ──")
+
+        # 1. Read chunk tokens
+        batch_tokens_list = []
+        for cm in batch_metas:
+            batch_tokens_list.append(read_chunk_tokens(INPUT_DIR, cm))
+        batch_tokens = np.concatenate(batch_tokens_list)
+        del batch_tokens_list
+
+        # Prepend carry-over from previous batch
+        if len(carry_over) > 0:
+            batch_tokens = np.concatenate([carry_over, batch_tokens])
+            carry_over = np.array([], dtype=DTYPE)
+
+        batch_token_count = len(batch_tokens)
+
+        # 2. Split into documents
+        doc_token_lists, carry_over = split_documents_from_tokens(batch_tokens)
+        del batch_tokens
+        num_docs = len(doc_token_lists)
+        total_docs_in += num_docs
+
+        # 3. Decode documents to text (tokenizer.decode_batch is multithreaded Rust)
+        t_dec = time.time()
+        raw_texts = tokenizer.decode_batch(doc_token_lists, skip_special_tokens=False)
+        del doc_token_lists
+        dec_time = time.time() - t_dec
+
+        # 4. Clean + smart filter in parallel across all cores
+        t_clean = time.time()
+        results = pool.map(clean_and_filter, raw_texts, chunksize=512)
+        del raw_texts
+        clean_time = time.time() - t_clean
+
+        # Collect cleaned texts
+        cleaned_texts = []
+        batch_dropped = 0
+        batch_boilerplate = 0
+        for cleaned, reason, bp in results:
+            if cleaned is not None:
+                cleaned_texts.append(cleaned)
+                batch_boilerplate += bp
+            else:
+                batch_dropped += 1
+                drop_reasons[reason] += 1
+        del results
+
+        batch_kept = len(cleaned_texts)
+        total_docs_kept += batch_kept
+        total_docs_dropped += batch_dropped
+        total_boilerplate += batch_boilerplate
+
+        # 5. Re-tokenize + stream to writer (encode_batch is multithreaded Rust)
+        t_tok = time.time()
+        for i in range(0, len(cleaned_texts), ENCODE_BATCH):
+            sub = cleaned_texts[i:i+ENCODE_BATCH]
+            encoded = tokenizer.encode_batch(sub, add_special_tokens=False)
+            writer.add_tokens_batch(encoded)
+        del cleaned_texts
+        tok_time = time.time() - t_tok
+
+        elapsed = time.time() - t_batch
+        print(f"    {batch_token_count:,} tokens → {num_docs:,} docs → kept {batch_kept:,} / dropped {batch_dropped:,}")
+        print(f"    decode {dec_time:.1f}s | clean {clean_time:.1f}s | tokenize {tok_time:.1f}s | total {elapsed:.1f}s")
+        print(f"    Running: {total_docs_kept:,} kept, {total_docs_dropped:,} dropped, {writer.total_tokens:,} tokens written")
+
+    pool.close()
+    pool.join()
+
+    # Handle carry-over (last partial doc if any)
+    if len(carry_over) > 0:
+        text = tokenizer.decode(carry_over.tolist(), skip_special_tokens=False)
+        result = deep_clean(text)
+        if result is not None:
+            keep, stripped, reason = smart_filter(result)
+            if keep:
+                encoded = tokenizer.encode(stripped, add_special_tokens=False)
+                writer.buffer.extend(encoded.ids)
+                writer.buffer.append(EOS_TOKEN_ID)
+                total_docs_kept += 1
+                total_boilerplate += len(result) - len(stripped)
+            else:
+                total_docs_dropped += 1
+                drop_reasons[reason] += 1
+        else:
+            total_docs_dropped += 1
+            drop_reasons["deep_clean_drop"] += 1
+        total_docs_in += 1
+
+    # Finalize output
+    final_tokens, discarded = writer.finalize()
+
+    total_time = time.time() - t_start
+
+    # ── Report ────────────────────────────────────────────────────────────
+    print(f"\n{'='*75}")
+    print(f"  RECLEAN REPORT - litdata_3b")
+    print(f"{'='*75}")
+    print(f"\n  Total time: {total_time:.0f}s ({total_time/60:.1f} min)")
+    print(f"  Workers: {NUM_WORKERS} CPU cores")
+    print(f"\n  INPUT")
+    print(f"  {'-'*60}")
+    print(f"    Chunks:    {num_chunks}")
+    print(f"    Tokens:    {total_input_tokens:,}")
+    print(f"    Documents: {total_docs_in:,}")
+    print(f"\n  CLEANING")
+    print(f"  {'-'*60}")
+    print(f"    Kept:      {total_docs_kept:,}")
+    print(f"    Dropped:   {total_docs_dropped:,} ({total_docs_dropped/(max(total_docs_in,1))*100:.2f}%)")
+    if drop_reasons:
+        print(f"    Drop reasons:")
+        for reason, count in sorted(drop_reasons.items(), key=lambda x: -x[1]):
+            print(f"      {reason:<35} {count:>8,}")
+    print(f"    Boilerplate stripped: {total_boilerplate:,} chars")
+    print(f"\n  OUTPUT")
+    print(f"  {'-'*60}")
+    print(f"    Location:  {OUTPUT_DIR}")
+    print(f"    Chunks:    {writer.chunk_idx}")
+    print(f"    Tokens:    {final_tokens:,}")
+    print(f"    Discarded: {discarded} trailing tokens (< 1 block)")
+    print(f"    Format:    litdata binary (int32, BLOCK_SIZE=1025, EOS=0)")
+    diff = total_input_tokens - final_tokens
+    print(f"\n  Token change: {total_input_tokens:,} → {final_tokens:,}")
+    print(f"  Difference:   {diff:,} ({diff/max(total_input_tokens,1)*100:.2f}%)")
+    print(f"\n{'='*75}")
+
+    # Save report
+    report_lines = [
+        f"RECLEAN REPORT - litdata_3b",
+        f"Time: {total_time:.0f}s ({total_time/60:.1f} min)",
+        f"Workers: {NUM_WORKERS}",
+        f"",
+        f"INPUT:  {num_chunks} chunks, {total_input_tokens:,} tokens, {total_docs_in:,} docs",
+        f"OUTPUT: {writer.chunk_idx} chunks, {final_tokens:,} tokens",
+        f"",
+        f"Docs kept:    {total_docs_kept:,}",
+        f"Docs dropped: {total_docs_dropped:,}",
+    ]
+    if drop_reasons:
+        for reason, count in sorted(drop_reasons.items(), key=lambda x: -x[1]):
+            report_lines.append(f"  {reason}: {count:,}")
+    report_lines.append(f"Boilerplate stripped: {total_boilerplate:,} chars")
+    report_lines.append(f"Token change: {total_input_tokens:,} -> {final_tokens:,} ({diff/max(total_input_tokens,1)*100:.2f}%)")
+
+    with open(OUTPUT_DIR / "CLEAN_REPORT.txt", "w", encoding="utf-8") as f:
+        f.write("\n".join(report_lines))
+    print(f"  Report saved to: {OUTPUT_DIR / 'CLEAN_REPORT.txt'}")
+    print(f"  Done!")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/reclean_english.py

@@ -0,0 +1,610 @@
+# -*- coding: utf-8 -*-
+"""
+Reclean litdata_english ONLY (separate from the main reclean running on 3b).
+
+Decodes all docs, applies English cleaning, re-tokenizes, rebuilds chunks.
+Includes detailed BEFORE vs AFTER comparison report with sample diffs.
+"""
+
+import json
+import os
+import re
+import time
+import unicodedata
+from pathlib import Path
+from collections import Counter
+
+import numpy as np
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+
+# -- Load tokenizer -----------------------------------------------------------
+print("Loading tokenizer...")
+tokenizer = Tokenizer.from_file(
+    str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+)
+
+# ==============================================================================
+# TEXT CLEANING (same pipeline as reclean_litdata.py)
+# ==============================================================================
+
+CONTROL_CHARS = [
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f", "\x7f",
+    "\ufeff", "\ufffd",
+]
+
+HTML_ENTITIES = [
+    ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+    ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"),
+    ("&nbsp;", " "), ("&mdash;", " - "), ("&ndash;", "-"),
+    ("&hellip;", "..."), ("&laquo;", '"'), ("&raquo;", '"'),
+    ("&bull;", "- "), ("&middot;", " "), ("&copy;", "(c)"),
+    ("&reg;", "(R)"), ("&trade;", "(TM)"), ("&deg;", " degrees"),
+]
+
+RE_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_FILE_PATH = re.compile(r'(?:[A-Z]:\\|/(?:home|usr|var|etc|opt)/)\S+')
+RE_HTML_TAG = re.compile(r'</?[a-zA-Z][a-zA-Z0-9]*(?:\s[^>]*)?\s*/?>')
+RE_HTML_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+RE_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
+RE_IMPORT = re.compile(r'^(?:import |from \S+ import |#include |using namespace |require\()', re.M)
+RE_REPEATED_LINE = re.compile(r'^(.{20,})\n(?:\1\n?)+', re.M)
+RE_REPEATED_PUNCT = re.compile(r'([!?.])\1{3,}')
+RE_REPEATED_CHAR = re.compile(r'(.)\1{5,}')
+RE_REPEATED_WORD = re.compile(r'\b(\w+)(?:\s+\1){2,}\b', re.I)
+RE_MULTI_NEWLINE = re.compile(r'\n{4,}')
+RE_MULTI_SPACE = re.compile(r'[ \t]{2,}')
+RE_TRAILING_SPACE = re.compile(r'[ \t]+$', re.M)
+RE_NO_SPACE_AFTER_PERIOD = re.compile(r'([.!?])([A-Z])')
+RE_DOUBLE_PERIOD = re.compile(r'\.{2}(?!\.)')
+RE_SPACE_BEFORE_PUNCT = re.compile(r'\s+([.,;:!?])')
+
+
+# -- Stats trackers -----------------------------------------------------------
+class CleaningStats:
+    def __init__(self):
+        self.control_chars_removed = 0
+        self.html_entities_fixed = 0
+        self.html_tags_removed = 0
+        self.urls_removed = 0
+        self.emails_removed = 0
+        self.code_blocks_removed = 0
+        self.repeated_lines_fixed = 0
+        self.repeated_punct_fixed = 0
+        self.repeated_words_fixed = 0
+        self.punctuation_fixed = 0
+        self.smart_quotes_fixed = 0
+        self.junk_lines_removed = 0
+        self.lines_capitalized = 0
+        self.duplicate_paragraphs_removed = 0
+        self.docs_dropped_short = 0
+        self.docs_dropped_nonenglish = 0
+        self.total_chars_before = 0
+        self.total_chars_after = 0
+
+stats = CleaningStats()
+
+
+def clean_text_tracked(text):
+    """Clean text with detailed change tracking for the report."""
+    global stats
+    if not text or len(text.strip()) < 30:
+        stats.docs_dropped_short += 1
+        return None
+
+    original = text
+    stats.total_chars_before += len(text)
+
+    # 1. Unicode normalization
+    text = unicodedata.normalize("NFKC", text)
+
+    # 2. Control chars
+    for ch in CONTROL_CHARS:
+        count = text.count(ch)
+        if count:
+            stats.control_chars_removed += count
+            text = text.replace(ch, "")
+
+    # 3. HTML entities
+    for old, new in HTML_ENTITIES:
+        count = text.count(old)
+        if count:
+            stats.html_entities_fixed += count
+            text = text.replace(old, new)
+
+    # 4. HTML tags/comments
+    m = RE_HTML_COMMENT.findall(text)
+    stats.html_tags_removed += len(m)
+    text = RE_HTML_COMMENT.sub("", text)
+    m = RE_HTML_TAG.findall(text)
+    stats.html_tags_removed += len(m)
+    text = RE_HTML_TAG.sub("", text)
+
+    # 5. URLs, emails, file paths
+    stats.urls_removed += len(RE_URL.findall(text))
+    text = RE_URL.sub("", text)
+    stats.emails_removed += len(RE_EMAIL.findall(text))
+    text = RE_EMAIL.sub("", text)
+    text = RE_FILE_PATH.sub("", text)
+
+    # 6. Code blocks
+    stats.code_blocks_removed += len(RE_CODE_BLOCK.findall(text))
+    text = RE_CODE_BLOCK.sub("", text)
+
+    # 7. Repeated content
+    stats.repeated_lines_fixed += len(RE_REPEATED_LINE.findall(text))
+    text = RE_REPEATED_LINE.sub(r'\1', text)
+    stats.repeated_punct_fixed += len(RE_REPEATED_PUNCT.findall(text))
+    text = RE_REPEATED_PUNCT.sub(r'\1\1\1', text)
+    text = RE_REPEATED_CHAR.sub(r'\1\1\1', text)
+    stats.repeated_words_fixed += len(RE_REPEATED_WORD.findall(text))
+    text = RE_REPEATED_WORD.sub(r'\1', text)
+
+    # 8. Whitespace
+    text = text.replace('\t', ' ')
+    text = RE_TRAILING_SPACE.sub('', text)
+    text = RE_MULTI_SPACE.sub(' ', text)
+    text = RE_MULTI_NEWLINE.sub('\n\n\n', text)
+
+    # 9. Punctuation
+    p1 = len(RE_DOUBLE_PERIOD.findall(text))
+    p2 = len(RE_NO_SPACE_AFTER_PERIOD.findall(text))
+    p3 = len(RE_SPACE_BEFORE_PUNCT.findall(text))
+    stats.punctuation_fixed += p1 + p2 + p3
+    text = RE_DOUBLE_PERIOD.sub('.', text)
+    text = RE_NO_SPACE_AFTER_PERIOD.sub(r'\1 \2', text)
+    text = RE_SPACE_BEFORE_PUNCT.sub(r'\1', text)
+
+    # 10. Smart quotes -> ASCII
+    sq = 0
+    for ch in ['\u2018', '\u2019', '\u201c', '\u201d', '\u2013', '\u2014', '\u2026', '\u2022', '\u00b7', '\u00a0']:
+        c = text.count(ch)
+        if c:
+            sq += c
+    stats.smart_quotes_fixed += sq
+    text = text.replace('\u2018', "'").replace('\u2019', "'")
+    text = text.replace('\u201c', '"').replace('\u201d', '"')
+    text = text.replace('\u2013', '-').replace('\u2014', ' - ')
+    text = text.replace('\u2026', '...')
+    text = text.replace('\u2022', '- ')
+    text = text.replace('\u00b7', ' ')
+    text = text.replace('\u00a0', ' ')
+
+    # 11. Line-by-line cleaning
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            clean_lines.append('')
+            continue
+        if len(line) > 10:
+            alpha_count = sum(1 for c in line if c.isalpha())
+            if alpha_count / len(line) < 0.40:
+                stats.junk_lines_removed += 1
+                continue
+        if line.count('|') > 3 or line.count('{') > 2 or line.count('}') > 2:
+            stats.junk_lines_removed += 1
+            continue
+        if RE_IMPORT.match(line):
+            stats.junk_lines_removed += 1
+            continue
+        if line and line[0].isalpha() and line[0].islower():
+            if not clean_lines or clean_lines[-1] == '' or clean_lines[-1].rstrip().endswith(('.', '!', '?', ':')):
+                line = line[0].upper() + line[1:]
+                stats.lines_capitalized += 1
+        clean_lines.append(line)
+
+    text = '\n'.join(clean_lines)
+    text = text.strip()
+
+    # 13. Deduplicate paragraphs
+    paragraphs = text.split('\n\n')
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        p_stripped = p.strip()
+        if not p_stripped:
+            continue
+        p_key = ' '.join(p_stripped.lower().split())
+        if p_key not in seen:
+            seen.add(p_key)
+            unique_paragraphs.append(p_stripped)
+        else:
+            stats.duplicate_paragraphs_removed += 1
+    text = '\n\n'.join(unique_paragraphs)
+
+    # 14. Final quality gate
+    text = text.strip()
+    if len(text) < 50:
+        stats.docs_dropped_short += 1
+        return None
+    if len(text.split()) < 10:
+        stats.docs_dropped_short += 1
+        return None
+    ascii_count = sum(1 for c in text if ord(c) < 128)
+    if ascii_count / max(len(text), 1) < 0.85:
+        stats.docs_dropped_nonenglish += 1
+        return None
+
+    stats.total_chars_after += len(text)
+    return text
+
+
+# ==============================================================================
+# LITDATA I/O (same as reclean_litdata.py)
+# ==============================================================================
+
+def read_all_tokens(litdata_dir):
+    with open(litdata_dir / "index.json") as f:
+        index = json.load(f)
+    chunks = index["chunks"]
+    total_tokens = sum(c["dim"] for c in chunks)
+    print(f"  Reading {len(chunks)} chunks ({total_tokens:,} tokens)...")
+    all_tokens = np.empty(total_tokens, dtype=DTYPE)
+    pos = 0
+    for i, chunk in enumerate(chunks):
+        chunk_path = litdata_dir / chunk["filename"]
+        n_blocks = chunk["chunk_size"]
+        header_ints = 1 + n_blocks + 1
+        header_bytes = header_ints * 4
+        with open(chunk_path, "rb") as f:
+            f.seek(header_bytes)
+            data = np.fromfile(f, dtype=DTYPE, count=chunk["dim"])
+        all_tokens[pos:pos + len(data)] = data
+        pos += len(data)
+        if (i + 1) == len(chunks):
+            print(f"    Read {i+1}/{len(chunks)} chunks ({pos:,} tokens)")
+    return all_tokens[:pos]
+
+
+def split_documents(token_stream):
+    eos_positions = np.where(token_stream == EOS_TOKEN_ID)[0]
+    docs = []
+    start = 0
+    for eos_pos in eos_positions:
+        if eos_pos > start:
+            docs.append(token_stream[start:eos_pos])
+        start = eos_pos + 1
+    if start < len(token_stream):
+        docs.append(token_stream[start:])
+    return docs
+
+
+def write_litdata_chunks(output_dir, token_stream, config):
+    os.makedirs(output_dir, exist_ok=True)
+    dtype_size = DTYPE().itemsize
+    tokens_per_chunk = CHUNK_BYTES_TARGET // dtype_size
+    tokens_per_chunk = (tokens_per_chunk // BLOCK_SIZE) * BLOCK_SIZE
+    chunks_metadata = []
+    pos = 0
+    chunk_idx = 0
+    while pos < len(token_stream):
+        remaining = len(token_stream) - pos
+        chunk_tokens = min(tokens_per_chunk, remaining)
+        num_blocks = chunk_tokens // BLOCK_SIZE
+        if num_blocks == 0:
+            break
+        actual_tokens = num_blocks * BLOCK_SIZE
+        chunk_data = token_stream[pos:pos + actual_tokens]
+        filename = f"chunk-0-{chunk_idx}.bin"
+        filepath = os.path.join(output_dir, filename)
+        header_num_items = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * dtype_size)
+        header = np.concatenate([header_num_items, offsets])
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        chunks_metadata.append(meta)
+        pos += actual_tokens
+        chunk_idx += 1
+        print(f"    Written chunk {chunk_idx} ({pos:,}/{len(token_stream):,} tokens)")
+    index = {"chunks": chunks_metadata, "config": config, "updated_at": str(time.time())}
+    with open(os.path.join(output_dir, "index.json"), "w") as f:
+        json.dump(index, f, indent=2)
+    return chunks_metadata
+
+
+# ==============================================================================
+# QUALITY METRICS (before/after comparison)
+# ==============================================================================
+
+def compute_quality_metrics(texts, label):
+    """Compute quality metrics on a list of texts."""
+    if not texts:
+        return {}
+    ascii_ratios = []
+    alpha_ratios = []
+    word_counts = []
+    sentence_counts = []
+    unique_word_ratios = []
+    url_counts = 0
+    html_counts = 0
+
+    for t in texts:
+        chars = len(t)
+        if chars == 0:
+            continue
+        ascii_ratios.append(sum(1 for c in t if ord(c) < 128) / chars)
+        alpha_ratios.append(sum(1 for c in t if c.isalpha()) / chars)
+        words = t.split()
+        word_counts.append(len(words))
+        sentence_counts.append(len(re.findall(r'[.!?]+\s', t)))
+        if words:
+            unique_word_ratios.append(len(set(w.lower() for w in words)) / len(words))
+        url_counts += len(re.findall(r'https?://\S+', t))
+        if re.search(r'<(html|body|div|span|script|style)\b', t, re.I):
+            html_counts += 1
+
+    n = len(texts)
+    return {
+        "label": label,
+        "documents": n,
+        "avg_ascii_ratio": sum(ascii_ratios) / max(len(ascii_ratios), 1),
+        "avg_alpha_ratio": sum(alpha_ratios) / max(len(alpha_ratios), 1),
+        "avg_word_count": sum(word_counts) / max(len(word_counts), 1),
+        "avg_sentences": sum(sentence_counts) / max(len(sentence_counts), 1),
+        "avg_unique_word_ratio": sum(unique_word_ratios) / max(len(unique_word_ratios), 1),
+        "total_urls": url_counts,
+        "docs_with_html": html_counts,
+        "min_words": min(word_counts) if word_counts else 0,
+        "max_words": max(word_counts) if word_counts else 0,
+    }
+
+
+def print_comparison(before_metrics, after_metrics):
+    """Print a side-by-side comparison table."""
+    print(f"\n  {'METRIC':<30} {'BEFORE':>15} {'AFTER':>15} {'CHANGE':>15}")
+    print(f"  {'='*75}")
+
+    rows = [
+        ("Documents", "documents", "d"),
+        ("Avg ASCII ratio", "avg_ascii_ratio", "f4"),
+        ("Avg alpha ratio", "avg_alpha_ratio", "f4"),
+        ("Avg word count", "avg_word_count", "f1"),
+        ("Avg sentences/doc", "avg_sentences", "f1"),
+        ("Avg unique word ratio", "avg_unique_word_ratio", "f4"),
+        ("URLs found", "total_urls", "d"),
+        ("Docs with HTML", "docs_with_html", "d"),
+        ("Min word count", "min_words", "d"),
+        ("Max word count", "max_words", "d"),
+    ]
+
+    for label, key, fmt in rows:
+        bv = before_metrics.get(key, 0)
+        av = after_metrics.get(key, 0)
+        if fmt == "d":
+            diff = av - bv
+            sign = "+" if diff > 0 else ""
+            print(f"  {label:<30} {bv:>15,} {av:>15,} {sign}{diff:>14,}")
+        else:
+            diff = av - bv
+            sign = "+" if diff > 0 else ""
+            print(f"  {label:<30} {bv:>15.4f} {av:>15.4f} {sign}{diff:>14.4f}")
+
+
+# ==============================================================================
+# MAIN
+# ==============================================================================
+
+if __name__ == "__main__":
+    t_start = time.time()
+    data_dir = ROOT / "Base" / "data"
+    input_dir = data_dir / "litdata_english"
+    output_dir = data_dir / "litdata_english_clean"
+
+    print(f"\n{'='*75}")
+    print(f"  RECLEANING: litdata_english (English Knowledge)")
+    print(f"  Input:  {input_dir}")
+    print(f"  Output: {output_dir}")
+    print(f"{'='*75}")
+
+    # 1. Read all tokens
+    t0 = time.time()
+    token_stream = read_all_tokens(input_dir)
+    print(f"  Read {len(token_stream):,} tokens in {time.time()-t0:.1f}s")
+
+    # 2. Split into documents
+    t1 = time.time()
+    doc_tokens = split_documents(token_stream)
+    print(f"  Found {len(doc_tokens):,} documents in {time.time()-t1:.1f}s")
+    del token_stream
+
+    # 3. Decode
+    t2 = time.time()
+    print(f"  Decoding {len(doc_tokens):,} documents...")
+    raw_texts = []
+    for i, doc in enumerate(doc_tokens):
+        text = tokenizer.decode(doc.tolist(), skip_special_tokens=False)
+        raw_texts.append(text)
+        if (i + 1) % 20000 == 0 or i == len(doc_tokens) - 1:
+            print(f"    Decoded {i+1:,}/{len(doc_tokens):,}")
+    del doc_tokens
+    print(f"  Decoded in {time.time()-t2:.1f}s")
+
+    # 4. Compute BEFORE metrics (ALL documents)
+    print(f"\n  Computing BEFORE quality metrics on ALL {len(raw_texts):,} documents...")
+    import random
+    random.seed(42)
+    sample_size = len(raw_texts)
+    before_metrics = compute_quality_metrics(raw_texts, "BEFORE")
+
+    # 5. Clean all documents (with tracking)
+    t3 = time.time()
+    print(f"\n  Cleaning {len(raw_texts):,} documents...")
+    cleaned_texts = []
+    dropped_texts = []  # keep some dropped for the report
+    before_after_pairs = []  # keep pairs for sample diffs
+    dropped = 0
+
+    for i, text in enumerate(raw_texts):
+        result = clean_text_tracked(text)
+        if result is not None:
+            cleaned_texts.append(result)
+            # Keep some before/after pairs for diff display
+            if len(before_after_pairs) < 20 and text != result:
+                before_after_pairs.append((text, result))
+        else:
+            dropped += 1
+            if len(dropped_texts) < 10:
+                dropped_texts.append(text)
+        if (i + 1) % 20000 == 0 or i == len(raw_texts) - 1:
+            print(f"    Processed {i+1:,}/{len(raw_texts):,} | kept={len(cleaned_texts):,} | dropped={dropped:,}")
+
+    del raw_texts
+    print(f"  Cleaning done in {time.time()-t3:.1f}s")
+    print(f"  Kept {len(cleaned_texts):,} | Dropped {dropped:,} ({dropped/max(dropped+len(cleaned_texts),1)*100:.1f}%)")
+
+    # 6. Compute AFTER metrics (ALL documents)
+    print(f"\n  Computing AFTER quality metrics on ALL {len(cleaned_texts):,} documents...")
+    after_metrics = compute_quality_metrics(cleaned_texts, "AFTER")
+
+    # 7. Re-tokenize
+    t4 = time.time()
+    print(f"\n  Re-tokenizing {len(cleaned_texts):,} documents...")
+    all_token_ids = []
+    total_new_tokens = 0
+    ENCODE_BATCH = 10000
+    for i in range(0, len(cleaned_texts), ENCODE_BATCH):
+        batch = cleaned_texts[i:i+ENCODE_BATCH]
+        encoded = tokenizer.encode_batch(batch, add_special_tokens=False)
+        for enc in encoded:
+            ids = enc.ids
+            all_token_ids.extend(ids)
+            all_token_ids.append(EOS_TOKEN_ID)
+            total_new_tokens += len(ids) + 1
+        done = min(i + ENCODE_BATCH, len(cleaned_texts))
+        if done == len(cleaned_texts):
+            print(f"    Tokenized {done:,}/{len(cleaned_texts):,} ({total_new_tokens:,} tokens)")
+    del cleaned_texts
+    print(f"  Tokenized in {time.time()-t4:.1f}s")
+
+    # 8. Write chunks
+    t5 = time.time()
+    print(f"\n  Building & writing litdata chunks...")
+    new_stream = np.array(all_token_ids, dtype=DTYPE)
+    del all_token_ids
+    with open(input_dir / "index.json") as f:
+        config = json.load(f)["config"]
+    chunks = write_litdata_chunks(str(output_dir), new_stream, config)
+    total_in = sum(c["dim"] for c in json.load(open(input_dir / "index.json"))["chunks"])
+    total_out = sum(c["dim"] for c in chunks)
+    print(f"  Written {len(chunks)} chunks in {time.time()-t5:.1f}s")
+
+    # ================================================================
+    # DETAILED COMPARISON REPORT
+    # ================================================================
+    elapsed = time.time() - t_start
+
+    report = []
+    report.append("")
+    report.append("=" * 75)
+    report.append("  LITDATA_ENGLISH RECLEANING - FULL COMPARISON REPORT")
+    report.append("=" * 75)
+
+    report.append(f"\n  Processing time: {elapsed:.0f}s ({elapsed/60:.1f} min)")
+
+    report.append(f"\n  TOKEN SUMMARY")
+    report.append(f"  {'-'*60}")
+    report.append(f"  Input tokens:  {total_in:>15,}")
+    report.append(f"  Output tokens: {total_out:>15,}")
+    report.append(f"  Removed:       {total_in - total_out:>15,} ({(total_in-total_out)/total_in*100:.2f}%)")
+
+    report.append(f"\n  CLEANING OPERATIONS PERFORMED")
+    report.append(f"  {'-'*60}")
+    report.append(f"  Control chars removed:        {stats.control_chars_removed:>10,}")
+    report.append(f"  HTML entities fixed:           {stats.html_entities_fixed:>10,}")
+    report.append(f"  HTML tags removed:             {stats.html_tags_removed:>10,}")
+    report.append(f"  URLs removed:                  {stats.urls_removed:>10,}")
+    report.append(f"  Emails removed:                {stats.emails_removed:>10,}")
+    report.append(f"  Code blocks removed:           {stats.code_blocks_removed:>10,}")
+    report.append(f"  Repeated lines fixed:          {stats.repeated_lines_fixed:>10,}")
+    report.append(f"  Repeated punctuation fixed:    {stats.repeated_punct_fixed:>10,}")
+    report.append(f"  Repeated words fixed:          {stats.repeated_words_fixed:>10,}")
+    report.append(f"  Punctuation fixes:             {stats.punctuation_fixed:>10,}")
+    report.append(f"  Smart quotes normalized:       {stats.smart_quotes_fixed:>10,}")
+    report.append(f"  Junk lines removed:            {stats.junk_lines_removed:>10,}")
+    report.append(f"  Lines capitalized:             {stats.lines_capitalized:>10,}")
+    report.append(f"  Duplicate paragraphs removed:  {stats.duplicate_paragraphs_removed:>10,}")
+    report.append(f"  Docs dropped (too short):      {stats.docs_dropped_short:>10,}")
+    report.append(f"  Docs dropped (non-English):    {stats.docs_dropped_nonenglish:>10,}")
+    report.append(f"  Total chars before:  {stats.total_chars_before:>15,}")
+    report.append(f"  Total chars after:   {stats.total_chars_after:>15,}")
+    char_diff = stats.total_chars_before - stats.total_chars_after
+    report.append(f"  Chars cleaned out:   {char_diff:>15,} ({char_diff/max(stats.total_chars_before,1)*100:.2f}%)")
+
+    report.append(f"\n  QUALITY METRICS COMPARISON (ALL {sample_size:,} docs)")
+    report.append(f"  {'-'*60}")
+    # Print table header
+    report.append(f"  {'METRIC':<30} {'BEFORE':>12} {'AFTER':>12} {'CHANGE':>12}")
+    report.append(f"  {'='*66}")
+    metrics_rows = [
+        ("Documents", "documents", "d"),
+        ("Avg ASCII ratio", "avg_ascii_ratio", "f"),
+        ("Avg alpha ratio", "avg_alpha_ratio", "f"),
+        ("Avg word count", "avg_word_count", "f1"),
+        ("Avg sentences/doc", "avg_sentences", "f1"),
+        ("Avg unique word ratio", "avg_unique_word_ratio", "f"),
+        ("URLs found", "total_urls", "d"),
+        ("Docs with HTML", "docs_with_html", "d"),
+    ]
+    for label, key, fmt in metrics_rows:
+        bv = before_metrics.get(key, 0)
+        av = after_metrics.get(key, 0)
+        diff = av - bv
+        sign = "+" if diff > 0 else ""
+        if fmt == "d":
+            report.append(f"  {label:<30} {bv:>12,} {av:>12,} {sign}{diff:>11,}")
+        elif fmt == "f1":
+            report.append(f"  {label:<30} {bv:>12.1f} {av:>12.1f} {sign}{diff:>11.1f}")
+        else:
+            report.append(f"  {label:<30} {bv:>12.4f} {av:>12.4f} {sign}{diff:>11.4f}")
+
+    # Sample diffs
+    report.append(f"\n  SAMPLE BEFORE/AFTER DIFFS (showing {min(5, len(before_after_pairs))} pairs)")
+    report.append(f"  {'-'*60}")
+    for idx, (before, after) in enumerate(before_after_pairs[:5]):
+        report.append(f"\n  --- Sample {idx+1} ---")
+        report.append(f"  BEFORE ({len(before)} chars, {len(before.split())} words):")
+        preview_b = before[:300].replace('\n', ' | ')
+        report.append(f"    \"{preview_b}...\"")
+        report.append(f"  AFTER  ({len(after)} chars, {len(after.split())} words):")
+        preview_a = after[:300].replace('\n', ' | ')
+        report.append(f"    \"{preview_a}...\"")
+
+    # Dropped samples
+    if dropped_texts:
+        report.append(f"\n  EXAMPLES OF DROPPED DOCUMENTS ({min(3, len(dropped_texts))} shown)")
+        report.append(f"  {'-'*60}")
+        for idx, dt in enumerate(dropped_texts[:3]):
+            report.append(f"  Dropped #{idx+1} ({len(dt)} chars): \"{dt[:150]}...\"")
+
+    report.append(f"\n  {'='*60}")
+    report.append(f"  VERDICT: litdata_english cleaned and ready!")
+    report.append(f"  Output: {output_dir}")
+    report.append(f"  {'='*60}")
+
+    # Print the full report
+    full_report = '\n'.join(report)
+    print(full_report)
+
+    # Save report
+    report_path = output_dir / "CLEANING_REPORT.txt"
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(full_report)
+    print(f"\n  Report saved to: {report_path}")

Base/scripts/reclean_litdata.py

@@ -0,0 +1,444 @@
+# -*- coding: utf-8 -*-
+"""
+Reclean & normalize all pretraining data for optimal 100M model learning.
+
+Reads litdata_3b and litdata_english, decodes all documents back to text,
+applies comprehensive English cleaning/normalization, re-tokenizes, and
+writes new litdata chunks.
+"""
+
+import json
+import os
+import re
+import sys
+import time
+import unicodedata
+from pathlib import Path
+
+import numpy as np
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+DTYPE_INDEX = 16
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+
+# -- Load tokenizer -----------------------------------------------------------
+print("Loading tokenizer...")
+tokenizer = Tokenizer.from_file(
+    str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+)
+
+# ==============================================================================
+# TEXT CLEANING PIPELINE
+# ==============================================================================
+
+# Null / control chars to strip
+CONTROL_CHARS = [
+    "\x00", "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07",
+    "\x08", "\x0b", "\x0c", "\x0e", "\x0f", "\x10", "\x11", "\x12",
+    "\x13", "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1a",
+    "\x1b", "\x1c", "\x1d", "\x1e", "\x1f", "\x7f",
+    "\ufeff", "\ufffd",
+]
+
+# HTML entities
+HTML_ENTITIES = [
+    ("&amp;", "&"), ("&lt;", "<"), ("&gt;", ">"),
+    ("&quot;", '"'), ("&#39;", "'"), ("&apos;", "'"),
+    ("&nbsp;", " "), ("&mdash;", " - "), ("&ndash;", "-"),
+    ("&hellip;", "..."), ("&laquo;", '"'), ("&raquo;", '"'),
+    ("&bull;", "- "), ("&middot;", " "), ("&copy;", "(c)"),
+    ("&reg;", "(R)"), ("&trade;", "(TM)"), ("&deg;", " degrees"),
+]
+
+# URL/email/path patterns
+RE_URL = re.compile(r'https?://\S+|www\.\S+', re.I)
+RE_EMAIL = re.compile(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b')
+RE_FILE_PATH = re.compile(r'(?:[A-Z]:\\|/(?:home|usr|var|etc|opt)/)\S+')
+
+# HTML tag leftovers
+RE_HTML_TAG = re.compile(r'</?[a-zA-Z][a-zA-Z0-9]*(?:\s[^>]*)?\s*/?>')
+RE_HTML_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+
+# Code/programming artifacts
+RE_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
+RE_IMPORT = re.compile(r'^(?:import |from \S+ import |#include |using namespace |require\()', re.M)
+
+# Repeated content
+RE_REPEATED_LINE = re.compile(r'^(.{20,})\n(?:\1\n?)+', re.M)
+RE_REPEATED_PUNCT = re.compile(r'([!?.])\1{3,}')
+RE_REPEATED_CHAR = re.compile(r'(.)\1{5,}')
+RE_REPEATED_WORD = re.compile(r'\b(\w+)(?:\s+\1){2,}\b', re.I)
+
+# Whitespace
+RE_MULTI_NEWLINE = re.compile(r'\n{4,}')
+RE_MULTI_SPACE = re.compile(r'[ \t]{2,}')
+RE_TRAILING_SPACE = re.compile(r'[ \t]+$', re.M)
+
+# Sentence fixing
+RE_NO_SPACE_AFTER_PERIOD = re.compile(r'([.!?])([A-Z])')
+RE_DOUBLE_PERIOD = re.compile(r'\.{2}(?!\.)')  # .. but not ...
+RE_SPACE_BEFORE_PUNCT = re.compile(r'\s+([.,;:!?])')
+
+
+def clean_text(text):
+    """Apply full cleaning pipeline to a single document text."""
+    if not text or len(text.strip()) < 30:
+        return None
+
+    # 1. Unicode normalization
+    text = unicodedata.normalize("NFKC", text)
+
+    # 2. Strip control characters
+    for ch in CONTROL_CHARS:
+        text = text.replace(ch, "")
+
+    # 3. Fix HTML entities
+    for old, new in HTML_ENTITIES:
+        text = text.replace(old, new)
+
+    # 4. Remove HTML tags and comments
+    text = RE_HTML_COMMENT.sub("", text)
+    text = RE_HTML_TAG.sub("", text)
+
+    # 5. Remove URLs, emails, file paths
+    text = RE_URL.sub("", text)
+    text = RE_EMAIL.sub("", text)
+    text = RE_FILE_PATH.sub("", text)
+
+    # 6. Remove code blocks
+    text = RE_CODE_BLOCK.sub("", text)
+
+    # 7. Fix repeated content
+    text = RE_REPEATED_LINE.sub(r'\1', text)
+    text = RE_REPEATED_PUNCT.sub(r'\1\1\1', text)
+    text = RE_REPEATED_CHAR.sub(r'\1\1\1', text)
+    text = RE_REPEATED_WORD.sub(r'\1', text)
+
+    # 8. Normalize whitespace
+    text = text.replace('\t', ' ')
+    text = RE_TRAILING_SPACE.sub('', text)
+    text = RE_MULTI_SPACE.sub(' ', text)
+    text = RE_MULTI_NEWLINE.sub('\n\n\n', text)
+
+    # 9. Fix punctuation
+    text = RE_DOUBLE_PERIOD.sub('.', text)
+    text = RE_NO_SPACE_AFTER_PERIOD.sub(r'\1 \2', text)
+    text = RE_SPACE_BEFORE_PUNCT.sub(r'\1', text)
+
+    # 10. Normalize smart quotes and special punctuation to ASCII
+    text = text.replace('\u2018', "'").replace('\u2019', "'")
+    text = text.replace('\u201c', '"').replace('\u201d', '"')
+    text = text.replace('\u2013', '-').replace('\u2014', ' - ')
+    text = text.replace('\u2026', '...')
+    text = text.replace('\u2022', '- ')
+    text = text.replace('\u00b7', ' ')
+    text = text.replace('\u00a0', ' ')  # non-breaking space
+
+    # 11. Process line by line: capitalize sentence starts, remove junk lines
+    lines = text.split('\n')
+    clean_lines = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            clean_lines.append('')
+            continue
+
+        # Skip lines that are mostly non-alphabetic (tables, code, etc.)
+        if len(line) > 10:
+            alpha_count = sum(1 for c in line if c.isalpha())
+            if alpha_count / len(line) < 0.40:
+                continue
+
+        # Skip lines with too many special chars (tables, markup)
+        if line.count('|') > 3 or line.count('{') > 2 or line.count('}') > 2:
+            continue
+
+        # Skip lines that look like code imports
+        if RE_IMPORT.match(line):
+            continue
+
+        # Capitalize first letter of sentences
+        if line and line[0].isalpha() and line[0].islower():
+            if not clean_lines or clean_lines[-1] == '' or clean_lines[-1].rstrip().endswith(('.', '!', '?', ':')):
+                line = line[0].upper() + line[1:]
+
+        clean_lines.append(line)
+
+    text = '\n'.join(clean_lines)
+
+    # 12. Remove leading/trailing whitespace
+    text = text.strip()
+
+    # 13. Remove duplicate paragraphs
+    paragraphs = text.split('\n\n')
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        p_stripped = p.strip()
+        if not p_stripped:
+            continue
+        p_key = ' '.join(p_stripped.lower().split())
+        if p_key not in seen:
+            seen.add(p_key)
+            unique_paragraphs.append(p_stripped)
+    text = '\n\n'.join(unique_paragraphs)
+
+    # 14. Final quality gate
+    text = text.strip()
+    if len(text) < 50:
+        return None
+    word_count = len(text.split())
+    if word_count < 10:
+        return None
+    # Must be mostly ASCII/English
+    ascii_count = sum(1 for c in text if ord(c) < 128)
+    if ascii_count / max(len(text), 1) < 0.85:
+        return None
+
+    return text
+
+
+# ==============================================================================
+# LITDATA I/O
+# ==============================================================================
+
+def read_all_tokens(litdata_dir):
+    """Read all chunks and return the full flat token stream as numpy array."""
+    with open(litdata_dir / "index.json") as f:
+        index = json.load(f)
+
+    chunks = index["chunks"]
+    total_tokens = sum(c["dim"] for c in chunks)
+    print(f"  Reading {len(chunks)} chunks ({total_tokens:,} tokens)...")
+
+    all_tokens = np.empty(total_tokens, dtype=DTYPE)
+    pos = 0
+
+    for i, chunk in enumerate(chunks):
+        chunk_path = litdata_dir / chunk["filename"]
+        n_blocks = chunk["chunk_size"]
+        header_ints = 1 + n_blocks + 1
+        header_bytes = header_ints * 4
+
+        with open(chunk_path, "rb") as f:
+            f.seek(header_bytes)
+            data = np.fromfile(f, dtype=DTYPE, count=chunk["dim"])
+
+        all_tokens[pos:pos + len(data)] = data
+        pos += len(data)
+
+        if (i + 1) % 50 == 0 or i == len(chunks) - 1:
+            print(f"    Read {i+1}/{len(chunks)} chunks ({pos:,} tokens)")
+
+    return all_tokens[:pos]
+
+
+def split_documents(token_stream):
+    """Split token stream by EOS token (0) into individual documents."""
+    eos_positions = np.where(token_stream == EOS_TOKEN_ID)[0]
+    docs = []
+    start = 0
+    for eos_pos in eos_positions:
+        if eos_pos > start:
+            docs.append(token_stream[start:eos_pos])
+        start = eos_pos + 1
+    if start < len(token_stream):
+        docs.append(token_stream[start:])
+    return docs
+
+
+def write_litdata_chunks(output_dir, token_stream, config):
+    """Write token stream as litdata chunks, returns index metadata."""
+    os.makedirs(output_dir, exist_ok=True)
+
+    dtype_size = DTYPE().itemsize
+    tokens_per_chunk = CHUNK_BYTES_TARGET // dtype_size
+    tokens_per_chunk = (tokens_per_chunk // BLOCK_SIZE) * BLOCK_SIZE
+
+    chunks_metadata = []
+    pos = 0
+    chunk_idx = 0
+
+    while pos < len(token_stream):
+        remaining = len(token_stream) - pos
+        chunk_tokens = min(tokens_per_chunk, remaining)
+        num_blocks = chunk_tokens // BLOCK_SIZE
+        if num_blocks == 0:
+            break
+        actual_tokens = num_blocks * BLOCK_SIZE
+
+        chunk_data = token_stream[pos:pos + actual_tokens]
+        filename = f"chunk-0-{chunk_idx}.bin"
+        filepath = os.path.join(output_dir, filename)
+
+        # Header: [num_items(uint32)] + [offsets 0..num_blocks(uint32)]
+        header_num_items = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * dtype_size)
+        header = np.concatenate([header_num_items, offsets])
+
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        chunks_metadata.append(meta)
+        pos += actual_tokens
+        chunk_idx += 1
+
+        if chunk_idx % 25 == 0 or pos >= len(token_stream):
+            print(f"    Written chunk {chunk_idx} ({pos:,}/{len(token_stream):,} tokens)")
+
+    # Write index.json
+    index = {
+        "chunks": chunks_metadata,
+        "config": config,
+        "updated_at": str(time.time()),
+    }
+    with open(os.path.join(output_dir, "index.json"), "w") as f:
+        json.dump(index, f, indent=2)
+
+    return chunks_metadata
+
+
+# ==============================================================================
+# MAIN PROCESSING
+# ==============================================================================
+
+def process_litdata(input_dir, output_dir, name):
+    print(f"\n{'='*65}")
+    print(f"  PROCESSING: {name}")
+    print(f"  Input:  {input_dir}")
+    print(f"  Output: {output_dir}")
+    print(f"{'='*65}")
+
+    # 1. Read all tokens
+    t0 = time.time()
+    token_stream = read_all_tokens(input_dir)
+    print(f"  Read {len(token_stream):,} tokens in {time.time()-t0:.1f}s")
+
+    # 2. Split into documents
+    t1 = time.time()
+    doc_tokens = split_documents(token_stream)
+    print(f"  Found {len(doc_tokens):,} documents in {time.time()-t1:.1f}s")
+    del token_stream
+
+    # 3. Decode all documents to text (batch for speed)
+    t2 = time.time()
+    print(f"  Decoding documents back to text...")
+    raw_texts = []
+    BATCH = 5000
+    for i in range(0, len(doc_tokens), BATCH):
+        batch = doc_tokens[i:i+BATCH]
+        for doc in batch:
+            text = tokenizer.decode(doc.tolist(), skip_special_tokens=False)
+            raw_texts.append(text)
+        done = min(i + BATCH, len(doc_tokens))
+        if done % 100000 == 0 or done == len(doc_tokens):
+            print(f"    Decoded {done:,}/{len(doc_tokens):,}")
+    del doc_tokens
+    print(f"  Decoded in {time.time()-t2:.1f}s")
+
+    # 4. Clean each document
+    t3 = time.time()
+    print(f"  Cleaning {len(raw_texts):,} documents...")
+    cleaned_texts = []
+    dropped = 0
+    for i, text in enumerate(raw_texts):
+        result = clean_text(text)
+        if result is not None:
+            cleaned_texts.append(result)
+        else:
+            dropped += 1
+        if (i + 1) % 200000 == 0 or i == len(raw_texts) - 1:
+            print(f"    Processed {i+1:,}/{len(raw_texts):,} | kept={len(cleaned_texts):,} | dropped={dropped:,}")
+    del raw_texts
+    print(f"  Cleaning done in {time.time()-t3:.1f}s")
+    print(f"  Kept {len(cleaned_texts):,} docs | Dropped {dropped:,} ({dropped/(max(dropped+len(cleaned_texts),1))*100:.1f}%)")
+
+    # 5. Re-tokenize cleaned texts
+    t4 = time.time()
+    print(f"  Re-tokenizing {len(cleaned_texts):,} documents...")
+    all_token_ids = []
+    total_new_tokens = 0
+    ENCODE_BATCH = 10000
+    for i in range(0, len(cleaned_texts), ENCODE_BATCH):
+        batch = cleaned_texts[i:i+ENCODE_BATCH]
+        encoded = tokenizer.encode_batch(batch, add_special_tokens=False)
+        for enc in encoded:
+            ids = enc.ids
+            all_token_ids.extend(ids)
+            all_token_ids.append(EOS_TOKEN_ID)
+            total_new_tokens += len(ids) + 1
+        done = min(i + ENCODE_BATCH, len(cleaned_texts))
+        if done % 200000 == 0 or done == len(cleaned_texts):
+            print(f"    Tokenized {done:,}/{len(cleaned_texts):,} ({total_new_tokens:,} tokens)")
+    del cleaned_texts
+    print(f"  Tokenized in {time.time()-t4:.1f}s")
+    print(f"  New total: {total_new_tokens:,} tokens")
+
+    # 6. Convert to numpy and write chunks
+    t5 = time.time()
+    print(f"  Building token stream array...")
+    new_stream = np.array(all_token_ids, dtype=DTYPE)
+    del all_token_ids
+
+    with open(input_dir / "index.json") as f:
+        config = json.load(f)["config"]
+
+    print(f"  Writing litdata chunks...")
+    chunks = write_litdata_chunks(str(output_dir), new_stream, config)
+    print(f"  Written {len(chunks)} chunks in {time.time()-t5:.1f}s")
+
+    total_in = sum(c["dim"] for c in json.load(open(input_dir / "index.json"))["chunks"])
+    total_out = sum(c["dim"] for c in chunks)
+    print(f"\n  SUMMARY for {name}:")
+    print(f"    Input tokens:  {total_in:,}")
+    print(f"    Output tokens: {total_out:,}")
+    print(f"    Difference:    {total_in - total_out:,} ({(total_in-total_out)/total_in*100:.2f}% removed)")
+
+    return total_in, total_out
+
+
+if __name__ == "__main__":
+    t_start = time.time()
+    data_dir = ROOT / "Base" / "data"
+
+    # Process litdata_3b
+    orig_3b, clean_3b = process_litdata(
+        data_dir / "litdata_3b",
+        data_dir / "litdata_3b_clean",
+        "litdata_3b (General Knowledge)",
+    )
+
+    # Process litdata_english
+    orig_en, clean_en = process_litdata(
+        data_dir / "litdata_english",
+        data_dir / "litdata_english_clean",
+        "litdata_english (English Knowledge)",
+    )
+
+    # Final report
+    print(f"\n{'='*65}")
+    print(f"  FINAL REPORT")
+    print(f"{'='*65}")
+    print(f"  litdata_3b:      {orig_3b:>15,} -> {clean_3b:>15,} tokens")
+    print(f"  litdata_english: {orig_en:>15,} -> {clean_en:>15,} tokens")
+    print(f"  ---------------------------------------------------------")
+    total_orig = orig_3b + orig_en
+    total_clean = clean_3b + clean_en
+    print(f"  TOTAL:           {total_orig:>15,} -> {total_clean:>15,} tokens")
+    print(f"  Removed:         {total_orig - total_clean:,} ({(total_orig-total_clean)/total_orig*100:.2f}%)")
+    print(f"\n  Total time: {time.time()-t_start:.0f}s")
+    print(f"\n  Clean data ready at:")
+    print(f"    {data_dir / 'litdata_3b_clean'}")
+    print(f"    {data_dir / 'litdata_english_clean'}")
+    print(f"\n  Update your training configs to point to the _clean directories!")

Base/scripts/smart_cleanup_english.py

@@ -0,0 +1,379 @@
+# -*- coding: utf-8 -*-
+"""
+Smart cleanup of litdata_english_clean based on deep audit results.
+
+Removes truly problematic docs while keeping legitimate educational content.
+Strategy:
+  - REMOVE: docs with non-English script fragments (Cyrillic, CJK, Arabic, Devanagari)
+  - REMOVE: docs under 50 words
+  - REMOVE: very repetitive docs (unique word ratio < 0.20)
+  - REMOVE: docs with heavy residual code (5+ code patterns)
+  - KEEP: educational/historical content that mentions historical terms in context
+    (these are Wikipedia articles about WWII, civil rights, etc. - valuable learning)
+  - STRIP: clickbait phrases, subscribe prompts, cookie/login boilerplate FROM docs
+    (remove the noise parts, keep the content)
+
+Rebuilds clean litdata chunks after filtering.
+"""
+
+import json
+import os
+import re
+import time
+import unicodedata
+from pathlib import Path
+from collections import Counter
+
+import numpy as np
+from tokenizers import Tokenizer
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+BLOCK_SIZE = 1025
+DTYPE = np.int32
+CHUNK_BYTES_TARGET = 64 * 1024 * 1024
+EOS_TOKEN_ID = 0
+
+print("Loading tokenizer...")
+tokenizer = Tokenizer.from_file(
+    str(ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m" / "tokenizer.json")
+)
+
+# ==============================================================================
+# PATTERNS
+# ==============================================================================
+
+# Non-English script detectors
+RE_CJK = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]{3,}')
+RE_ARABIC = re.compile(r'[\u0600-\u06ff]{5,}')
+RE_CYRILLIC = re.compile(r'[\u0400-\u04ff]{5,}')
+RE_DEVANAGARI = re.compile(r'[\u0900-\u097f]{5,}')
+
+# Residual code
+RE_RESIDUAL_CODE = re.compile(r'(function\s*\(|var\s+\w+\s*=|console\.log|document\.get|if\s*\(\s*\w+\s*[!=]==)', re.I)
+
+# Boilerplate to STRIP from docs (not remove doc, just strip these lines)
+RE_COOKIE_LINE = re.compile(r'^.*(?:cookie|cookies)\s+(?:policy|consent|notice|preferences|settings).*$', re.I | re.M)
+RE_SUBSCRIBE_LINE = re.compile(r'^.*(?:subscribe|sign\s*up\s+(?:for|to)\s+(?:our|the)\s+newsletter|unsubscribe|opt[\s-]*out\s+of).*$', re.I | re.M)
+RE_CLICKBAIT_LINE = re.compile(r'^.*(?:you\s+won\'?t\s+believe|click\s+here|read\s+more\s*\.{0,3}$|share\s+this\s+(?:article|post|story)|trending\s+now|sponsored\s+content|advertisement).*$', re.I | re.M)
+RE_SOCIAL_LINE = re.compile(r'^.*(?:follow\s+us\s+on|share\s+on\s+(?:facebook|twitter|linkedin|instagram)|like\s+us\s+on|tweet\s+this).*$', re.I | re.M)
+RE_NAV_LINE = re.compile(r'^.*(?:skip\s+to\s+(?:main\s+)?content|back\s+to\s+top|previous\s+article|next\s+article|related\s+(?:articles|posts)).*$', re.I | re.M)
+RE_LOGIN_LINE = re.compile(r'^.*(?:log\s*in\s+to\s+(?:your|an)\s+account|create\s+(?:a\s+)?(?:free\s+)?account|forgot\s+(?:your\s+)?password|already\s+(?:a\s+)?member).*$', re.I | re.M)
+RE_COMMENT_LINE = re.compile(r'^.*(?:leave\s+a\s+(?:comment|reply)|post\s+a\s+comment|\d+\s+comments?$|logged\s+in\s+as).*$', re.I | re.M)
+
+# Lines that are just copyright
+RE_COPYRIGHT_LINE = re.compile(r'^.*(?:all\s+rights\s+reserved|\(c\)\s*\d{4}|copyright\s+\d{4}).*$', re.I | re.M)
+
+# Number-heavy lines (tables of just numbers)
+RE_NUMBER_TABLE_LINE = re.compile(r'^[\d\s,.\-+/%$]+$', re.M)
+
+# ==============================================================================
+# LITDATA I/O
+# ==============================================================================
+
+def read_all_tokens(litdata_dir):
+    with open(litdata_dir / "index.json") as f:
+        index = json.load(f)
+    chunks = index["chunks"]
+    total_tokens = sum(c["dim"] for c in chunks)
+    print(f"  Reading {len(chunks)} chunks ({total_tokens:,} tokens)...")
+    all_tokens = np.empty(total_tokens, dtype=DTYPE)
+    pos = 0
+    for i, chunk in enumerate(chunks):
+        chunk_path = litdata_dir / chunk["filename"]
+        n_blocks = chunk["chunk_size"]
+        header_ints = 1 + n_blocks + 1
+        header_bytes = header_ints * 4
+        with open(chunk_path, "rb") as f:
+            f.seek(header_bytes)
+            data = np.fromfile(f, dtype=DTYPE, count=chunk["dim"])
+        all_tokens[pos:pos + len(data)] = data
+        pos += len(data)
+    print(f"    Read {len(chunks)} chunks ({pos:,} tokens)")
+    return all_tokens[:pos]
+
+
+def split_documents(token_stream):
+    eos_positions = np.where(token_stream == EOS_TOKEN_ID)[0]
+    docs = []
+    start = 0
+    for eos_pos in eos_positions:
+        if eos_pos > start:
+            docs.append(token_stream[start:eos_pos])
+        start = eos_pos + 1
+    if start < len(token_stream):
+        docs.append(token_stream[start:])
+    return docs
+
+
+def write_litdata_chunks(output_dir, token_stream, config):
+    os.makedirs(output_dir, exist_ok=True)
+    dtype_size = DTYPE().itemsize
+    tokens_per_chunk = CHUNK_BYTES_TARGET // dtype_size
+    tokens_per_chunk = (tokens_per_chunk // BLOCK_SIZE) * BLOCK_SIZE
+    chunks_metadata = []
+    pos = 0
+    chunk_idx = 0
+    while pos < len(token_stream):
+        remaining = len(token_stream) - pos
+        chunk_tokens = min(tokens_per_chunk, remaining)
+        num_blocks = chunk_tokens // BLOCK_SIZE
+        if num_blocks == 0:
+            break
+        actual_tokens = num_blocks * BLOCK_SIZE
+        chunk_data = token_stream[pos:pos + actual_tokens]
+        filename = f"chunk-0-{chunk_idx}.bin"
+        filepath = os.path.join(output_dir, filename)
+        header_num_items = np.array([num_blocks], dtype=np.uint32)
+        offsets = np.arange(num_blocks + 1, dtype=np.uint32) * (BLOCK_SIZE * dtype_size)
+        header = np.concatenate([header_num_items, offsets])
+        with open(filepath, "wb") as f:
+            header.tofile(f)
+            chunk_data.tofile(f)
+        meta = {
+            "chunk_bytes": int(header.nbytes + chunk_data.nbytes),
+            "chunk_size": num_blocks,
+            "dim": int(actual_tokens),
+            "filename": filename,
+        }
+        chunks_metadata.append(meta)
+        pos += actual_tokens
+        chunk_idx += 1
+        print(f"    Written chunk {chunk_idx} ({pos:,}/{len(token_stream):,} tokens)")
+    index = {"chunks": chunks_metadata, "config": config, "updated_at": str(time.time())}
+    with open(os.path.join(output_dir, "index.json"), "w") as f:
+        json.dump(index, f, indent=2)
+    return chunks_metadata
+
+
+# ==============================================================================
+# SMART CLEANUP
+# ==============================================================================
+
+def should_remove(text):
+    """Returns (remove: bool, reason: str or None)."""
+    words = text.split()
+    word_count = len(words)
+
+    # Too short
+    if word_count < 50:
+        return True, f"too short ({word_count} words)"
+
+    # Non-English scripts
+    scripts_found = []
+    if RE_CJK.search(text):
+        scripts_found.append("CJK")
+    if RE_ARABIC.search(text):
+        scripts_found.append("Arabic")
+    if RE_CYRILLIC.search(text):
+        scripts_found.append("Cyrillic")
+    if RE_DEVANAGARI.search(text):
+        scripts_found.append("Devanagari")
+    if scripts_found:
+        return True, f"non-English scripts: {', '.join(scripts_found)}"
+
+    # Very repetitive
+    if word_count > 50:
+        unique_ratio = len(set(w.lower() for w in words)) / word_count
+        if unique_ratio < 0.20:
+            return True, f"very repetitive (unique ratio: {unique_ratio:.3f})"
+
+    # Heavy residual code (5+ code patterns)
+    code_matches = RE_RESIDUAL_CODE.findall(text)
+    if len(code_matches) >= 5:
+        return True, f"residual code ({len(code_matches)} matches)"
+
+    return False, None
+
+
+def strip_boilerplate(text):
+    """Strip boilerplate lines from a document, keeping the educational content."""
+    original_len = len(text)
+
+    # Strip specific boilerplate patterns (line by line removal)
+    for pattern in [
+        RE_COOKIE_LINE, RE_SUBSCRIBE_LINE, RE_CLICKBAIT_LINE,
+        RE_SOCIAL_LINE, RE_NAV_LINE, RE_LOGIN_LINE,
+        RE_COMMENT_LINE, RE_COPYRIGHT_LINE,
+    ]:
+        text = pattern.sub('', text)
+
+    # Strip pure number table lines (> 80% digits/spaces)
+    lines = text.split('\n')
+    clean_lines = []
+    stripped_number_lines = 0
+    for line in lines:
+        stripped = line.strip()
+        if stripped and len(stripped) > 10:
+            digit_count = sum(1 for c in stripped if c.isdigit() or c in ' ,.\t-+/%$')
+            if digit_count / len(stripped) > 0.80:
+                stripped_number_lines += 1
+                continue
+        clean_lines.append(line)
+    text = '\n'.join(clean_lines)
+
+    # Clean up resulting whitespace
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = text.strip()
+
+    chars_stripped = original_len - len(text)
+    return text, chars_stripped, stripped_number_lines
+
+
+# ==============================================================================
+# MAIN
+# ==============================================================================
+
+def main():
+    input_dir = ROOT / "Base" / "data" / "litdata_english_clean"
+    output_dir = ROOT / "Base" / "data" / "litdata_english_clean"
+
+    print(f"\n{'='*75}")
+    print(f"  SMART CLEANUP: litdata_english_clean")
+    print(f"  Input/Output: {input_dir}")
+    print(f"{'='*75}")
+
+    # 1. Read and decode
+    t0 = time.time()
+    token_stream = read_all_tokens(input_dir)
+    doc_tokens = split_documents(token_stream)
+    total_input = len(doc_tokens)
+    print(f"  Found {total_input:,} documents")
+    del token_stream
+
+    print(f"  Decoding ALL {total_input:,} documents...")
+    texts = []
+    t1 = time.time()
+    for i, toks in enumerate(doc_tokens):
+        text = tokenizer.decode(toks.tolist(), skip_special_tokens=False)
+        texts.append(text)
+        if (i + 1) % 20000 == 0 or i == total_input - 1:
+            print(f"    Decoded {i+1:,}/{total_input:,}")
+    del doc_tokens
+    print(f"  Decoded in {time.time()-t1:.1f}s")
+
+    # 2. Filter and strip
+    print(f"\n  Processing {total_input:,} documents...")
+    t2 = time.time()
+
+    kept_texts = []
+    removed_reasons = Counter()
+    total_boilerplate_chars = 0
+    total_number_lines_stripped = 0
+    removed_examples = []
+
+    for i, text in enumerate(texts):
+        # Check if doc should be removed entirely
+        remove, reason = should_remove(text)
+        if remove:
+            removed_reasons[reason.split('(')[0].strip().split(':')[0].strip()] += 1
+            if len(removed_examples) < 20:
+                removed_examples.append((i, reason, text[:300]))
+            continue
+
+        # Strip boilerplate from kept docs
+        cleaned, chars_stripped, num_lines = strip_boilerplate(text)
+        total_boilerplate_chars += chars_stripped
+        total_number_lines_stripped += num_lines
+
+        # Final check: did stripping make it too short?
+        if len(cleaned.split()) < 50:
+            removed_reasons["stripped too short"] += 1
+            continue
+
+        kept_texts.append(cleaned)
+
+        if (i + 1) % 10000 == 0 or i == total_input - 1:
+            print(f"    Processed {i+1:,}/{total_input:,} | kept={len(kept_texts):,} | removed={i+1-len(kept_texts):,}")
+
+    total_removed = total_input - len(kept_texts)
+    print(f"  Processing done in {time.time()-t2:.1f}s")
+    print(f"  Kept: {len(kept_texts):,} | Removed: {total_removed:,} ({total_removed/total_input*100:.2f}%)")
+
+    # 3. Re-tokenize
+    t3 = time.time()
+    print(f"\n  Re-tokenizing {len(kept_texts):,} documents...")
+    all_token_ids = []
+    total_new_tokens = 0
+    for i, text in enumerate(kept_texts):
+        enc = tokenizer.encode(text)
+        ids = enc.ids
+        all_token_ids.extend(ids)
+        all_token_ids.append(EOS_TOKEN_ID)
+        total_new_tokens += len(ids) + 1
+        if (i + 1) % 20000 == 0 or i == len(kept_texts) - 1:
+            print(f"    Tokenized {i+1:,}/{len(kept_texts):,} ({total_new_tokens:,} tokens)")
+    print(f"  Tokenized in {time.time()-t3:.1f}s")
+
+    # 4. Rebuild chunks
+    t4 = time.time()
+    print(f"\n  Rebuilding litdata chunks...")
+    token_array = np.array(all_token_ids, dtype=DTYPE)
+    del all_token_ids
+
+    import gc
+    gc.collect()
+
+    os.makedirs(str(output_dir), exist_ok=True)
+    config = {
+        "block_size": BLOCK_SIZE,
+        "vocab_size": tokenizer.get_vocab_size(),
+    }
+    chunks = write_litdata_chunks(str(output_dir), token_array, config)
+    del token_array
+    print(f"  Written {len(chunks)} chunks in {time.time()-t4:.1f}s")
+
+    # 5. Summary report
+    total_time = time.time() - t0
+
+    report = []
+    report.append(f"\n{'='*75}")
+    report.append(f"  SMART CLEANUP REPORT - litdata_english_clean")
+    report.append(f"{'='*75}")
+    report.append(f"\n  Processing time: {total_time:.0f}s ({total_time/60:.1f} min)")
+    report.append(f"\n  DOCUMENT COUNTS")
+    report.append(f"  {'-'*50}")
+    report.append(f"    Input documents:     {total_input:>10,}")
+    report.append(f"    Output documents:    {len(kept_texts):>10,}")
+    report.append(f"    Removed:             {total_removed:>10,} ({total_removed/total_input*100:.2f}%)")
+
+    report.append(f"\n  TOKEN COUNTS")
+    report.append(f"  {'-'*50}")
+    report.append(f"    Input tokens:        {53446575:>15,}")
+    report.append(f"    Output tokens:       {total_new_tokens:>15,}")
+    report.append(f"    Tokens after align:  {sum(c['dim'] for c in chunks):>15,}")
+
+    report.append(f"\n  REMOVAL REASONS")
+    report.append(f"  {'-'*50}")
+    for reason, count in sorted(removed_reasons.items(), key=lambda x: -x[1]):
+        report.append(f"    {reason:<35} {count:>6,}")
+
+    report.append(f"\n  BOILERPLATE STRIPPED (from kept docs)")
+    report.append(f"  {'-'*50}")
+    report.append(f"    Total boilerplate chars stripped: {total_boilerplate_chars:>10,}")
+    report.append(f"    Number-heavy lines removed:      {total_number_lines_stripped:>10,}")
+
+    report.append(f"\n  REMOVED DOCUMENT EXAMPLES (first 15)")
+    report.append(f"  {'-'*50}")
+    for idx, reason, preview in removed_examples[:15]:
+        preview_clean = preview.replace('\n', ' ')[:200]
+        report.append(f"\n    Doc #{idx} - {reason}")
+        report.append(f"      \"{preview_clean}...\"")
+
+    report.append(f"\n  VERDICT")
+    report.append(f"  {'-'*50}")
+    report.append(f"    Removed non-English fragments, very short docs, repetitive content,")
+    report.append(f"    and heavy code. Stripped boilerplate from all remaining docs.")
+    report.append(f"    Dataset is now optimized for English language learning!")
+    report.append(f"\n{'='*75}")
+
+    full_report = '\n'.join(report)
+    print(full_report)
+
+    report_path = output_dir / "SMART_CLEANUP_REPORT.txt"
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(full_report)
+    print(f"\n  Report saved to: {report_path}")
+
+
+if __name__ == "__main__":
+    main()

Base/scripts/validate_model.py

@@ -0,0 +1,137 @@
+"""
+Post-training validation script.
+Tests the trained model by generating text from prompts.
+
+Usage:
+    python Base/scripts/validate_model.py --checkpoint_dir Base/out/pretrain/custom-100m-10m-test/final
+    python Base/scripts/validate_model.py --checkpoint_dir Base/out/pretrain/custom-100m-3b/final
+"""
+
+import argparse
+from pathlib import Path
+
+import torch
+from litgpt import Tokenizer
+from litgpt.config import Config
+from litgpt.model import GPT
+
+
+def load_model(checkpoint_dir: str):
+    """Load a pretrained model from a litgpt checkpoint directory."""
+    checkpoint_dir = Path(checkpoint_dir)
+
+    if not checkpoint_dir.exists():
+        raise FileNotFoundError(f"Checkpoint directory not found: {checkpoint_dir}")
+
+    config = Config.from_checkpoint(checkpoint_dir)
+    model_path = checkpoint_dir / "lit_model.pth"
+
+    if not model_path.exists():
+        raise FileNotFoundError(f"Model file not found: {model_path}")
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"Using device: {device}")
+
+    with torch.device("meta"):
+        model = GPT(config)
+
+    checkpoint = torch.load(str(model_path), map_location="cpu", weights_only=False)
+
+    # Handle both direct state_dict and wrapped checkpoint formats
+    if "model" in checkpoint:
+        state_dict = checkpoint["model"]
+    else:
+        state_dict = checkpoint
+
+    model.load_state_dict(state_dict, assign=True)
+    model = model.to(device)
+    model.eval()
+
+    return model, config, device
+
+
+def generate_text(model, tokenizer, device, prompt, max_new_tokens=200, temperature=0.8, top_k=50):
+    """Generate text from a prompt."""
+    input_ids = tokenizer.encode(prompt, device=device).unsqueeze(0)
+
+    with torch.no_grad():
+        for _ in range(max_new_tokens):
+            # Crop to block_size if needed
+            idx_cond = input_ids[:, -model.max_seq_length:]
+            logits = model(idx_cond)
+            logits = logits[:, -1, :] / temperature
+
+            # Top-k filtering
+            if top_k is not None:
+                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
+                logits[logits < v[:, [-1]]] = float("-inf")
+
+            probs = torch.softmax(logits, dim=-1)
+            next_id = torch.multinomial(probs, num_samples=1)
+            input_ids = torch.cat([input_ids, next_id], dim=1)
+
+            # Stop on EOS
+            if next_id.item() == tokenizer.eos_id:
+                break
+
+    return tokenizer.decode(input_ids[0])
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Validate a pretrained LitGPT model")
+    parser.add_argument(
+        "--checkpoint_dir",
+        type=str,
+        required=True,
+        help="Path to the checkpoint directory (e.g., Base/out/pretrain/custom-100m-10m-test/final)",
+    )
+    parser.add_argument(
+        "--tokenizer_dir",
+        type=str,
+        default="Base/checkpoints/EleutherAI/pythia-160m",
+        help="Path to the tokenizer directory",
+    )
+    parser.add_argument(
+        "--max_new_tokens",
+        type=int,
+        default=200,
+        help="Maximum tokens to generate",
+    )
+    args = parser.parse_args()
+
+    print(f"Loading model from: {args.checkpoint_dir}")
+    model, config, device = load_model(args.checkpoint_dir)
+
+    print(f"\nModel config: {config.name}")
+    print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")
+    print(f"Block size: {config.block_size}")
+
+    tokenizer = Tokenizer(Path(args.tokenizer_dir))
+
+    test_prompts = [
+        "The future of artificial intelligence",
+        "In a groundbreaking study, researchers found that",
+        "The most important thing about education is",
+        "Once upon a time, in a land far away,",
+    ]
+
+    print(f"\n{'='*60}")
+    print("GENERATION TESTS")
+    print(f"{'='*60}")
+
+    for i, prompt in enumerate(test_prompts, 1):
+        print(f"\n--- Prompt {i}: \"{prompt}\" ---")
+        output = generate_text(
+            model, tokenizer, device, prompt,
+            max_new_tokens=args.max_new_tokens,
+        )
+        print(f"Generated:\n{output}\n")
+
+    print(f"{'='*60}")
+    print("Validation complete!")
+    print("Note: A freshly pretrained model will produce semi-coherent text.")
+    print("Quality improves with more training data and compute.")
+
+
+if __name__ == "__main__":
+    main()

README.md

@@ -1,2 +1,119 @@
-# LUNA
-Model for Desk Assitant
+# LUNA - 100M Parameter LLM from Scratch
+
+Custom ~100M parameter GPT model (Pythia-like architecture) pretrained on 4.5B tokens of clean English text.
+
+## Quick Start (RunPod / Cloud GPU)
+
+### 1. Clone & Install (one command)
+
+```bash
+git clone https://huggingface.co/spaces/ASTERIZER/LUNA /workspace/LUNA && \
+cd /workspace/LUNA && \
+pip install -q -r requirements.txt
+```
+
+### 2. Get Dataset + Train (one command)
+
+**From HuggingFace (recommended):**
+```bash
+bash setup_and_train.sh huggingface ASTERIZER/LUNA-pretrain-data
+```
+
+**From Google Drive:**
+```bash
+bash setup_and_train.sh gdrive YOUR_GDRIVE_FOLDER_ID
+```
+
+**Smoke test (10M tokens only):**
+```bash
+bash setup_and_train.sh huggingface ASTERIZER/LUNA-pretrain-data 10000000
+```
+
+That's it. The script auto-detects your GPU, VRAM, RAM, CPU cores and configures everything for maximum utilization.
+
+---
+
+## How It Works
+
+### Auto vs Manual Config
+
+All hyperparameters live in `train_config.yaml`:
+
+```yaml
+auto_config: true   # auto-detect everything from hardware
+auto_config: false  # use exact values below, no overrides
+```
+
+When `auto_config: true` (default), the trainer:
+- **Probes VRAM** via binary search to find max micro_batch_size (82% safety)
+- **Sets grad_accum** to hit the target global_batch_size
+- **Picks precision** (bf16 on Ampere+, fp16 otherwise)
+- **Scales workers** to half your CPU cores, capped by RAM
+- **Enables torch.compile** if Triton is available (Linux)
+
+When `auto_config: false`, every value in the YAML is used exactly as-is.
+
+### CLI Overrides
+
+Any config value can be overridden from the command line:
+
+```bash
+python train.py --config train_config.yaml --data_path /data/litdata --max_tokens 100000000
+```
+
+Priority: CLI args > train_config.yaml > auto-detection
+
+---
+
+## Dataset
+
+- **4,515,286,950 tokens** (4.5B) in 270 binary chunks
+- Sources: Wikipedia, FineWeb-Edu, OpenWebText (deduplicated, cleaned)
+- Format: LitData binary (int32, block_size=1025, TokensLoader)
+- Tokenizer: EleutherAI/pythia-160m (50,254 vocab)
+
+## Model Architecture
+
+| Parameter | Value |
+|-----------|-------|
+| Layers | 10 |
+| Hidden dim | 768 |
+| Attention heads | 12 |
+| Vocab size | 50,304 (padded) |
+| Context length | 1,024 |
+| Total params | ~109M (70M unique, tied embeddings) |
+| Rotary % | 25% |
+
+## File Structure
+
+```
+LUNA/
+  train.py              # Main training script (config-driven, auto-detects hardware)
+  train_config.yaml     # All hyperparameters (auto_config: true/false)
+  fetch_data.py         # Downloads dataset from HuggingFace / GDrive
+  setup_and_train.sh    # One-command cloud entrypoint
+  benchmark_runpod.py   # Local benchmark + RunPod cost calculator
+  requirements.txt      # Python dependencies
+  Base/
+    checkpoints/EleutherAI/pythia-160m/   # Tokenizer files
+    configs/             # Legacy litgpt YAML configs (reference only)
+    scripts/             # Data preprocessing scripts
+```
+
+## Estimated Training Times (RunPod)
+
+| GPU | $/hr | tok/s | Hours | Cost USD | Cost INR |
+|-----|------|-------|-------|----------|----------|
+| RTX A5000 | $0.16 | ~6,400 | ~196h | ~$31 | ~2,700 |
+| RTX 3090 | $0.22 | ~7,600 | ~165h | ~$36 | ~3,100 |
+| RTX 4090 | $0.34 | ~10,000 | ~125h | ~$42 | ~3,600 |
+| RTX 5090 | $0.69 | ~16,000 | ~78h | ~$54 | ~4,600 |
+| H100 NVL | $2.59 | ~43,000 | ~29h | ~$75 | ~6,400 |
+
+## Resume Training
+
+Training auto-saves `latest.pt` every save_interval steps. If interrupted, just re-run the same command -- it picks up where it left off.
+
+## License
+
+Private / ASTERIZER 2026

benchmark_runpod.py

@@ -0,0 +1,412 @@
+"""
+LUNA 100M - Local Benchmark + RunPod Cost Calculator
+=====================================================
+Uses PyTorch SDPA (Flash Attention) for realistic training throughput.
+Matches the exact LUNA model architecture and training config.
+"""
+
+import os
+import sys
+import time
+import math
+import json
+import gc
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.amp import autocast, GradScaler
+
+# ─── Model Architecture (matches your config exactly) ─────────────────────────
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_seq_len=1024):
+        super().__init__()
+        inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer("inv_freq", inv_freq)
+        t = torch.arange(max_seq_len).float()
+        freqs = torch.einsum("i,j->ij", t, inv_freq)
+        emb = torch.cat([freqs, freqs], dim=-1)
+        self.register_buffer("cos_cached", emb.cos())
+        self.register_buffer("sin_cached", emb.sin())
+
+    def forward(self, seq_len):
+        return self.cos_cached[:seq_len], self.sin_cached[:seq_len]
+
+def rotate_half(x):
+    x1, x2 = x.chunk(2, dim=-1)
+    return torch.cat([-x2, x1], dim=-1)
+
+def apply_rotary(x, cos, sin):
+    cos = cos.unsqueeze(0).unsqueeze(0)
+    sin = sin.unsqueeze(0).unsqueeze(0)
+    return x * cos + rotate_half(x) * sin
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, n_embd, n_head, block_size, rotary_pct=0.25):
+        super().__init__()
+        self.n_head = n_head
+        self.head_dim = n_embd // n_head
+        self.rotary_dim = int(self.head_dim * rotary_pct)
+        self.c_attn = nn.Linear(n_embd, 3 * n_embd, bias=True)
+        self.c_proj = nn.Linear(n_embd, n_embd, bias=True)
+        self.rotary = RotaryEmbedding(self.rotary_dim, block_size)
+
+    def forward(self, x):
+        B, T, C = x.size()
+        qkv = self.c_attn(x).reshape(B, T, 3, self.n_head, self.head_dim).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv.unbind(0)
+
+        cos, sin = self.rotary(T)
+        q_rot = apply_rotary(q[..., :self.rotary_dim], cos, sin)
+        k_rot = apply_rotary(k[..., :self.rotary_dim], cos, sin)
+        q = torch.cat([q_rot, q[..., self.rotary_dim:]], dim=-1)
+        k = torch.cat([k_rot, k[..., self.rotary_dim:]], dim=-1)
+
+        # SDPA = Flash Attention / Memory Efficient Attention
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        y = y.transpose(1, 2).contiguous().view(B, T, C)
+        return self.c_proj(y)
+
+class MLP(nn.Module):
+    def __init__(self, n_embd):
+        super().__init__()
+        self.c_fc = nn.Linear(n_embd, 4 * n_embd, bias=True)
+        self.gelu = nn.GELU()
+        self.c_proj = nn.Linear(4 * n_embd, n_embd, bias=True)
+    def forward(self, x):
+        return self.c_proj(self.gelu(self.c_fc(x)))
+
+class Block(nn.Module):
+    def __init__(self, n_embd, n_head, block_size):
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(n_embd)
+        self.attn = CausalSelfAttention(n_embd, n_head, block_size)
+        self.ln_2 = nn.LayerNorm(n_embd)
+        self.mlp = MLP(n_embd)
+    def forward(self, x):
+        x = x + self.attn(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+class LUNAModel(nn.Module):
+    def __init__(self, vocab_size=50254, block_size=1024, n_layer=10, n_embd=768, n_head=12):
+        super().__init__()
+        self.block_size = block_size
+        self.wte = nn.Embedding(vocab_size, n_embd)
+        self.blocks = nn.ModuleList([Block(n_embd, n_head, block_size) for _ in range(n_layer)])
+        self.ln_f = nn.LayerNorm(n_embd)
+        self.lm_head = nn.Linear(n_embd, vocab_size, bias=False)
+        self.lm_head.weight = self.wte.weight
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            module.weight.data.normal_(mean=0.0, std=0.02)
+            if isinstance(module, nn.Linear) and module.bias is not None:
+                module.bias.data.zero_()
+
+    def forward(self, idx, targets=None):
+        B, T = idx.size()
+        x = self.wte(idx)
+        for block in self.blocks:
+            x = block(x)
+        x = self.ln_f(x)
+        logits = self.lm_head(x)
+        loss = None
+        if targets is not None:
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
+        return logits, loss
+
+# ─── Config ────────────────────────────────────────────────────────────────────
+
+DATASET_TOTAL_TOKENS = 4_515_286_950  # Verified from index.json: 270 chunks, sum of all dims
+BLOCK_SIZE = 1024
+VOCAB_SIZE = 50254
+N_LAYER = 10
+N_EMBD = 768
+N_HEAD = 12
+GLOBAL_BATCH_SIZE = 120
+MAX_SEQ_LENGTH = 1024
+
+WARMUP_STEPS = 3
+BENCHMARK_STEPS = 4
+
+# RunPod GPUs (Community Cloud pricing April 2026)
+RUNPOD_GPUS = [
+    # (name, $/hr, VRAM_GB, bf16_TF_nonsparse, mem_bw_GBs, arch)
+    ("RTX A5000",        0.16, 24,   65,   768, "Ampere"),
+    ("RTX 3090",         0.22, 24,   71,   936, "Ampere"),
+    ("RTX A6000",        0.33, 48,   77,   768, "Ampere"),
+    ("RTX 4090",         0.34, 24,  165,  1008, "Ada"),
+    ("A40",              0.35, 48,   75,   696, "Ampere"),
+    ("L4",               0.44, 24,  121,   300, "Ada"),
+    ("RTX 5090",         0.69, 32,  210,  1792, "Blackwell"),
+    ("L40",              0.69, 48,  181,   864, "Ada"),
+    ("RTX 6000 Ada",     0.74, 48,  181,   960, "Ada"),
+    ("L40S",             0.79, 48,  183,   864, "Ada"),
+    ("A100 PCIe 80GB",   1.19, 80,  312, 2039, "Ampere"),
+    ("A100 SXM 80GB",    1.39, 80,  312, 2039, "Ampere"),
+    ("RTX Pro 6000",     1.69, 96,  260, 1280, "Blackwell"),
+    ("H100 PCIe",        1.99, 80,  756, 2039, "Hopper"),
+    ("H100 NVL",         2.59, 94,  835, 3938, "Hopper"),
+    ("H100 SXM",         2.69, 80,  990, 3352, "Hopper"),
+    ("H200",             3.59,141,  990, 4800, "Hopper"),
+]
+
+USD_TO_INR = 86.0
+
+
+def find_max_micro_batch(model, device, seq_len=1024, start=32):
+    """Binary search for max micro_batch_size, with 0.65 safety factor."""
+    model.train()
+    lo, hi, best = 1, start, 1
+    opt_tmp = torch.optim.AdamW(model.parameters(), lr=1e-4)
+
+    while hi >= lo:
+        mid = (lo + hi) // 2
+        try:
+            torch.cuda.empty_cache()
+            torch.cuda.reset_peak_memory_stats()
+            opt_tmp.zero_grad(set_to_none=True)
+            x = torch.randint(0, VOCAB_SIZE, (mid, seq_len), device=device)
+            t = torch.randint(0, VOCAB_SIZE, (mid, seq_len), device=device)
+            with autocast(device_type='cuda', dtype=torch.bfloat16):
+                _, loss = model(x, t)
+            loss.backward()
+            opt_tmp.step()
+            opt_tmp.zero_grad(set_to_none=True)
+            best = mid
+            lo = mid + 1
+            del x, t, loss
+            torch.cuda.empty_cache()
+        except (torch.cuda.OutOfMemoryError, RuntimeError):
+            try:
+                del x, t, loss
+            except:
+                pass
+            torch.cuda.empty_cache()
+            opt_tmp.zero_grad(set_to_none=True)
+            hi = mid - 1
+
+    safe = max(1, int(best * 0.65))
+    del opt_tmp
+    torch.cuda.empty_cache()
+    gc.collect()
+    return safe
+
+
+def run_benchmark():
+    device = torch.device("cuda")
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.allow_tf32 = True
+
+    print("=" * 72)
+    print("  LUNA 100M - TRAINING BENCHMARK & RUNPOD COST CALCULATOR")
+    print("=" * 72)
+
+    gpu_name = torch.cuda.get_device_name(0)
+    gpu_mem = torch.cuda.get_device_properties(0).total_memory / 1024**3
+    print(f"\n  Local GPU: {gpu_name}")
+    print(f"  VRAM: {gpu_mem:.1f} GB")
+    print(f"  PyTorch: {torch.__version__}, CUDA: {torch.version.cuda}")
+
+    print(f"\n  Creating LUNA-100M (SDPA/Flash Attention)...")
+    model = LUNAModel(VOCAB_SIZE, BLOCK_SIZE, N_LAYER, N_EMBD, N_HEAD).to(device)
+
+    total_params = sum(p.numel() for p in model.parameters())
+    # Tied embeddings: wte(50254*768) = 38,595,072 counted once
+    unique_params = total_params - model.wte.weight.numel()
+    print(f"  Parameters: {total_params:,} total, {unique_params:,} unique")
+
+    print(f"\n  Probing max micro_batch_size...")
+    max_mbs = find_max_micro_batch(model, device, MAX_SEQ_LENGTH, start=40)
+    print(f"  Safe micro_batch_size: {max_mbs}")
+
+    grad_accum = max(1, GLOBAL_BATCH_SIZE // max_mbs)
+    effective_batch = max_mbs * grad_accum
+    tokens_per_step = effective_batch * MAX_SEQ_LENGTH
+    print(f"  grad_accum={grad_accum}, effective_batch={effective_batch}")
+    print(f"  Tokens/step: {tokens_per_step:,}")
+
+    optimizer = torch.optim.AdamW(
+        model.parameters(), lr=6e-4, weight_decay=0.1,
+        betas=(0.9, 0.95), eps=1e-8
+    )
+    scaler = GradScaler()
+
+    total_steps = WARMUP_STEPS + BENCHMARK_STEPS
+    print(f"\n  Running {WARMUP_STEPS} warmup + {BENCHMARK_STEPS} benchmark steps...")
+
+    model.train()
+    step_times = []
+    torch.cuda.synchronize()
+    torch.cuda.reset_peak_memory_stats()
+
+    for step in range(total_steps):
+        t0 = time.perf_counter()
+        optimizer.zero_grad(set_to_none=True)
+        step_loss = 0.0
+
+        for _ in range(grad_accum):
+            x = torch.randint(0, VOCAB_SIZE, (max_mbs, MAX_SEQ_LENGTH), device=device)
+            tgt = torch.randint(0, VOCAB_SIZE, (max_mbs, MAX_SEQ_LENGTH), device=device)
+            with autocast(device_type='cuda', dtype=torch.bfloat16):
+                _, loss = model(x, tgt)
+                loss = loss / grad_accum
+            scaler.scale(loss).backward()
+            step_loss += loss.item()
+            del x, tgt, loss
+
+        scaler.unscale_(optimizer)
+        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+        scaler.step(optimizer)
+        scaler.update()
+        torch.cuda.synchronize()
+        dt = time.perf_counter() - t0
+
+        if step >= WARMUP_STEPS:
+            step_times.append(dt)
+
+        tps = tokens_per_step / dt
+        phase = "WARM" if step < WARMUP_STEPS else "BENCH"
+        print(f"    [{phase}] Step {step:3d} | Loss {step_loss:.4f} | {dt:.2f}s | {tps:,.0f} tok/s")
+
+    # ─── Results ──────────────────────────────────────────────────────────────
+    peak_vram_gb = torch.cuda.max_memory_allocated() / 1024**3
+
+    avg_time = sum(step_times) / len(step_times)
+    med_time = sorted(step_times)[len(step_times) // 2]
+    avg_tps = tokens_per_step / avg_time
+    med_tps = tokens_per_step / med_time
+    peak_tps = tokens_per_step / min(step_times)
+
+    flops_per_token = 6 * unique_params
+    achieved_tf = (avg_tps * flops_per_token) / 1e12
+    LOCAL_TF = 88.0   # RTX 4060 Ti BF16 non-sparse
+    LOCAL_BW = 288.0   # GB/s
+    mfu = achieved_tf / LOCAL_TF
+
+    print("\n" + "=" * 72)
+    print("  LOCAL BENCHMARK RESULTS")
+    print("=" * 72)
+    print(f"  GPU: {gpu_name} ({gpu_mem:.1f} GB)")
+    print(f"  Peak VRAM: {peak_vram_gb:.2f} GB ({peak_vram_gb/gpu_mem*100:.0f}%)")
+    print(f"  Batch: micro={max_mbs}, accum={grad_accum}, global={effective_batch}")
+    print(f"  Step time: avg={avg_time:.3f}s, median={med_time:.3f}s")
+    print(f"  Tokens/sec: avg={avg_tps:,.0f}, median={med_tps:,.0f}, peak={peak_tps:,.0f}")
+    print(f"  TFLOPS: {achieved_tf:.2f}, MFU: {mfu*100:.1f}%")
+
+    n_steps = math.ceil(DATASET_TOTAL_TOKENS / tokens_per_step)
+    local_hrs = (n_steps * avg_time) / 3600
+
+    print(f"  Dataset: 4,515,286,950 tokens | Steps needed: {n_steps:,}")
+    print(f"  Local training: {local_hrs:.1f} hrs ({local_hrs/24:.1f} days)")
+
+    # ─── RunPod Estimates ─────────────────────────────────────────────────────
+    print("\n" + "=" * 72)
+    print("  RUNPOD GPU COMPARISON (Community Cloud, INR/86/USD)")
+    print("=" * 72)
+
+    results = []
+    for name, price, vram, bf16, bw, arch in RUNPOD_GPUS:
+        # Memory estimation for each GPU
+        fixed_gb = (unique_params * (2 + 8 + 2)) / 1024**3 + 0.5
+        avail_gb = vram - fixed_gb
+        act_per_sample = max(0.05, (peak_vram_gb - fixed_gb) / max(max_mbs, 1))
+        est_mbs = max(1, min(128, int(avail_gb / act_per_sample)))
+        est_ga = max(1, GLOBAL_BATCH_SIZE // est_mbs)
+        est_tps_step = est_mbs * est_ga * MAX_SEQ_LENGTH
+
+        # Scaling: 50% compute + 50% bandwidth (validated for small transformers)
+        speedup = 0.50 * (bf16 / LOCAL_TF) + 0.50 * (bw / LOCAL_BW)
+        est_tps = avg_tps * speedup * 0.90  # 0.90 cloud overhead
+
+        est_steps = math.ceil(DATASET_TOTAL_TOKENS / est_tps_step)
+        est_sec = (est_tps_step / est_tps) * est_steps
+        est_hrs = est_sec / 3600
+        cost_usd = est_hrs * price
+        cost_inr = cost_usd * USD_TO_INR
+
+        results.append({
+            "gpu": name, "price": price, "vram": vram, "bf16": bf16,
+            "bw": bw, "arch": arch, "mbs": est_mbs, "ga": est_ga,
+            "tps": round(est_tps), "hours": round(est_hrs, 1),
+            "usd": round(cost_usd, 2), "inr": round(cost_inr),
+        })
+
+    results.sort(key=lambda r: r["inr"])
+
+    print(f"\n  {'#':<3} {'GPU':<18} {'$/hr':>5} {'VRAM':>5} {'tok/s':>10} "
+          f"{'Hours':>7} {'$ USD':>8} {'INR':>10}")
+    print("  " + "─" * 72)
+    for i, r in enumerate(results):
+        s = " *" if i < 3 else ""
+        print(f"  {i+1:<3} {r['gpu']:<18} {r['price']:>5.2f} {r['vram']:>4}G "
+              f"{r['tps']:>10,} {r['hours']:>7.1f} {r['usd']:>8.2f} {r['inr']:>10,}{s}")
+
+    # Top 5 detailed
+    print("\n" + "=" * 72)
+    print("  TOP 5 CHEAPEST - DETAILS")
+    print("=" * 72)
+    for i, r in enumerate(results[:5]):
+        sx = r["tps"] / avg_tps if avg_tps > 0 else 0
+        print(f"\n  #{i+1}: {r['gpu']} ({r['arch']})")
+        print(f"  ├── ${r['price']:.2f}/hr | {r['vram']}GB VRAM | {r['bf16']} TF | {r['bw']} GB/s")
+        print(f"  ├── micro_batch: {r['mbs']}, grad_accum: {r['ga']}")
+        print(f"  ├── {r['tps']:,} tok/s ({sx:.2f}× local)")
+        print(f"  ├── {r['hours']:.1f} hrs ({r['hours']/24:.1f} days)")
+        print(f"  +-- ${r['usd']:.2f} = INR {r['inr']:,}")
+
+    # Local reference
+    print("\n" + "=" * 72)
+    print("  YOUR LOCAL GPU")
+    print("=" * 72)
+    print(f"  RTX 4060 Ti 16GB: {avg_tps:,.0f} tok/s")
+    print(f"  Training: {local_hrs:.1f} hrs ({local_hrs/24:.1f} days)")
+    print(f"  Electricity: ~INR {local_hrs * 0.16 * 8:,.0f} (160W x Rs8/kWh)")
+
+    # Recommendation
+    best = results[0]
+    print("\n" + "=" * 72)
+    print("  * RECOMMENDATION")
+    print("=" * 72)
+    print(f"  Most affordable: {best['gpu']} @ ${best['price']:.2f}/hr")
+    print(f"  Time: {best['hours']:.1f} hrs ({best['hours']/24:.1f} days)")
+    print(f"  Cost: INR {best['inr']:,} (${best['usd']:.2f})")
+    print(f"  Speed: {best['tps']/avg_tps:.1f}× local" if avg_tps > 0 else "")
+
+    fast = [r for r in results if r["hours"] < max(8, local_hrs * 0.15)]
+    if fast:
+        fb = min(fast, key=lambda r: r["inr"])
+        if fb["gpu"] != best["gpu"]:
+            print(f"\n  Fastest affordable: {fb['gpu']} @ ${fb['price']:.2f}/hr")
+            print(f"  Time: {fb['hours']:.1f} hrs | Cost: INR {fb['inr']:,}")
+
+    print("\n" + "=" * 72)
+
+    # Save JSON
+    out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runpod_cost_estimate.json")
+    with open(out, "w") as f:
+        json.dump({
+            "benchmark": {
+                "gpu": gpu_name, "vram_gb": round(gpu_mem, 1),
+                "peak_vram_gb": round(peak_vram_gb, 2),
+                "micro_batch": max_mbs, "grad_accum": grad_accum,
+                "tokens_per_step": tokens_per_step,
+                "avg_tok_per_sec": round(avg_tps),
+                "median_tok_per_sec": round(med_tps),
+                "achieved_tflops": round(achieved_tf, 2),
+                "mfu_pct": round(mfu*100, 1),
+            },
+            "dataset": {"tokens": DATASET_TOTAL_TOKENS, "chunks": 270},
+            "model": {"total_params": total_params, "unique_params": unique_params},
+            "local_hours": round(local_hrs, 1),
+            "runpod": results,
+            "usd_to_inr": USD_TO_INR,
+        }, f, indent=2)
+    print(f"  Saved: {out}")
+    print("=" * 72)
+
+
+if __name__ == "__main__":
+    run_benchmark()

fetch_data.py

@@ -0,0 +1,162 @@
+"""
+LUNA — Dataset Fetcher
+======================
+Downloads the tokenized litdata dataset from either:
+  - HuggingFace Hub (recommended, free, fast)
+  - Google Drive (direct link, requires gdown)
+
+Usage:
+    python fetch_data.py --source huggingface --hf_repo YourName/LUNA-pretrain-data --out_dir /workspace/data
+    python fetch_data.py --source gdrive     --gdrive_id <FILE_OR_FOLDER_ID>       --out_dir /workspace/data
+    python fetch_data.py --source local      --local_path Base/data/litdata_pretrain_final  --out_dir /workspace/data
+
+After running, pass --data_path /workspace/data/litdata_pretrain_final to train.py
+"""
+
+import os
+import sys
+import json
+import shutil
+import argparse
+from pathlib import Path
+
+
+# ─── HuggingFace Download ─────────────────────────────────────────────────────
+
+def download_huggingface(repo_id: str, out_dir: Path, hf_token: str = None):
+    try:
+        from huggingface_hub import snapshot_download, hf_hub_download
+    except ImportError:
+        print("  Installing huggingface_hub...")
+        os.system(f"{sys.executable} -m pip install -q huggingface_hub")
+        from huggingface_hub import snapshot_download
+
+    print(f"  Downloading from HuggingFace: {repo_id}")
+    out_dir.mkdir(parents=True, exist_ok=True)
+    snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",
+        local_dir=str(out_dir),
+        token=hf_token,
+        ignore_patterns=["*.md", ".gitattributes"],
+    )
+    print(f"  Downloaded to: {out_dir}")
+    _verify(out_dir)
+
+
+# ─── Google Drive Download ────────────────────────────────────────────────────
+
+def download_gdrive(gdrive_id: str, out_dir: Path):
+    try:
+        import gdown
+    except ImportError:
+        print("  Installing gdown...")
+        os.system(f"{sys.executable} -m pip install -q gdown")
+        import gdown
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+    # Try as folder first, then single file
+    url = f"https://drive.google.com/drive/folders/{gdrive_id}"
+    print(f"  Attempting GDrive folder download: {gdrive_id}")
+    try:
+        gdown.download_folder(url=url, output=str(out_dir), quiet=False, use_cookies=False)
+    except Exception as e:
+        print(f"  Folder download failed ({e}), trying single file...")
+        url = f"https://drive.google.com/uc?id={gdrive_id}"
+        dest = out_dir / "data.zip"
+        gdown.download(url, str(dest), quiet=False)
+        if dest.suffix == ".zip":
+            print("  Extracting zip...")
+            import zipfile
+            with zipfile.ZipFile(dest) as z:
+                z.extractall(out_dir)
+            dest.unlink()
+    print(f"  Downloaded to: {out_dir}")
+    _verify(out_dir)
+
+
+# ─── Local Copy ───────────────────────────────────────────────────────────────
+
+def copy_local(local_path: str, out_dir: Path):
+    src = Path(local_path)
+    if not src.exists():
+        raise FileNotFoundError(f"Local path not found: {src}")
+    if out_dir.resolve() == src.resolve():
+        print(f"  Source == destination, no copy needed.")
+        _verify(out_dir)
+        return
+    print(f"  Copying {src} → {out_dir}")
+    if out_dir.exists():
+        shutil.rmtree(out_dir)
+    shutil.copytree(src, out_dir)
+    print(f"  Copied to: {out_dir}")
+    _verify(out_dir)
+
+
+# ─── Verify ───────────────────────────────────────────────────────────────────
+
+def _verify(data_dir: Path):
+    idx_path = data_dir / "index.json"
+    if not idx_path.exists():
+        # Search one level deeper
+        found = list(data_dir.glob("**/index.json"))
+        if found:
+            print(f"  Note: index.json found at {found[0]}, not root. Check your --out_dir.")
+        else:
+            print(f"  WARNING: index.json NOT found in {data_dir}")
+        return
+
+    with open(idx_path) as f:
+        idx = json.load(f)
+    chunks = idx.get("chunks", [])
+    total_tokens = sum(c.get("dim", 0) for c in chunks)
+    present = sum(1 for c in chunks if (data_dir / c["filename"]).exists())
+    missing = len(chunks) - present
+
+    print(f"\n  Dataset verified:")
+    print(f"  Chunks declared : {len(chunks)}")
+    print(f"  Chunks on disk  : {present}")
+    print(f"  Missing chunks  : {missing}")
+    print(f"  Total tokens    : {total_tokens:,}")
+    if missing > 0:
+        print(f"  WARNING: {missing} chunk(s) missing — training will error on those blocks!")
+    else:
+        print(f"  All chunks present. Ready to train.")
+
+
+# ─── Args ─────────────────────────────────────────────────────────────────────
+
+def parse_args():
+    p = argparse.ArgumentParser(description="LUNA dataset fetcher")
+    p.add_argument("--source", choices=["huggingface", "gdrive", "local"], required=True)
+    p.add_argument("--out_dir", type=str, default="/workspace/data/litdata_pretrain_final",
+                   help="Where to save the dataset")
+    p.add_argument("--hf_repo", type=str, default="",
+                   help="HuggingFace dataset repo ID (e.g. YourName/LUNA-pretrain-data)")
+    p.add_argument("--hf_token", type=str, default=os.environ.get("HF_TOKEN", ""),
+                   help="HuggingFace token (or set HF_TOKEN env var)")
+    p.add_argument("--gdrive_id", type=str, default="",
+                   help="Google Drive file/folder ID")
+    p.add_argument("--local_path", type=str, default="Base/data/litdata_pretrain_final",
+                   help="Local path to the dataset (for local source)")
+    return p.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    out = Path(args.out_dir)
+
+    if args.source == "huggingface":
+        if not args.hf_repo:
+            print("ERROR: --hf_repo required for HuggingFace source")
+            sys.exit(1)
+        download_huggingface(args.hf_repo, out, hf_token=args.hf_token or None)
+
+    elif args.source == "gdrive":
+        if not args.gdrive_id:
+            print("ERROR: --gdrive_id required for GDrive source")
+            sys.exit(1)
+        download_gdrive(args.gdrive_id, out)
+
+    elif args.source == "local":
+        copy_local(args.local_path, out)

quantisations/convert_to_gguf.py

@@ -0,0 +1,195 @@
+"""
+Convert a litgpt GPT-NeoX checkpoint to GGUF format (F32) for LM Studio.
+
+Usage:
+    python quantisations/convert_to_gguf.py
+    python quantisations/convert_to_gguf.py --checkpoint_dir Base/out/finetune/custom-100m-english-instruct/step-008000
+"""
+
+import argparse
+import json
+import torch
+import numpy as np
+import yaml
+from pathlib import Path
+from gguf import GGUFWriter
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert litgpt checkpoint to GGUF")
+    parser.add_argument("--checkpoint_dir", type=str,
+                        default="Base/out/finetune/custom-100m-english-instruct/final")
+    parser.add_argument("--tokenizer_dir", type=str,
+                        default="Base/checkpoints/EleutherAI/pythia-160m")
+    parser.add_argument("--output", type=str, default=None,
+                        help="Output GGUF path (default: quantisations/LUNA-100M-F32.gguf)")
+    parser.add_argument("--model_name", type=str, default="LUNA-100M")
+    args = parser.parse_args()
+
+    ckpt_dir = Path(args.checkpoint_dir)
+    tok_dir = Path(args.tokenizer_dir)
+
+    if args.output is None:
+        args.output = f"quantisations/{args.model_name}-F32.gguf"
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # ── Load checkpoint ──────────────────────────────────────────────
+    print(f"Loading checkpoint from {ckpt_dir}...")
+    sd = torch.load(ckpt_dir / "lit_model.pth", map_location="cpu", weights_only=False)
+    if "model" in sd:
+        sd = sd["model"]
+
+    # ── Read model config ────────────────────────────────────────────
+    with open(ckpt_dir / "model_config.yaml") as f:
+        cfg = yaml.safe_load(f)
+
+    n_layer = cfg["n_layer"]
+    n_embd = cfg["n_embd"]
+    n_head = cfg["n_head"]
+    n_query_groups = cfg.get("n_query_groups", n_head)
+    head_size = cfg.get("head_size", n_embd // n_head)
+    intermediate_size = cfg["intermediate_size"]
+    block_size = cfg["block_size"]
+    vocab_size = cfg["vocab_size"]
+    padded_vocab_size = cfg["padded_vocab_size"]
+    norm_eps = float(cfg.get("norm_eps", 1e-5))
+    rotary_pct = float(cfg.get("rotary_percentage", 0.25))
+    rope_dim = int(head_size * rotary_pct)
+    rope_base = float(cfg.get("rope_base", 10000))
+    parallel_residual = cfg.get("parallel_residual", True)
+
+    total_params = sum(v.numel() for v in sd.values())
+    print(f"Model: {n_layer}L, {n_embd}d, {n_head}h, vocab={vocab_size} "
+          f"(padded {padded_vocab_size}), {total_params:,} params")
+
+    # ── Create GGUF writer ───────────────────────────────────────────
+    print(f"Writing GGUF to {output_path}...")
+    writer = GGUFWriter(str(output_path), arch="gptneox")
+
+    # General metadata
+    writer.add_name(args.model_name)
+    writer.add_description(f"{args.model_name} by Asterizer — GPT-NeoX {total_params // 1_000_000}M")
+
+    # Architecture parameters
+    writer.add_context_length(block_size)
+    writer.add_embedding_length(n_embd)
+    writer.add_block_count(n_layer)
+    writer.add_feed_forward_length(intermediate_size)
+    writer.add_head_count(n_head)
+    writer.add_head_count_kv(n_query_groups)
+    writer.add_layer_norm_eps(norm_eps)
+    writer.add_rope_dimension_count(rope_dim)
+    writer.add_rope_freq_base(rope_base)
+    writer.add_parallel_residual(parallel_residual)
+    writer.add_file_type(0)  # ALL_F32
+
+    # ── Tokenizer ────────────────────────────────────────────────────
+    print("Processing tokenizer...")
+    with open(tok_dir / "tokenizer.json", "r", encoding="utf-8") as f:
+        tok_data = json.load(f)
+
+    vocab = tok_data["model"]["vocab"]
+    merges = tok_data["model"]["merges"]
+
+    # Identify special tokens
+    special_ids = set()
+    if "added_tokens" in tok_data:
+        for at in tok_data["added_tokens"]:
+            if at.get("special", False):
+                special_ids.add(at["id"])
+
+    # Build sorted token list
+    sorted_tokens = sorted(vocab.items(), key=lambda x: x[1])
+
+    tokens = []
+    token_types = []
+    for token_str, token_id in sorted_tokens:
+        tokens.append(token_str.encode("utf-8"))
+        token_types.append(3 if token_id in special_ids else 1)
+
+    # Pad to padded_vocab_size
+    while len(tokens) < padded_vocab_size:
+        tokens.append(f"[PAD{len(tokens)}]".encode("utf-8"))
+        token_types.append(3)  # CONTROL
+
+    writer.add_tokenizer_model("gpt2")
+    writer.add_token_list(tokens)
+    writer.add_token_types(token_types)
+    writer.add_token_merges([m.encode("utf-8") for m in merges])
+    writer.add_bos_token_id(0)
+    writer.add_eos_token_id(0)
+
+    print(f"  Tokens: {len(sorted_tokens)} real + {padded_vocab_size - len(sorted_tokens)} padding")
+    print(f"  Merges: {len(merges)}")
+
+    # ── Add tensors ──────────────────────────────────────────────────
+    # litgpt GPT-NeoX → GGUF GPT-NeoX tensor name mapping:
+    #   transformer.wte.weight          → token_embd.weight
+    #   transformer.ln_f.{weight,bias}  → output_norm.{weight,bias}
+    #   lm_head.weight                  → output.weight
+    #   transformer.h.{i}.norm_1.*      → blk.{i}.attn_norm.*
+    #   transformer.h.{i}.norm_2.*      → blk.{i}.ffn_norm.*
+    #   transformer.h.{i}.attn.qkv.*    → blk.{i}.attn_qkv.*      (interleaved per-head)
+    #   transformer.h.{i}.attn.proj.*   → blk.{i}.attn_output.*
+    #   transformer.h.{i}.mlp.fc.*      → blk.{i}.ffn_up.*
+    #   transformer.h.{i}.mlp.proj.*    → blk.{i}.ffn_down.*
+
+    print("Adding tensors...")
+    tensor_count = 0
+
+    def add(gguf_name, litgpt_key):
+        nonlocal tensor_count
+        tensor = sd[litgpt_key].float().numpy()
+        writer.add_tensor(gguf_name, tensor)
+        tensor_count += 1
+
+    # Global tensors
+    add("token_embd.weight", "transformer.wte.weight")
+    add("output_norm.weight", "transformer.ln_f.weight")
+    add("output_norm.bias", "transformer.ln_f.bias")
+    add("output.weight", "lm_head.weight")
+
+    # Per-layer tensors
+    for i in range(n_layer):
+        pfx = f"transformer.h.{i}"
+        blk = f"blk.{i}"
+        print(f"  Layer {i + 1}/{n_layer}")
+
+        # Layer norms
+        add(f"{blk}.attn_norm.weight", f"{pfx}.norm_1.weight")
+        add(f"{blk}.attn_norm.bias", f"{pfx}.norm_1.bias")
+        add(f"{blk}.ffn_norm.weight", f"{pfx}.norm_2.weight")
+        add(f"{blk}.ffn_norm.bias", f"{pfx}.norm_2.bias")
+
+        # Attention QKV (litgpt uses same interleaved-per-head layout as HF GPT-NeoX)
+        add(f"{blk}.attn_qkv.weight", f"{pfx}.attn.qkv.weight")
+        add(f"{blk}.attn_qkv.bias", f"{pfx}.attn.qkv.bias")
+
+        # Attention output projection
+        add(f"{blk}.attn_output.weight", f"{pfx}.attn.proj.weight")
+        add(f"{blk}.attn_output.bias", f"{pfx}.attn.proj.bias")
+
+        # FFN
+        add(f"{blk}.ffn_up.weight", f"{pfx}.mlp.fc.weight")
+        add(f"{blk}.ffn_up.bias", f"{pfx}.mlp.fc.bias")
+        add(f"{blk}.ffn_down.weight", f"{pfx}.mlp.proj.weight")
+        add(f"{blk}.ffn_down.bias", f"{pfx}.mlp.proj.bias")
+
+    # ── Write file ───────────────────────────────────────────────────
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file()
+    writer.close()
+
+    size_mb = output_path.stat().st_size / (1024 * 1024)
+    print(f"\nDone!")
+    print(f"  Output: {output_path}")
+    print(f"  Tensors: {tensor_count}")
+    print(f"  File size: {size_mb:.1f} MB")
+    print(f"\nLoad in LM Studio or llama.cpp:")
+    print(f"  llama-cli -m {output_path} -p \"Hello\"")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,6 @@
+torch>=2.3.0
+psutil>=5.9
+pyyaml>=6.0
+huggingface_hub>=0.22
+gdown>=5.1
+tensorboard>=2.16

setup_and_train.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# =============================================================================
+#  LUNA 100M — Cloud Setup & Train Entrypoint
+#  Runs on RunPod, Vast.ai, Lambda Labs, or any Linux GPU pod.
+#
+#  USAGE (after cloning repo):
+#    bash setup_and_train.sh [gdrive|huggingface] [SOURCE_ID] [MAX_TOKENS]
+#
+#  EXAMPLES:
+#    # Full dataset from Google Drive folder:
+#    bash setup_and_train.sh gdrive 1AbCdEfGhIjKlMnOpQrStUvWx
+#
+#    # Full dataset from HuggingFace:
+#    bash setup_and_train.sh huggingface YourName/LUNA-pretrain-data
+#
+#    # Quick smoke test (10M tokens only):
+#    bash setup_and_train.sh gdrive 1AbCdEfGhIjKlMnOpQrStUvWx 10000000
+#
+#    # Dataset already on disk:
+#    bash setup_and_train.sh local /workspace/data/litdata_pretrain_final
+# =============================================================================
+
+set -e
+
+DATA_SOURCE="${1:-local}"
+DATA_ID="${2:-Base/data/litdata_pretrain_final}"
+MAX_TOKENS="${3:-4515286950}"
+DATA_DIR="/workspace/data/litdata_pretrain_final"
+OUT_DIR="/workspace/out/pretrain/luna-100m"
+
+echo "=========================================="
+echo "  LUNA 100M — Cloud Setup"
+echo "  Source  : $DATA_SOURCE"
+echo "  ID/Path : $DATA_ID"
+echo "  Tokens  : $MAX_TOKENS"
+echo "=========================================="
+
+# ── 1. Python packages ────────────────────────────────────────────────────────
+echo ""
+echo "[1/4] Installing dependencies..."
+
+pip install -q --upgrade pip
+pip install -q \
+    torch torchvision \
+    psutil \
+    huggingface_hub \
+    gdown \
+    tensorboard \
+    litgpt 2>/dev/null || true
+
+echo "      Done."
+
+# ── 2. Download dataset ───────────────────────────────────────────────────────
+echo ""
+echo "[2/4] Fetching dataset..."
+
+if [ "$DATA_SOURCE" = "gdrive" ]; then
+    python fetch_data.py --source gdrive --gdrive_id "$DATA_ID" --out_dir "$DATA_DIR"
+elif [ "$DATA_SOURCE" = "huggingface" ]; then
+    HF_TOKEN="${HF_TOKEN:-}"
+    python fetch_data.py --source huggingface --hf_repo "$DATA_ID" --out_dir "$DATA_DIR" --hf_token "$HF_TOKEN"
+elif [ "$DATA_SOURCE" = "local" ]; then
+    python fetch_data.py --source local --local_path "$DATA_ID" --out_dir "$DATA_DIR"
+else
+    echo "Unknown source: $DATA_SOURCE (use: gdrive | huggingface | local)"
+    exit 1
+fi
+
+# ── 3. System + batch size probe ──────────────────────────────────────────────
+echo ""
+echo "[3/4] System probe (auto-detects VRAM, RAM, CPU)..."
+python -c "
+import torch, psutil, os
+props = torch.cuda.get_device_properties(0) if torch.cuda.is_available() else None
+print(f'  GPU  : {props.name if props else \"CPU only\"} ({props.total_memory/1024**3:.1f} GB)' if props else '  GPU: None')
+print(f'  RAM  : {psutil.virtual_memory().total/1024**3:.1f} GB')
+print(f'  CPUs : {os.cpu_count()}')
+"
+
+# ── 4. Train ──────────────────────────────────────────────────────────────────
+echo ""
+echo "[4/4] Starting training (auto_config reads from train_config.yaml)..."
+echo ""
+
+python train.py \
+    --config train_config.yaml \
+    --data_path "$DATA_DIR" \
+    --out_dir "$OUT_DIR" \
+    --max_tokens "$MAX_TOKENS"
+
+echo ""
+echo "=========================================="
+echo "  Training complete!  Output: $OUT_DIR"
+echo "=========================================="

train.py

@@ -0,0 +1,608 @@
+"""
+LUNA 100M — Config-Driven Dynamic Training Script
+==================================================
+Reads train_config.yaml for all hyperparameters.
+
+  auto_config: true  -> hardware probed; batch/lr/workers set automatically
+  auto_config: false -> every value in config used exactly as-is
+
+Usage:
+    python train.py                                         # uses train_config.yaml defaults
+    python train.py --config train_config.yaml             # explicit config path
+    python train.py --data_path /mnt/data/litdata_final    # override data path only
+    python train.py --max_tokens 10000000                  # short smoke-test run
+"""
+
+import os
+import gc
+import sys
+import math
+import time
+import json
+import argparse
+import yaml
+import psutil
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.amp import autocast, GradScaler
+from pathlib import Path
+
+
+# ─── Model ────────────────────────────────────────────────────────────────────
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_seq_len=1024):
+        super().__init__()
+        inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer("inv_freq", inv_freq)
+        t = torch.arange(max_seq_len).float()
+        freqs = torch.einsum("i,j->ij", t, inv_freq)
+        emb = torch.cat([freqs, freqs], dim=-1)
+        self.register_buffer("cos_cached", emb.cos())
+        self.register_buffer("sin_cached", emb.sin())
+
+    def forward(self, seq_len):
+        return self.cos_cached[:seq_len], self.sin_cached[:seq_len]
+
+
+def rotate_half(x):
+    x1, x2 = x.chunk(2, dim=-1)
+    return torch.cat([-x2, x1], dim=-1)
+
+
+def apply_rotary(x, cos, sin):
+    c = cos.unsqueeze(0).unsqueeze(0)
+    s = sin.unsqueeze(0).unsqueeze(0)
+    return x * c + rotate_half(x) * s
+
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, n_embd, n_head, block_size, rotary_pct=0.25):
+        super().__init__()
+        self.n_head = n_head
+        self.head_dim = n_embd // n_head
+        self.rot_dim = int(self.head_dim * rotary_pct)
+        self.c_attn = nn.Linear(n_embd, 3 * n_embd, bias=True)
+        self.c_proj = nn.Linear(n_embd, n_embd, bias=True)
+        self.rotary = RotaryEmbedding(self.rot_dim, block_size)
+
+    def forward(self, x):
+        B, T, C = x.size()
+        qkv = self.c_attn(x).reshape(B, T, 3, self.n_head, self.head_dim).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv.unbind(0)
+        cos, sin = self.rotary(T)
+        q = torch.cat([apply_rotary(q[..., :self.rot_dim], cos, sin), q[..., self.rot_dim:]], dim=-1)
+        k = torch.cat([apply_rotary(k[..., :self.rot_dim], cos, sin), k[..., self.rot_dim:]], dim=-1)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, C))
+
+
+class MLP(nn.Module):
+    def __init__(self, n_embd):
+        super().__init__()
+        self.fc = nn.Linear(n_embd, 4 * n_embd, bias=True)
+        self.gelu = nn.GELU()
+        self.proj = nn.Linear(4 * n_embd, n_embd, bias=True)
+
+    def forward(self, x):
+        return self.proj(self.gelu(self.fc(x)))
+
+
+class Block(nn.Module):
+    def __init__(self, n_embd, n_head, block_size):
+        super().__init__()
+        self.ln1 = nn.LayerNorm(n_embd)
+        self.attn = CausalSelfAttention(n_embd, n_head, block_size)
+        self.ln2 = nn.LayerNorm(n_embd)
+        self.mlp = MLP(n_embd)
+
+    def forward(self, x):
+        x = x + self.attn(self.ln1(x))
+        x = x + self.mlp(self.ln2(x))
+        return x
+
+
+class LUNAModel(nn.Module):
+    def __init__(self, vocab_size, block_size, n_layer, n_embd, n_head):
+        super().__init__()
+        self.wte = nn.Embedding(vocab_size, n_embd)
+        self.blocks = nn.ModuleList([Block(n_embd, n_head, block_size) for _ in range(n_layer)])
+        self.ln_f = nn.LayerNorm(n_embd)
+        self.lm_head = nn.Linear(n_embd, vocab_size, bias=False)
+        self.lm_head.weight = self.wte.weight  # tie
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, (nn.Linear, nn.Embedding)):
+            m.weight.data.normal_(mean=0.0, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                m.bias.data.zero_()
+
+    def forward(self, idx, targets=None):
+        x = self.wte(idx)
+        for block in self.blocks:
+            x = block(x)
+        x = self.ln_f(x)
+        logits = self.lm_head(x)
+        loss = None
+        if targets is not None:
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
+        return logits, loss
+
+    @property
+    def num_params(self):
+        return sum(p.numel() for p in self.parameters()) - self.wte.weight.numel()
+
+
+# ─── Dataset ───────────────────────���──────────────────────────────────────────
+
+class LitDataDataset(torch.utils.data.Dataset):
+    def __init__(self, data_path: str, block_size: int = 1024):
+        import struct, numpy as np
+        self.block_size = block_size
+        self.data_path = Path(data_path)
+        with open(self.data_path / "index.json") as f:
+            idx = json.load(f)
+        self.chunks_meta = idx["chunks"]
+        self._cum_blocks = []
+        total = 0
+        for c in self.chunks_meta:
+            n = c["dim"] // (block_size + 1)
+            total += n
+            self._cum_blocks.append(total)
+        self.total_blocks = total
+        self._chunk_cache = {}
+
+    def _load_chunk(self, chunk_idx: int):
+        if chunk_idx in self._chunk_cache:
+            return self._chunk_cache[chunk_idx]
+        import struct, numpy as np
+        meta = self.chunks_meta[chunk_idx]
+        with open(self.data_path / meta["filename"], "rb") as f:
+            raw = f.read()
+        num_items = struct.unpack_from("<I", raw, 0)[0]
+        header_bytes = (num_items + 2) * 4
+        tokens = torch.from_numpy(np.frombuffer(raw[header_bytes:], dtype=np.int32).copy())
+        if len(self._chunk_cache) >= 4:
+            del self._chunk_cache[next(iter(self._chunk_cache))]
+        self._chunk_cache[chunk_idx] = tokens
+        return tokens
+
+    def __len__(self):
+        return self.total_blocks
+
+    def __getitem__(self, idx):
+        chunk_idx = 0
+        for i, cum in enumerate(self._cum_blocks):
+            if idx < cum:
+                chunk_idx = i
+                break
+        prev = self._cum_blocks[chunk_idx - 1] if chunk_idx > 0 else 0
+        tokens = self._load_chunk(chunk_idx)
+        s = (idx - prev) * (self.block_size + 1)
+        e = s + self.block_size + 1
+        chunk = tokens[s:e]
+        if len(chunk) < self.block_size + 1:
+            pad = torch.zeros(self.block_size + 1, dtype=torch.int32)
+            pad[:len(chunk)] = chunk
+            chunk = pad
+        chunk = chunk.long()
+        return chunk[:self.block_size], chunk[1:self.block_size + 1]
+
+
+# ─── Hardware Detection ────────────────────────────────────────────────────────
+
+def probe_hardware():
+    info = {
+        "cpu_cores": os.cpu_count() or 4,
+        "ram_gb": psutil.virtual_memory().total / 1024**3,
+    }
+    if torch.cuda.is_available():
+        props = torch.cuda.get_device_properties(0)
+        info.update({
+            "device": "cuda",
+            "gpu_name": props.name,
+            "vram_gb": props.total_memory / 1024**3,
+            "sm_major": props.major,
+        })
+        if props.major >= 8:
+            torch.backends.cuda.matmul.allow_tf32 = True
+            torch.backends.cudnn.allow_tf32 = True
+            info["precision"] = "bf16"
+            info["dtype"] = torch.bfloat16
+        else:
+            info["precision"] = "fp16"
+            info["dtype"] = torch.float16
+    else:
+        info.update({
+            "device": "cpu",
+            "gpu_name": "CPU",
+            "vram_gb": 0,
+            "sm_major": 0,
+            "precision": "fp32",
+            "dtype": torch.float32,
+        })
+    return info
+
+
+def probe_max_batch(model, device, dtype, seq_len, vocab_size, max_search=256):
+    """Binary search: max micro_batch that survives fwd+bwd+optim. Safety: ×0.82."""
+    tmp_opt = torch.optim.AdamW(model.parameters(), lr=1e-4)
+    lo, hi, best = 1, max_search, 1
+    while lo <= hi:
+        mid = (lo + hi) // 2
+        try:
+            torch.cuda.empty_cache(); gc.collect()
+            tmp_opt.zero_grad(set_to_none=True)
+            x = torch.randint(0, vocab_size, (mid, seq_len), device=device)
+            t = torch.randint(0, vocab_size, (mid, seq_len), device=device)
+            with autocast(device_type="cuda", dtype=dtype):
+                _, loss = model(x, t)
+            loss.backward()
+            tmp_opt.step()
+            tmp_opt.zero_grad(set_to_none=True)
+            best = mid; lo = mid + 1
+            del x, t, loss; torch.cuda.empty_cache()
+        except (torch.cuda.OutOfMemoryError, RuntimeError):
+            try: del x, t, loss
+            except: pass
+            torch.cuda.empty_cache()
+            tmp_opt.zero_grad(set_to_none=True)
+            hi = mid - 1
+    del tmp_opt; torch.cuda.empty_cache(); gc.collect()
+    return max(1, int(best * 0.82))
+
+
+# ─── LR Schedule ──────────────────────────────────────────────────────────────
+
+def cosine_lr(step, warmup, total, lr_max, lr_min):
+    if step < warmup:
+        return lr_max * (step + 1) / warmup
+    p = (step - warmup) / max(1, total - warmup)
+    return lr_min + 0.5 * (1 + math.cos(math.pi * p)) * (lr_max - lr_min)
+
+
+# ─── Config Loading ───────────────────────────────────────────────────────────
+
+def load_config(config_path: str) -> dict:
+    """Load YAML config and return flat namespace dict."""
+    with open(config_path, encoding="utf-8") as f:
+        raw = yaml.safe_load(f)
+
+    cfg = {
+        # top-level
+        "auto_config":    raw.get("auto_config", True),
+        "data_path":      raw.get("data_path", "Base/data/litdata_pretrain_final"),
+        "out_dir":        raw.get("out_dir", "out/pretrain/luna-100m"),
+        "tokenizer_dir":  raw.get("tokenizer_dir", "Base/checkpoints/EleutherAI/pythia-160m"),
+        # model
+        "vocab_size":     raw["model"]["vocab_size"],
+        "seq_len":        raw["model"]["seq_len"],
+        "n_layer":        raw["model"]["n_layer"],
+        "n_embd":         raw["model"]["n_embd"],
+        "n_head":         raw["model"]["n_head"],
+        # train
+        "max_tokens":     raw["train"]["max_tokens"],
+        "lr_warmup_steps":raw["train"]["lr_warmup_steps"],
+        "save_interval":  raw["train"]["save_interval"],
+        "log_interval":   raw["train"]["log_interval"],
+        "max_norm":       raw["train"]["max_norm"],
+        # optimizer
+        "lr":             raw["optimizer"]["lr"],
+        "min_lr":         raw["optimizer"]["min_lr"],
+        "weight_decay":   raw["optimizer"]["weight_decay"],
+        "betas":          tuple(raw["optimizer"]["betas"]),
+        "eps":            raw["optimizer"]["eps"],
+        # batch
+        "global_batch":   raw["batch"]["global_batch"],
+        "micro_batch":    raw["batch"]["micro_batch"],
+        "grad_accum":     raw["batch"]["grad_accum"],
+        # dataloader
+        "num_workers":    raw["dataloader"]["num_workers"],
+        "pin_memory":     raw["dataloader"]["pin_memory"],
+        # hardware
+        "precision":      raw["hardware"]["precision"],
+        "compile":        raw["hardware"]["compile"],
+    }
+    return cfg
+
+
+def apply_cli_overrides(cfg: dict, cli_args: argparse.Namespace) -> dict:
+    """CLI args override config values (only if explicitly provided)."""
+    for key, val in vars(cli_args).items():
+        if key == "config":
+            continue
+        if val is not None:  # argparse default=None means "not provided"
+            cfg[key] = val
+    return cfg
+
+
+def resolve_auto(cfg: dict, hw: dict) -> dict:
+    """
+    When auto_config=True: override batch, workers, lr-warmup, pin_memory,
+    precision from real hardware. Never touches model arch or max_tokens.
+    Returns updated cfg plus injected hw info.
+    """
+    if not cfg["auto_config"]:
+        print("  [CONFIG] auto_config=false -- using manual values as-is")
+        cfg.update({"_hw": hw})
+        return cfg
+
+    print("  [CONFIG] auto_config=true -- tuning settings to this hardware")
+
+    # Precision
+    cfg["precision"] = hw["precision"]
+    cfg["_dtype"] = hw["dtype"]
+
+    # Workers
+    auto_workers = hw["cpu_cores"] // 2
+    # Cap by RAM: each worker caches up to 4 chunks × ~67MB
+    max_by_ram = max(0, int(hw["ram_gb"] * 0.25 * 1024 / 268))
+    cfg["num_workers"] = min(auto_workers, max_by_ram, hw["cpu_cores"])
+    if cfg["num_workers"] == -1:
+        cfg["num_workers"] = 0
+
+    # Pin memory
+    cfg["pin_memory"] = hw["ram_gb"] > 16 and hw["device"] == "cuda"
+
+    # LR warmup: 5% of total steps (will be computed again in train())
+    cfg["_auto_warmup"] = True   # flag: recompute once total_steps is known
+
+    # LR scaling: sqrt(global_batch / 120) relative to base lr
+    base_global = 120
+    cfg["lr"] = cfg["lr"] * math.sqrt(cfg["global_batch"] / base_global)
+    cfg["min_lr"] = cfg["min_lr"] * math.sqrt(cfg["global_batch"] / base_global)
+
+    cfg["_hw"] = hw
+    return cfg
+
+
+# ─── Training ─────────────────────────────────────────────────────────────────
+
+SEP = "=" * 72
+
+def train(cfg: dict):
+    hw = cfg["_hw"]
+    device = torch.device(hw["device"])
+
+    # Pick precision dtype
+    if cfg["auto_config"]:
+        dtype = hw.get("dtype", torch.float32)
+    else:
+        dtype = {"bf16": torch.bfloat16, "fp16": torch.float16,
+                 "fp32": torch.float32}.get(cfg["precision"], torch.float32)
+
+    print(SEP)
+    print("  LUNA 100M - Training")
+    print(SEP)
+    mode = "AUTO" if cfg["auto_config"] else "MANUAL"
+    print(f"  Config mode  : {mode}")
+    print(f"  GPU          : {hw['gpu_name']}  ({hw['vram_gb']:.1f} GB)")
+    print(f"  RAM          : {hw['ram_gb']:.1f} GB   CPU: {hw['cpu_cores']} cores")
+    print(f"  Precision    : {cfg['precision']}   dtype={dtype}")
+    print(f"  Workers      : {cfg['num_workers']}   pin_memory={cfg['pin_memory']}")
+
+    # ── Model ──────────────────────────────────────────���──────────────────────
+    print(f"\n  Building LUNA-100M...")
+    model = LUNAModel(
+        vocab_size=cfg["vocab_size"],
+        block_size=cfg["seq_len"],
+        n_layer=cfg["n_layer"],
+        n_embd=cfg["n_embd"],
+        n_head=cfg["n_head"],
+    ).to(device)
+
+    if cfg["compile"] and device.type == "cuda":
+        try:
+            import triton  # noqa
+            model = torch.compile(model, mode="reduce-overhead")
+            print("  torch.compile: enabled (reduce-overhead)")
+        except (ImportError, Exception):
+            print("  torch.compile: skipped (Triton unavailable - Linux/cloud only)")
+
+    print(f"  Parameters: {model.num_params:,} (unique)")
+
+    # ── Batch sizing ──────────────────────────────────────────────────────────
+    if cfg["auto_config"] and device.type == "cuda":
+        print(f"\n  Probing max micro_batch_size (VRAM search)...")
+        probe_m = LUNAModel(
+            vocab_size=cfg["vocab_size"], block_size=cfg["seq_len"],
+            n_layer=cfg["n_layer"], n_embd=cfg["n_embd"], n_head=cfg["n_head"],
+        ).to(device)
+        max_mbs = probe_max_batch(
+            probe_m, device, dtype, cfg["seq_len"], cfg["vocab_size"]
+        )
+        del probe_m; torch.cuda.empty_cache(); gc.collect()
+        # grad_accum to hit global_batch
+        grad_accum = max(1, math.ceil(cfg["global_batch"] / max_mbs))
+        effective_batch = max_mbs * grad_accum
+        print(f"  AUTO -> micro_batch={max_mbs}, grad_accum={grad_accum}, "
+              f"effective_batch={effective_batch}")
+    else:
+        max_mbs = cfg["micro_batch"]
+        grad_accum = cfg["grad_accum"]
+        effective_batch = max_mbs * grad_accum
+        print(f"\n  MANUAL -> micro_batch={max_mbs}, grad_accum={grad_accum}, "
+              f"effective_batch={effective_batch}")
+
+    tokens_per_step = effective_batch * cfg["seq_len"]
+    print(f"  Tokens/step : {tokens_per_step:,}")
+
+    # ── Dataset ───────────────────────────────────────────────────────────────
+    print(f"\n  Dataset: {cfg['data_path']}")
+    dataset = LitDataDataset(cfg["data_path"], block_size=cfg["seq_len"])
+    print(f"  Blocks  : {len(dataset):,}   ({len(dataset) * cfg['seq_len']:,} tokens)")
+
+    loader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=max_mbs,
+        shuffle=True,
+        num_workers=cfg["num_workers"],
+        pin_memory=cfg["pin_memory"],
+        drop_last=True,
+        prefetch_factor=4 if cfg["num_workers"] > 0 else None,
+        persistent_workers=cfg["num_workers"] > 0,
+    )
+
+    # ── Optimiser ─────────────────────────────────────────────────────────────
+    fused_ok = device.type == "cuda" and hasattr(torch.optim, "AdamW")
+    try:
+        optimizer = torch.optim.AdamW(
+            model.parameters(),
+            lr=cfg["lr"], weight_decay=cfg["weight_decay"],
+            betas=cfg["betas"], eps=cfg["eps"],
+            fused=True,
+        )
+    except TypeError:
+        optimizer = torch.optim.AdamW(
+            model.parameters(),
+            lr=cfg["lr"], weight_decay=cfg["weight_decay"],
+            betas=cfg["betas"], eps=cfg["eps"],
+        )
+
+    use_scaler = dtype == torch.float16
+    scaler = GradScaler(enabled=use_scaler)
+
+    # ── Schedule ──────────────────────────────────────────────────────────────
+    total_steps = max(1, cfg["max_tokens"] // tokens_per_step)
+    if cfg["auto_config"] and cfg.get("_auto_warmup"):
+        warmup_steps = max(50, min(500, total_steps // 20))
+    else:
+        warmup_steps = min(cfg["lr_warmup_steps"], total_steps)
+
+    out_dir = Path(cfg["out_dir"])
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"\n  max_tokens   : {cfg['max_tokens']:,}")
+    print(f"  total_steps  : {total_steps:,}")
+    print(f"  warmup_steps : {warmup_steps}")
+    print(f"  lr           : {cfg['lr']:.2e} -> {cfg['min_lr']:.2e}")
+    print(f"  save every   : {cfg['save_interval']} steps")
+    print(f"  out_dir      : {out_dir}")
+    print(SEP)
+
+    # ── Resume ────────────────────────────────────────────────────────────────
+    start_step = 0
+    ckpt_path = out_dir / "latest.pt"
+    if ckpt_path.exists():
+        print(f"\n  Resuming from {ckpt_path}...")
+        ckpt = torch.load(ckpt_path, map_location=device, weights_only=True)
+        model.load_state_dict(ckpt["model"])
+        optimizer.load_state_dict(ckpt["optimizer"])
+        start_step = ckpt["step"]
+        print(f"  Resumed at step {start_step}")
+
+    # ── Loop ──────────────────────────────────────────────────────────────────
+    model.train()
+    data_iter = iter(loader)
+
+    def get_batch():
+        nonlocal data_iter
+        try:
+            return next(data_iter)
+        except StopIteration:
+            data_iter = iter(loader)
+            return next(data_iter)
+
+    run_t0 = time.perf_counter()
+    tokens_seen = start_step * tokens_per_step
+    step = start_step
+
+    print(f"\n  Starting training (step {start_step} -> {total_steps})...")
+
+    while step < total_steps:
+        t0 = time.perf_counter()
+        lr_now = cosine_lr(step, warmup_steps, total_steps, cfg["lr"], cfg["min_lr"])
+        for pg in optimizer.param_groups:
+            pg["lr"] = lr_now
+
+        optimizer.zero_grad(set_to_none=True)
+        total_loss = 0.0
+
+        for _ in range(grad_accum):
+            x, t = get_batch()
+            x = x.to(device, non_blocking=True)
+            t = t.to(device, non_blocking=True)
+            with autocast(device_type=device.type, dtype=dtype, enabled=(device.type == "cuda")):
+                _, loss = model(x, t)
+                loss = loss / grad_accum
+            scaler.scale(loss).backward()
+            total_loss += loss.item()
+
+        scaler.unscale_(optimizer)
+        torch.nn.utils.clip_grad_norm_(model.parameters(), cfg["max_norm"])
+        scaler.step(optimizer)
+        scaler.update()
+
+        if device.type == "cuda":
+            torch.cuda.synchronize()
+
+        dt = time.perf_counter() - t0
+        step += 1
+        tokens_seen += tokens_per_step
+
+        if step % cfg["log_interval"] == 0 or step <= 2:
+            tps = tokens_per_step / dt
+            steps_left = total_steps - step
+            eta_h = steps_left * dt / 3600
+            vram = torch.cuda.memory_allocated() / 1024**3 if device.type == "cuda" else 0
+            print(f"  step {step:6d}/{total_steps} | loss {total_loss:.4f} | "
+                  f"lr {lr_now:.2e} | {tps:,.0f} tok/s | VRAM {vram:.1f}GB | ETA {eta_h:.1f}h")
+
+        if step % cfg["save_interval"] == 0 or step == total_steps:
+            raw = model._orig_mod if hasattr(model, "_orig_mod") else model
+            step_dir = out_dir / f"step-{step:08d}"
+            step_dir.mkdir(parents=True, exist_ok=True)
+            torch.save(raw.state_dict(), step_dir / "lit_model.pth")
+            torch.save({"step": step, "model": raw.state_dict(),
+                        "optimizer": optimizer.state_dict(),
+                        "tokens_seen": tokens_seen},
+                       out_dir / "latest.pt")
+            print(f"  Saved -> {step_dir}")
+
+    # ── Final ─────────────────────────────────────────────────────────────────
+    final_dir = out_dir / "final"
+    final_dir.mkdir(parents=True, exist_ok=True)
+    raw = model._orig_mod if hasattr(model, "_orig_mod") else model
+    torch.save(raw.state_dict(), final_dir / "lit_model.pth")
+
+    import shutil
+    tok_src = Path(cfg["tokenizer_dir"])
+    if tok_src.exists():
+        shutil.copytree(tok_src, final_dir / "tokenizer", dirs_exist_ok=True)
+
+    total_h = (time.perf_counter() - run_t0) / 3600
+    print(SEP)
+    print(f"  Done! {total_h:.2f} h -> {final_dir}")
+    print(SEP)
+
+
+# ─── Entry point ──────────────────────────────────────────────────────────────
+
+def parse_args():
+    p = argparse.ArgumentParser(description="LUNA 100M Trainer")
+    p.add_argument("--config", type=str, default="train_config.yaml",
+                   help="Path to train_config.yaml")
+    # CLI overrides (all optional - omit to use config value)
+    p.add_argument("--data_path",   type=str,   default=None)
+    p.add_argument("--out_dir",     type=str,   default=None)
+    p.add_argument("--max_tokens",  type=int,   default=None)
+    p.add_argument("--micro_batch", type=int,   default=None)
+    p.add_argument("--global_batch",type=int,   default=None)
+    p.add_argument("--lr",          type=float, default=None)
+    p.add_argument("--num_workers", type=int,   default=None)
+    p.add_argument("--save_interval",type=int,  default=None)
+    p.add_argument("--log_interval",type=int,   default=None)
+    p.add_argument("--auto_config", type=lambda x: x.lower() in ("1","true","yes"),
+                   default=None, help="Override auto_config (true/false)")
+    return p.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    cfg = load_config(args.config)
+    cfg = apply_cli_overrides(cfg, args)
+    hw = probe_hardware()
+    cfg = resolve_auto(cfg, hw)
+    train(cfg)

train_config.yaml

@@ -0,0 +1,67 @@
+# ─────────────────────────────────────────────────────────────────────────────
+#  LUNA 100M — Training Configuration
+#  Single source of truth for all hyperparameters.
+#
+#  auto_config: true   → All batch/LR/worker settings are auto-detected from
+#                        available VRAM / RAM / CPU at runtime. Your values below
+#                        are used as FALLBACKS only if detection fails.
+#
+#  auto_config: false  → Every value below is used as-is. Nothing is overridden.
+#                        Use this when you've already benchmarked and want
+#                        repeatable, fixed runs.
+# ─────────────────────────────────────────────────────────────────────────────
+
+auto_config: true   # ← flip to false to lock everything below
+
+# ── Data ──────────────────────────────────────────────────────────────────────
+data_path:     "Base/data/litdata_pretrain_final"   # local default; overridden by --data_path
+out_dir:       "out/pretrain/luna-100m"
+tokenizer_dir: "Base/checkpoints/EleutherAI/pythia-160m"
+
+# ── Model (fixed for LUNA-100M — do not change) ───────────────────────────────
+model:
+  vocab_size: 50304      # ceil(50277/128)*128 — pythia tokenizer with EOS padding
+  seq_len:    1024
+  n_layer:    10
+  n_embd:     768
+  n_head:     12
+
+# ── Training budget ───────────────────────────────────────────────────────────
+train:
+  max_tokens:       4515286950   # full dataset (verified from index.json, 270 chunks)
+  lr_warmup_steps:  500          # [AUTO] scaled to 5% of total_steps if auto_config
+  save_interval:    1000         # save checkpoint every N optimizer steps
+  log_interval:     10           # print log every N steps
+  max_norm:         1.0          # gradient clip norm
+
+# ── Optimiser ─────────────────────────────────────────────────────────────────
+optimizer:
+  lr:           0.0006    # 6e-4  [AUTO] scaled by sqrt(global_batch/120) if auto_config
+  min_lr:       0.00006   # 6e-5
+  weight_decay: 0.1
+  betas:        [0.9, 0.95]
+  eps:          1.0e-8
+
+# ── Batch sizing ──────────────────────────────────────────────────────────────
+# When auto_config: true  → micro_batch and grad_accum are ignored; the script
+#                           probes VRAM and fills it to ~82% saturation, then
+#                           computes grad_accum to hit global_batch.
+# When auto_config: false → micro_batch × grad_accum must equal global_batch.
+batch:
+  global_batch: 120     # target total samples per optimizer step
+  micro_batch:  12      # [MANUAL] samples per GPU forward pass (ignored when auto)
+  grad_accum:   10      # [MANUAL] accumulation steps      (ignored when auto)
+
+# ── DataLoader ────────────────────────────────────────────────────────────────
+# When auto_config: true  → num_workers auto = cpu_cores // 2, capped by RAM
+# When auto_config: false → num_workers used as-is
+dataloader:
+  num_workers:  -1      # -1 = auto; set to 0 to disable multiprocessing
+  pin_memory:   true    # [AUTO] disabled if RAM < 16GB
+
+# ── Hardware / precision ──────────────────────────────────────────────────────
+# When auto_config: true  → precision detected from GPU compute capability
+# When auto_config: false → use the value below
+hardware:
+  precision:    "bf16"  # bf16 | fp16 | fp32
+  compile:      true    # torch.compile (requires Triton — Linux/cloud only)