train_ram_18m.py

#!/usr/bin/env python3
"""ram-18m: 18.3M-param LLaMA-style language model trained from scratch.

Architecture:
  d_model=384, n_heads=6, n_kv_heads=2 (GQA), n_layers=7
  SwiGLU FFN (4x), RoPE, RMSNorm, vocab 8192, tied embed/head, ctx 512
  Total: 18,290,304 learnable parameters

Default training:
  ~2B tokens (FineWeb-Edu L3), AdamW 2e-4, cosine + warmup
  batch 32 (effective), seq 512, ~12,207 steps

Usage:
  python3 train_ram_18m.py --stage prepare   # download + tokenize data
  python3 train_ram_18m.py --stage train     # train the model
  python3 train_ram_18m.py --stage all
  python3 train_ram_18m.py --stage eval --ckpt path/to/ckpt.pt

Requirements:
  pip install torch transformers datasets numpy tokenizers
"""
import os, sys, math, json, time, argparse, glob, random
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import AdamW

# ============================================================================
# Architecture
# ============================================================================
VOCAB = 8192
D_MODEL = 384
N_HEADS = 6
N_KV_HEADS = 2
N_LAYERS = 7
HEAD_DIM = D_MODEL // N_HEADS  # 64
KV_DIM = N_KV_HEADS * HEAD_DIM  # 128
FFN_DIM = D_MODEL * 4  # 1536
SEQ_LEN = 512
ROPE_THETA = 10000.0

# Verified param count: 18,290,304


class RMSNorm(nn.Module):
    def __init__(self, dim, eps=1e-6):
        super().__init__()
        self.eps = eps
        self.weight = nn.Parameter(torch.ones(dim))

    def forward(self, x):
        norm = x.float().pow(2).mean(-1, keepdim=True).add(self.eps).rsqrt()
        return (x.float() * norm).type_as(x) * self.weight


class RoPE(nn.Module):
    def __init__(self, head_dim, theta=ROPE_THETA):
        super().__init__()
        freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
        self.register_buffer("freqs", freqs, persistent=False)

    def forward(self, x, pos):
        freqs = self.freqs
        angles = pos[:, None].float() * freqs[None, :]
        cos = angles.cos()[None, None, :, :]
        sin = angles.sin()[None, None, :, :]
        x1 = x[..., 0::2]
        x2 = x[..., 1::2]
        out1 = x1 * cos - x2 * sin
        out2 = x1 * sin + x2 * cos
        return torch.stack([out1, out2], dim=-1).flatten(-2)


class GQAAttention(nn.Module):
    def __init__(self):
        super().__init__()
        self.q_proj = nn.Linear(D_MODEL, N_HEADS * HEAD_DIM, bias=False)
        self.k_proj = nn.Linear(D_MODEL, N_KV_HEADS * HEAD_DIM, bias=False)
        self.v_proj = nn.Linear(D_MODEL, N_KV_HEADS * HEAD_DIM, bias=False)
        self.o_proj = nn.Linear(N_HEADS * HEAD_DIM, D_MODEL, bias=False)
        self.rope = RoPE(HEAD_DIM)

    def forward(self, x, mask=None):
        B, T, _ = x.shape
        q = self.q_proj(x).view(B, T, N_HEADS, HEAD_DIM).transpose(1, 2)
        k = self.k_proj(x).view(B, T, N_KV_HEADS, HEAD_DIM).transpose(1, 2)
        v = self.v_proj(x).view(B, T, N_KV_HEADS, HEAD_DIM).transpose(1, 2)
        pos = torch.arange(T, device=x.device)
        q = self.rope(q, pos)
        k = self.rope(k, pos)
        rep = N_HEADS // N_KV_HEADS
        k = k.repeat_interleave(rep, dim=1)
        v = v.repeat_interleave(rep, dim=1)
        scale = HEAD_DIM ** -0.5
        attn = (q @ k.transpose(-2, -1)) * scale
        if mask is not None:
            attn = attn.masked_fill(mask[:, None, None, :] == 0, float("-inf"))
        attn = F.softmax(attn, dim=-1)
        out = attn @ v
        out = out.transpose(1, 2).contiguous().view(B, T, N_HEADS * HEAD_DIM)
        return self.o_proj(out)


class SwiGLU(nn.Module):
    def __init__(self):
        super().__init__()
        self.gate = nn.Linear(D_MODEL, FFN_DIM, bias=False)
        self.up = nn.Linear(D_MODEL, FFN_DIM, bias=False)
        self.down = nn.Linear(FFN_DIM, D_MODEL, bias=False)

    def forward(self, x):
        return self.down(F.silu(self.gate(x)) * self.up(x))


class TransformerBlock(nn.Module):
    def __init__(self):
        super().__init__()
        self.attn_norm = RMSNorm(D_MODEL)
        self.attn = GQAAttention()
        self.ffn_norm = RMSNorm(D_MODEL)
        self.ffn = SwiGLU()

    def forward(self, x, mask=None):
        x = x + self.attn(self.attn_norm(x), mask)
        x = x + self.ffn(self.ffn_norm(x))
        return x


class RAM18M(nn.Module):
    def __init__(self):
        super().__init__()
        self.tok_emb = nn.Embedding(VOCAB, D_MODEL)
        self.layers = nn.ModuleList([TransformerBlock() for _ in range(N_LAYERS)])
        self.norm = RMSNorm(D_MODEL)
        self.head = nn.Linear(D_MODEL, VOCAB, bias=False)
        self.head.weight = self.tok_emb.weight
        self.apply(self._init_weights)

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)
            if module.bias is not None:
                nn.init.zeros_(module.bias)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)

    def forward(self, input_ids, targets=None):
        B, T = input_ids.shape
        h = self.tok_emb(input_ids)
        mask = torch.tril(torch.ones(T, T, device=input_ids.device))
        for layer in self.layers:
            h = layer(h, mask)
        h = self.norm(h)
        logits = self.head(h)
        loss = None
        if targets is not None:
            loss = F.cross_entropy(logits.view(-1, VOCAB), targets.view(-1))
        return logits, loss

    def count_params(self):
        return sum(p.numel() for p in self.parameters() if p.requires_grad)


def get_lr(step, max_steps, warmup, base_lr, min_lr):
    if step < warmup:
        return base_lr * (step + 1) / warmup
    if step >= max_steps:
        return min_lr
    progress = (step - warmup) / (max_steps - warmup)
    return min_lr + 0.5 * (base_lr - min_lr) * (1 + math.cos(math.pi * progress))


# ============================================================================
# Data
# ============================================================================
DATASET = "HuggingFaceFW/fineweb-edu"
DATASET_CONFIG = "sample-100BT"
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
TOK_DIR = os.path.join(SCRIPT_DIR, "tokens")
CKPT_DIR = SCRIPT_DIR


def prepare_data(target_tokens=2_000_000_000):
    """Download and tokenize FineWeb-Edu. Saves .npy files of token ids."""
    from datasets import load_dataset
    from tokenizers import Tokenizer
    from tokenizers.models import BPE
    from tokenizers.pre_tokenizers import Whitespace
    from tokenizers.trainers import BpeTrainer

    os.makedirs(TOK_DIR, exist_ok=True)

    print("Training BPE tokenizer (vocab 8192)...")
    ds = load_dataset(DATASET, DATASET_CONFIG, split="train", streaming=True)
    texts = []
    for i, row in enumerate(ds):
        texts.append(row["text"])
        if i >= 200000:
            break

    tokenizer = Tokenizer(BPE(unk_token="<unk>"))
    tokenizer.pre_tokenizer = Whitespace()
    trainer = BpeTrainer(vocab_size=VOCAB, special_tokens=["<unk>", "<pad>", "<bos>", "<eos>"])
    tokenizer.train_from_iterator(texts, trainer)
    tok_path = os.path.join(TOK_DIR, "tokenizer.json")
    tokenizer.save(tok_path)
    print(f"Tokenizer saved to {tok_path}")

    print(f"Tokenizing up to {target_tokens:,} tokens...")
    ds = load_dataset(DATASET, DATASET_CONFIG, split="train", streaming=True)
    all_tokens = []
    n_docs = 0
    for row in ds:
        ids = tokenizer.encode(row["text"])
        if len(ids) < 10:
            continue
        all_tokens.extend(ids)
        n_docs += 1
        if len(all_tokens) >= target_tokens:
            break
        if n_docs % 100000 == 0:
            print(f"  {n_docs} docs, {len(all_tokens):,} tokens")

    print(f"Total: {n_docs} docs, {len(all_tokens):,} tokens")
    arr = np.array(all_tokens, dtype=np.int32)
    part_size = 100_000_000
    for i in range(0, len(arr), part_size):
        part = arr[i:i+part_size]
        path = os.path.join(TOK_DIR, f"part_{i//part_size:03d}.npy")
        np.save(path, part)
        print(f"  Saved {path}: {len(part):,} tokens")
    print("Data prep complete.")


class DataIterator:
    """Streams tokenized data from .npy parts, yielding (input, target) batches."""
    def __init__(self, tok_dir, batch_size, seq_len, device="cpu"):
        self.parts = sorted(glob.glob(os.path.join(tok_dir, "part_*.npy")))
        if not self.parts:
            raise FileNotFoundError(f"No .npy files in {tok_dir}. Run --stage prepare first.")
        self.batch_size = batch_size
        self.seq_len = seq_len
        self.device = device
        self._buf = np.array([], dtype=np.int32)
        self._part_idx = 0
        self._rng = np.random.default_rng(42)

    def _refill(self):
        need = self.batch_size * self.seq_len + self.seq_len
        while len(self._buf) < need:
            if self._part_idx >= len(self.parts):
                self._part_idx = 0
            part = np.load(self.parts[self._part_idx], mmap_mode="r")
            self._buf = np.concatenate([self._buf, np.array(part)])
            self._part_idx += 1

    def __iter__(self):
        while True:
            self._refill()
            max_start = len(self._buf) - self.batch_size * self.seq_len - self.seq_len
            if max_start < 0:
                self._refill()
                continue
            start = int(self._rng.integers(0, max_start))
            chunk = self._buf[start:start + self.batch_size * self.seq_len + self.seq_len]
            flat = chunk.reshape(self.batch_size, self.seq_len + 1)
            x = torch.tensor(flat[:, :-1], dtype=torch.long, device=self.device)
            y = torch.tensor(flat[:, 1:], dtype=torch.long, device=self.device)
            yield x, y


# ============================================================================
# Training
# ============================================================================
def train(
    steps=12207,
    batch_size=32,
    seq_len=SEQ_LEN,
    lr=2e-4,
    min_lr=2e-5,
    warmup=200,
    accum=1,
    ckpt_every=500,
    eval_every=500,
    device="cuda",
    resume=None,
):
    """Train ram-18m from scratch."""
    torch.manual_seed(42)
    model = RAM18M()
    n_params = model.count_params()
    print(f"Model: {n_params:,} params")

    start_step = 0
    if resume:
        ckpt = torch.load(resume, map_location="cpu")
        model.load_state_dict(ckpt["model"])
        start_step = ckpt["step"]
        print(f"Resumed from {resume} at step {start_step}")

    if device == "cuda" and torch.cuda.is_available():
        model = model.cuda()
    else:
        device = "cpu"
        model = model.to(device)
    print(f"Device: {device}")

    data_iter = DataIterator(TOK_DIR, batch_size, seq_len, device)
    opt = AdamW(model.parameters(), lr=lr, betas=(0.9, 0.95), weight_decay=0.0)

    model.train()
    t0 = time.time()

    for step in range(start_step, steps):
        opt.zero_grad()
        loss_accum = 0.0
        for _ in range(accum):
            x, y = next(iter(data_iter))
            _, loss = model(x, y)
            loss = loss / accum
            loss.backward()
            loss_accum += loss.item()

        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        loss_val = loss_accum

        if (step + 1) % 10 == 0:
            elapsed = time.time() - t0
            tok_per_sec = (step + 1 - start_step) * batch_size * seq_len / max(elapsed, 1)
            lr_now = get_lr(step, steps, warmup, lr, min_lr)
            print(f"step {step+1}/{steps} | loss {loss_val:.4f} | lr {lr_now:.6f} | {tok_per_sec:.0f} tok/s | {elapsed:.0f}s")

        if (step + 1) % ckpt_every == 0:
            path = os.path.join(CKPT_DIR, f"ckpt_step{step+1}.pt")
            torch.save({"model": model.state_dict(), "opt": opt.state_dict(),
                        "step": step + 1, "loss": loss_val}, path)
            print(f"  checkpoint -> {path}")

        if (step + 1) % eval_every == 0:
            model.eval()
            with torch.no_grad():
                x, y = next(iter(data_iter))
                _, eval_loss = model(x, y)
            model.train()
            print(f"  eval_loss (1 batch): {eval_loss.item():.4f}")

    path = os.path.join(CKPT_DIR, "final.pt")
    torch.save({"model": model.state_dict(), "step": steps, "loss": loss_val}, path)
    print(f"Training complete. Final model -> {path}")

    # Print sample
    model.eval()
    torch.manual_seed(0)
    with torch.no_grad():
        prompt = torch.tensor([[3]], device=device)
        for _ in range(200):
            logits, _ = model(prompt)
            next_tok = logits[0, -1].argmax()
            prompt = torch.cat([prompt, next_tok.unsqueeze(0)], dim=1)
    try:
        from tokenizers import Tokenizer
        tok_path = os.path.join(TOK_DIR, "tokenizer.json")
        if os.path.exists(tok_path):
            tok = Tokenizer.from_file(tok_path)
            text = tok.decode(prompt[0].tolist())
            print(f"\nSample generation:\n{text[:500]}")
    except Exception:
        pass


# ============================================================================
# Eval: zero-shot loglikelihood on standard benchmarks
# ============================================================================
def eval_benchmarks(ckpt_path, device="cuda", n_samples=500):
    """Run zero-shot loglikelihood eval on PIQA, ARC-Easy, ARC-Challenge, HellaSwag."""
    from datasets import load_dataset

    model = RAM18M()
    ckpt = torch.load(ckpt_path, map_location="cpu")
    model.load_state_dict(ckpt["model"])
    if device == "cuda" and torch.cuda.is_available():
        model = model.cuda()
    else:
        device = "cpu"
    model.eval()

    tok_path = os.path.join(TOK_DIR, "tokenizer.json")
    from tokenizers import Tokenizer
    tokenizer = Tokenizer.from_file(tok_path)

    def encode(text):
        return tokenizer.encode(text).ids

    def loglikelihood(context, continuation):
        full_ids = encode(context + " " + continuation)
        ctx_ids = encode(context)
        ctx_len = min(len(ctx_ids), len(full_ids) - 1)
        if ctx_len < 1:
            return -1000.0
        input_ids = torch.tensor([full_ids], device=device)
        with torch.no_grad():
            logits, _ = model(input_ids)
        log_probs = F.log_softmax(logits[0, ctx_len-1:-1, :], dim=-1)
        target_ids = torch.tensor(full_ids[ctx_len:], device=device)
        if len(target_ids) == 0:
            return -1000.0
        return log_probs.gather(1, target_ids.unsqueeze(1)).sum().item()

    def accuracy(items, n=n_samples):
        correct = 0
        total = 0
        for item in items[:n]:
            ctx = item["context"]
            options = item["options"]
            label = item["label"]
            lls = [loglikelihood(ctx, opt) for opt in options]
            pred = max(range(len(lls)), key=lambda i: lls[i])
            if pred == label:
                correct += 1
            total += 1
            if total % 50 == 0:
                print(f"    {total}/{n} done, acc so far: {100*correct/total:.1f}%")
        return 100.0 * correct / max(total, 1)

    results = {}

    print("Loading PIQA...")
    piqa = load_dataset("ybisk/piqa", split="validation")
    piqa_items = [{"context":