model_v5.py

"""
V5 Model — 200M parametre, V4 mimari + büyütülmüş.

Mimari:
  - Layer: 18 (V4: 8)
  - Head: 14 (V4: 10)
  - Embd: 896 (V4: 640)
  - Vocab: 32000 (V4: 16000)
  - Context: 2048 (V4: 512)
  - Toplam: ~210M parametre

Modern teknikler (V4'ten):
  - RoPE (real-valued)
  - RMSNorm
  - SwiGLU (hidden ~2560)
  - QK-norm
  - Logit soft-cap
  - Tied embeddings
  - Scaled init
"""

import math
import os
from dataclasses import dataclass

import torch
import torch.nn as nn
import torch.nn.functional as F

# Liger Kernel — Blackwell uyumlu Triton kernel'leri (RMSNorm, SwiGLU, CE)
# State_dict formati DEGISMEZ — resume guvenli.
# Devre disi birakmak icin: NANOGPT_NO_LIGER=1
HAS_LIGER = False
if not os.environ.get("NANOGPT_NO_LIGER"):
    try:
        from liger_kernel.ops.rms_norm import LigerRMSNormFunction
        from liger_kernel.ops.swiglu import LigerSiLUMulFunction
        from liger_kernel.transformers.cross_entropy import LigerCrossEntropyLoss
        HAS_LIGER = True
    except ImportError:
        HAS_LIGER = False


@dataclass
class GPTConfigV5:
    block_size: int = 2048
    vocab_size: int = 32000
    n_layer: int = 18
    n_head: int = 14
    n_embd: int = 896
    dropout: float = 0.0
    rope_theta: float = 10000.0
    logit_softcap: float = 30.0


class RMSNorm(nn.Module):
    """RMSNorm. Liger varsa fused Triton kernel kullanir, yoksa plain PyTorch.
    State dict: sadece 'weight' — Liger ile compat, resume guvenli."""
    def __init__(self, dim: int, eps: float = 1e-6):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(dim))
        self.eps = eps
        self.dim = dim

    def forward(self, x):
        if HAS_LIGER and x.is_cuda:
            # Liger args: (X, W, eps, offset, casting_mode, in_place)
            return LigerRMSNormFunction.apply(
                x, self.weight, self.eps, 0.0, "llama", False
            )
        dtype = x.dtype
        x = x.float()
        rms = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
        return (self.weight * (x * rms)).to(dtype)


def precompute_rope(dim, end, theta=10000.0, device=None):
    inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, device=device).float() / dim))
    t = torch.arange(end, device=device, dtype=torch.float32)
    freqs = torch.outer(t, inv_freq)
    emb = torch.cat([freqs, freqs], dim=-1)
    return emb.cos(), emb.sin()


def rotate_half(x):
    x1, x2 = x.chunk(2, dim=-1)
    return torch.cat([-x2, x1], dim=-1)


def apply_rotary_emb(xq, xk, cos, sin):
    cos = cos.unsqueeze(0).unsqueeze(0)
    sin = sin.unsqueeze(0).unsqueeze(0)
    xq_out = (xq * cos) + (rotate_half(xq) * sin)
    xk_out = (xk * cos) + (rotate_half(xk) * sin)
    return xq_out.to(xq.dtype), xk_out.to(xk.dtype)


class CausalSelfAttention(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        assert cfg.n_embd % cfg.n_head == 0
        self.n_head = cfg.n_head
        self.n_embd = cfg.n_embd
        self.head_dim = cfg.n_embd // cfg.n_head
        self.c_attn = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=False)
        self.c_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=False)
        self.dropout = cfg.dropout
        self.q_norm = RMSNorm(self.head_dim)
        self.k_norm = RMSNorm(self.head_dim)

    def forward(self, x, cos, sin):
        B, T, C = x.shape
        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
        q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
        k = k.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
        v = v.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
        q = self.q_norm(q)
        k = self.k_norm(k)
        q, k = apply_rotary_emb(q, k, cos[:T], sin[:T])
        y = F.scaled_dot_product_attention(
            q, k, v, dropout_p=self.dropout if self.training else 0.0, is_causal=True
        )
        y = y.transpose(1, 2).contiguous().view(B, T, C)
        return self.c_proj(y)


class SwiGLU(nn.Module):
    """SwiGLU MLP. Liger varsa fused silu*mul Triton kernel kullanir.
    Param isimleri (w1/w2/w3) korunur — resume guvenli."""
    def __init__(self, dim, hidden_dim=None):
        super().__init__()
        if hidden_dim is None:
            hidden_dim = int(8 * dim / 3)
            hidden_dim = ((hidden_dim + 255) // 256) * 256
        self.w1 = nn.Linear(dim, hidden_dim, bias=False)
        self.w2 = nn.Linear(dim, hidden_dim, bias=False)
        self.w3 = nn.Linear(hidden_dim, dim, bias=False)
        self.hidden_dim = hidden_dim

    def forward(self, x):
        gate = self.w1(x)
        up = self.w2(x)
        if HAS_LIGER and x.is_cuda:
            # Fused silu(gate) * up — tek kernel, intermediate memory tasarrufu
            hidden = LigerSiLUMulFunction.apply(gate, up)
        else:
            hidden = F.silu(gate) * up
        return self.w3(hidden)


class Block(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.norm1 = RMSNorm(cfg.n_embd)
        self.attn = CausalSelfAttention(cfg)
        self.norm2 = RMSNorm(cfg.n_embd)
        self.mlp = SwiGLU(cfg.n_embd)

    def forward(self, x, cos, sin):
        x = x + self.attn(self.norm1(x), cos, sin)
        x = x + self.mlp(self.norm2(x))
        return x


class GPTV5(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.cfg = cfg
        self.wte = nn.Embedding(cfg.vocab_size, cfg.n_embd)
        self.h = nn.ModuleList([Block(cfg) for _ in range(cfg.n_layer)])
        self.norm_f = RMSNorm(cfg.n_embd)
        self.lm_head = nn.Linear(cfg.n_embd, cfg.vocab_size, bias=False)
        self.lm_head.weight = self.wte.weight  # tied

        head_dim = cfg.n_embd // cfg.n_head
        cos, sin = precompute_rope(head_dim, cfg.block_size * 2, cfg.rope_theta)
        self.register_buffer("rope_cos", cos, persistent=False)
        self.register_buffer("rope_sin", sin, persistent=False)

        self.apply(self._init_weights)
        for pn, p in self.named_parameters():
            if pn.endswith("c_proj.weight") or pn.endswith("w3.weight"):
                nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * cfg.n_layer))

        # Liger CrossEntropy (chunked, vocab=32K icin BUYUK bellek tasarrufu)
        if HAS_LIGER:
            self._ce_loss = LigerCrossEntropyLoss(ignore_index=-1, reduction="mean")
        else:
            self._ce_loss = None

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)

    def num_params(self):
        return sum(p.numel() for p in self.parameters())

    def forward(self, idx, targets=None):
        B, T = idx.shape
        max_t = self.rope_cos.size(0)
        assert T <= max_t, f"T={T} > rope buffer={max_t}"

        x = self.wte(idx)
        cos = self.rope_cos[:T]
        sin = self.rope_sin[:T]
        for block in self.h:
            x = block(x, cos, sin)
        x = self.norm_f(x)

        if targets is not None:
            logits = self.lm_head(x)
            if self.cfg.logit_softcap > 0:
                cap = self.cfg.logit_softcap
                logits = cap * torch.tanh(logits / cap)
            flat_logits = logits.view(-1, logits.size(-1))
            flat_targets = targets.view(-1)
            if self._ce_loss is not None:
                # Liger chunked CE — softmax materyalizasyonu yok, ~5-10GB tasarruf
                loss = self._ce_loss(flat_logits, flat_targets)
            else:
                loss = F.cross_entropy(flat_logits, flat_targets, ignore_index=-1)
            return logits, loss
        else:
            logits = self.lm_head(x[:, [-1], :])
            if self.cfg.logit_softcap > 0:
                cap = self.cfg.logit_softcap
                logits = cap * torch.tanh(logits / cap)
            return logits, None

    @torch.no_grad()
    def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None,
                 repetition_penalty=1.15, no_repeat_ngram_size=3,
                 max_context=None, eos_token_id=None):
        """eos_token_id verildiyse model bu token'i sample ettiginde
        early stop. SFT modelleri icin EOT (=0) onerilir."""
        if max_context is None:
            max_context = self.rope_cos.size(0)
        for _ in range(max_new_tokens):
            idx_cond = idx if idx.size(1) <= max_context else idx[:, -max_context:]
            logits, _ = self(idx_cond)
            logits = logits[:, -1, :] / temperature

            if repetition_penalty != 1.0:
                for b in range(idx.size(0)):
                    seen = set(idx[b].tolist()[-256:])
                    for tok in seen:
                        # EOS token'a rep penalty UYGULAMA — durmasi gerek
                        if eos_token_id is not None and tok == eos_token_id:
                            continue
                        if logits[b, tok] > 0:
                            logits[b, tok] /= repetition_penalty
                        else:
                            logits[b, tok] *= repetition_penalty

            if no_repeat_ngram_size > 0 and idx.size(1) >= no_repeat_ngram_size:
                for b in range(idx.size(0)):
                    tokens = idx[b].tolist()
                    n = no_repeat_ngram_size
                    prefix = tuple(tokens[-(n-1):])
                    banned = set()
                    for i in range(len(tokens) - n + 1):
                        if tuple(tokens[i:i+n-1]) == prefix:
                            banned.add(tokens[i+n-1])
                    # EOS asla banlanmasin
                    if eos_token_id is not None:
                        banned.discard(eos_token_id)
                    for tok in banned:
                        logits[b, tok] = -float("inf")

            if top_k is not None:
                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
                logits[logits < v[:, [-1]]] = -float("inf")
            probs = F.softmax(logits, dim=-1)
            next_id = torch.multinomial(probs, num_samples=1)
            idx = torch.cat([idx, next_id], dim=1)

            # Early stop — batch'teki TUM ornek EOS'a ulastiysa
            if eos_token_id is not None:
                if (next_id == eos_token_id).all():
                    break
        return idx


if __name__ == "__main__":
    cfg = GPTConfigV5()
    m = GPTV5(cfg)
    print(f"V5: {m.num_params()/1e6:.2f}M param")
    print(f"  Layer: {cfg.n_layer}, Head: {cfg.n_head}, Embd: {cfg.n_embd}")
    print(f"  Vocab: {cfg.vocab_size}, Block: {cfg.block_size}")
    print(f"  SwiGLU hidden: {m.h[0].mlp.hidden_dim}")
    x = torch.randint(0, cfg.vocab_size, (2, 64))
    logits, loss = m(x, x)
    print(f"Forward: loss {loss.item():.4f}")