File size: 7,546 Bytes
1839975 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | """
Byte-level GPT. nanoGPT-style minimalism, modern internals:
- vocab = 256 (raw bytes). No tokenizer anywhere.
- RMSNorm (pre-norm), RoPE, SwiGLU MLP, no biases
- flash attention via F.scaled_dot_product_attention
- weight tying between byte embedding and output head
The byte embedding table is 256 x d_model -- essentially free. In a BPE
model, the 50k x d_model embedding can be a third of all parameters; here
virtually all parameters do actual computation.
"""
import math
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class GPTConfig:
vocab_size: int = 256
block_size: int = 8192 # context length in bytes
n_layer: int = 12
n_head: int = 12
n_embd: int = 768
dropout: float = 0.0
rope_theta: float = 100000.0
PRESETS = {
# ~10M params: smoke tests and pipeline debugging
"micro": dict(n_layer=6, n_head=6, n_embd=384, block_size=2048),
# ~29M params: scaling-ladder point
"mini": dict(n_layer=9, n_head=9, n_embd=576, block_size=8192),
# ~85M non-embedding params: GPT-2 small compute equivalent
"small": dict(n_layer=12, n_head=12, n_embd=768, block_size=8192),
# BPE twin of "small": same backbone, 32k vocab (+25M embedding params),
# 2048-token context ~ 9.5KB of text (information-matched to 8192 bytes)
"small-bpe": dict(n_layer=12, n_head=12, n_embd=768, block_size=2048,
vocab_size=32768),
}
class RMSNorm(nn.Module):
def __init__(self, dim: int):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
dtype = x.dtype
x = x.float()
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6)
return (x * self.weight.float()).to(dtype)
def precompute_rope(head_dim: int, max_seq: int, theta: float):
"""Rotation frequencies for RoPE: positions are encoded by rotating
q/k pairs of dims, so attention scores depend on *relative* offsets."""
inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
t = torch.arange(max_seq).float()
freqs = torch.outer(t, inv_freq) # (max_seq, head_dim/2)
return torch.cos(freqs), torch.sin(freqs)
def apply_rope(x, cos, sin):
# x: (B, n_head, T, head_dim)
T = x.size(2)
cos, sin = cos[:T].to(x.dtype), sin[:T].to(x.dtype)
x1, x2 = x.chunk(2, dim=-1)
return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1)
class Attention(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
assert cfg.n_embd % cfg.n_head == 0
self.n_head = cfg.n_head
self.head_dim = cfg.n_embd // cfg.n_head
self.wqkv = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=False)
self.wo = nn.Linear(cfg.n_embd, cfg.n_embd, bias=False)
self.dropout = cfg.dropout
def forward(self, x, cos, sin):
B, T, C = x.shape
q, k, v = self.wqkv(x).split(C, dim=2)
q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
k = k.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
v = v.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
y = F.scaled_dot_product_attention(
q, k, v, is_causal=True,
dropout_p=self.dropout if self.training else 0.0,
)
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.wo(y)
class SwiGLU(nn.Module):
"""Gated MLP: out = W2( silu(W_gate x) * (W_up x) ).
Hidden dim chosen ~8/3 * d_model (rounded to 128) so total params match
the classic 4x GELU MLP."""
def __init__(self, cfg: GPTConfig):
super().__init__()
hidden = int(8 * cfg.n_embd / 3)
hidden = 128 * round(hidden / 128)
self.w_gate = nn.Linear(cfg.n_embd, hidden, bias=False)
self.w_up = nn.Linear(cfg.n_embd, hidden, bias=False)
self.w_down = nn.Linear(hidden, cfg.n_embd, bias=False)
def forward(self, x):
return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x))
class Block(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
self.norm1 = RMSNorm(cfg.n_embd)
self.attn = Attention(cfg)
self.norm2 = RMSNorm(cfg.n_embd)
self.mlp = SwiGLU(cfg)
def forward(self, x, cos, sin):
x = x + self.attn(self.norm1(x), cos, sin)
x = x + self.mlp(self.norm2(x))
return x
class ByteGPT(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
self.cfg = cfg
self.embed = nn.Embedding(cfg.vocab_size, cfg.n_embd)
self.blocks = nn.ModuleList(Block(cfg) for _ in range(cfg.n_layer))
self.norm_f = RMSNorm(cfg.n_embd)
self.lm_head = nn.Linear(cfg.n_embd, cfg.vocab_size, bias=False)
self.lm_head.weight = self.embed.weight # weight tying
cos, sin = precompute_rope(cfg.n_embd // cfg.n_head, cfg.block_size, cfg.rope_theta)
self.register_buffer("rope_cos", cos, persistent=False)
self.register_buffer("rope_sin", sin, persistent=False)
self.apply(self._init_weights)
# scale down residual-branch output projections (GPT-2 trick):
# keeps the residual stream variance stable across depth
for pn, p in self.named_parameters():
if pn.endswith("wo.weight") or pn.endswith("w_down.weight"):
nn.init.normal_(p, std=0.02 / math.sqrt(2 * cfg.n_layer))
def _init_weights(self, m):
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, std=0.02)
elif isinstance(m, nn.Embedding):
nn.init.normal_(m.weight, std=0.02)
def num_params(self):
return sum(p.numel() for p in self.parameters())
def forward(self, idx, targets=None):
x = self.embed(idx)
for blk in self.blocks:
x = blk(x, self.rope_cos, self.rope_sin)
x = self.norm_f(x)
if targets is not None:
logits = self.lm_head(x)
loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1))
return logits, loss
# inference: only the last position is needed
logits = self.lm_head(x[:, [-1], :])
return logits, None
def configure_optimizer(self, weight_decay, lr, betas, device_type):
# decay only matrices; norms and embeddings stay undecayed
decay = [p for n, p in self.named_parameters() if p.requires_grad and p.dim() >= 2]
nodecay = [p for n, p in self.named_parameters() if p.requires_grad and p.dim() < 2]
groups = [
{"params": decay, "weight_decay": weight_decay},
{"params": nodecay, "weight_decay": 0.0},
]
fused = device_type == "cuda"
return torch.optim.AdamW(groups, lr=lr, betas=betas, fused=fused)
@torch.no_grad()
def generate(self, idx, max_new_bytes, temperature=1.0, top_k=None):
for _ in range(max_new_bytes):
idx_cond = idx[:, -self.cfg.block_size:]
logits, _ = self(idx_cond)
logits = logits[:, -1, :] / max(temperature, 1e-6)
if top_k is not None:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[:, [-1]]] = -float("inf")
probs = F.softmax(logits, dim=-1)
nxt = torch.multinomial(probs, num_samples=1)
idx = torch.cat([idx, nxt], dim=1)
return idx
|