bytegpt-byte-85M / model.py
ShadeEngine's picture
Upload folder using huggingface_hub
1839975 verified
Raw History Blame Contribute Delete
7.55 kB
"""
Byte-level GPT. nanoGPT-style minimalism, modern internals:
- vocab = 256 (raw bytes). No tokenizer anywhere.
- RMSNorm (pre-norm), RoPE, SwiGLU MLP, no biases
- flash attention via F.scaled_dot_product_attention
- weight tying between byte embedding and output head
The byte embedding table is 256 x d_model -- essentially free. In a BPE
model, the 50k x d_model embedding can be a third of all parameters; here
virtually all parameters do actual computation.
"""
import math
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class GPTConfig:
vocab_size: int = 256
block_size: int = 8192 # context length in bytes
n_layer: int = 12
n_head: int = 12
n_embd: int = 768
dropout: float = 0.0
rope_theta: float = 100000.0
PRESETS = {
# ~10M params: smoke tests and pipeline debugging
"micro": dict(n_layer=6, n_head=6, n_embd=384, block_size=2048),
# ~29M params: scaling-ladder point
"mini": dict(n_layer=9, n_head=9, n_embd=576, block_size=8192),
# ~85M non-embedding params: GPT-2 small compute equivalent
"small": dict(n_layer=12, n_head=12, n_embd=768, block_size=8192),
# BPE twin of "small": same backbone, 32k vocab (+25M embedding params),
# 2048-token context ~ 9.5KB of text (information-matched to 8192 bytes)
"small-bpe": dict(n_layer=12, n_head=12, n_embd=768, block_size=2048,
vocab_size=32768),
}
class RMSNorm(nn.Module):
def __init__(self, dim: int):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
dtype = x.dtype
x = x.float()
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6)
return (x * self.weight.float()).to(dtype)
def precompute_rope(head_dim: int, max_seq: int, theta: float):
"""Rotation frequencies for RoPE: positions are encoded by rotating
q/k pairs of dims, so attention scores depend on *relative* offsets."""
inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
t = torch.arange(max_seq).float()
freqs = torch.outer(t, inv_freq) # (max_seq, head_dim/2)
return torch.cos(freqs), torch.sin(freqs)
def apply_rope(x, cos, sin):
# x: (B, n_head, T, head_dim)
T = x.size(2)
cos, sin = cos[:T].to(x.dtype), sin[:T].to(x.dtype)
x1, x2 = x.chunk(2, dim=-1)
return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1)
class Attention(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
assert cfg.n_embd % cfg.n_head == 0
self.n_head = cfg.n_head
self.head_dim = cfg.n_embd // cfg.n_head
self.wqkv = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=False)
self.wo = nn.Linear(cfg.n_embd, cfg.n_embd, bias=False)
self.dropout = cfg.dropout
def forward(self, x, cos, sin):
B, T, C = x.shape
q, k, v = self.wqkv(x).split(C, dim=2)
q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
k = k.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
v = v.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
y = F.scaled_dot_product_attention(
q, k, v, is_causal=True,
dropout_p=self.dropout if self.training else 0.0,
)
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.wo(y)
class SwiGLU(nn.Module):
"""Gated MLP: out = W2( silu(W_gate x) * (W_up x) ).
Hidden dim chosen ~8/3 * d_model (rounded to 128) so total params match
the classic 4x GELU MLP."""
def __init__(self, cfg: GPTConfig):
super().__init__()
hidden = int(8 * cfg.n_embd / 3)
hidden = 128 * round(hidden / 128)
self.w_gate = nn.Linear(cfg.n_embd, hidden, bias=False)
self.w_up = nn.Linear(cfg.n_embd, hidden, bias=False)
self.w_down = nn.Linear(hidden, cfg.n_embd, bias=False)
def forward(self, x):
return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x))
class Block(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
self.norm1 = RMSNorm(cfg.n_embd)
self.attn = Attention(cfg)
self.norm2 = RMSNorm(cfg.n_embd)
self.mlp = SwiGLU(cfg)
def forward(self, x, cos, sin):
x = x + self.attn(self.norm1(x), cos, sin)
x = x + self.mlp(self.norm2(x))
return x
class ByteGPT(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
self.cfg = cfg
self.embed = nn.Embedding(cfg.vocab_size, cfg.n_embd)
self.blocks = nn.ModuleList(Block(cfg) for _ in range(cfg.n_layer))
self.norm_f = RMSNorm(cfg.n_embd)
self.lm_head = nn.Linear(cfg.n_embd, cfg.vocab_size, bias=False)
self.lm_head.weight = self.embed.weight # weight tying
cos, sin = precompute_rope(cfg.n_embd // cfg.n_head, cfg.block_size, cfg.rope_theta)
self.register_buffer("rope_cos", cos, persistent=False)
self.register_buffer("rope_sin", sin, persistent=False)
self.apply(self._init_weights)
# scale down residual-branch output projections (GPT-2 trick):
# keeps the residual stream variance stable across depth
for pn, p in self.named_parameters():
if pn.endswith("wo.weight") or pn.endswith("w_down.weight"):
nn.init.normal_(p, std=0.02 / math.sqrt(2 * cfg.n_layer))
def _init_weights(self, m):
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, std=0.02)
elif isinstance(m, nn.Embedding):
nn.init.normal_(m.weight, std=0.02)
def num_params(self):
return sum(p.numel() for p in self.parameters())
def forward(self, idx, targets=None):
x = self.embed(idx)
for blk in self.blocks:
x = blk(x, self.rope_cos, self.rope_sin)
x = self.norm_f(x)
if targets is not None:
logits = self.lm_head(x)
loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1))
return logits, loss
# inference: only the last position is needed
logits = self.lm_head(x[:, [-1], :])
return logits, None
def configure_optimizer(self, weight_decay, lr, betas, device_type):
# decay only matrices; norms and embeddings stay undecayed
decay = [p for n, p in self.named_parameters() if p.requires_grad and p.dim() >= 2]
nodecay = [p for n, p in self.named_parameters() if p.requires_grad and p.dim() < 2]
groups = [
{"params": decay, "weight_decay": weight_decay},
{"params": nodecay, "weight_decay": 0.0},
]
fused = device_type == "cuda"
return torch.optim.AdamW(groups, lr=lr, betas=betas, fused=fused)
@torch.no_grad()
def generate(self, idx, max_new_bytes, temperature=1.0, top_k=None):
for _ in range(max_new_bytes):
idx_cond = idx[:, -self.cfg.block_size:]
logits, _ = self(idx_cond)
logits = logits[:, -1, :] / max(temperature, 1e-6)
if top_k is not None:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[:, [-1]]] = -float("inf")
probs = F.softmax(logits, dim=-1)
nxt = torch.multinomial(probs, num_samples=1)
idx = torch.cat([idx, nxt], dim=1)
return idx