Download model.py from ShadeEngine/bytegpt-byte-85M: direct link, hf CLI and curl.
- Browser
- Download file 7.55 kB
-
https://huggingface.co/ShadeEngine/bytegpt-byte-85M/resolve/main/model.py
- Command line
-
hf download hf://ShadeEngine/bytegpt-byte-85M/model.py
-
curl -L -o model.py https://huggingface.co/ShadeEngine/bytegpt-byte-85M/resolve/main/model.py
7.55 kB
| """ | |
| Byte-level GPT. nanoGPT-style minimalism, modern internals: | |
| - vocab = 256 (raw bytes). No tokenizer anywhere. | |
| - RMSNorm (pre-norm), RoPE, SwiGLU MLP, no biases | |
| - flash attention via F.scaled_dot_product_attention | |
| - weight tying between byte embedding and output head | |
| The byte embedding table is 256 x d_model -- essentially free. In a BPE | |
| model, the 50k x d_model embedding can be a third of all parameters; here | |
| virtually all parameters do actual computation. | |
| """ | |
| import math | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class GPTConfig: | |
| vocab_size: int = 256 | |
| block_size: int = 8192 # context length in bytes | |
| n_layer: int = 12 | |
| n_head: int = 12 | |
| n_embd: int = 768 | |
| dropout: float = 0.0 | |
| rope_theta: float = 100000.0 | |
| PRESETS = { | |
| # ~10M params: smoke tests and pipeline debugging | |
| "micro": dict(n_layer=6, n_head=6, n_embd=384, block_size=2048), | |
| # ~29M params: scaling-ladder point | |
| "mini": dict(n_layer=9, n_head=9, n_embd=576, block_size=8192), | |
| # ~85M non-embedding params: GPT-2 small compute equivalent | |
| "small": dict(n_layer=12, n_head=12, n_embd=768, block_size=8192), | |
| # BPE twin of "small": same backbone, 32k vocab (+25M embedding params), | |
| # 2048-token context ~ 9.5KB of text (information-matched to 8192 bytes) | |
| "small-bpe": dict(n_layer=12, n_head=12, n_embd=768, block_size=2048, | |
| vocab_size=32768), | |
| } | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim: int): | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x): | |
| dtype = x.dtype | |
| x = x.float() | |
| x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6) | |
| return (x * self.weight.float()).to(dtype) | |
| def precompute_rope(head_dim: int, max_seq: int, theta: float): | |
| """Rotation frequencies for RoPE: positions are encoded by rotating | |
| q/k pairs of dims, so attention scores depend on *relative* offsets.""" | |
| inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim)) | |
| t = torch.arange(max_seq).float() | |
| freqs = torch.outer(t, inv_freq) # (max_seq, head_dim/2) | |
| return torch.cos(freqs), torch.sin(freqs) | |
| def apply_rope(x, cos, sin): | |
| # x: (B, n_head, T, head_dim) | |
| T = x.size(2) | |
| cos, sin = cos[:T].to(x.dtype), sin[:T].to(x.dtype) | |
| x1, x2 = x.chunk(2, dim=-1) | |
| return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1) | |
| class Attention(nn.Module): | |
| def __init__(self, cfg: GPTConfig): | |
| super().__init__() | |
| assert cfg.n_embd % cfg.n_head == 0 | |
| self.n_head = cfg.n_head | |
| self.head_dim = cfg.n_embd // cfg.n_head | |
| self.wqkv = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=False) | |
| self.wo = nn.Linear(cfg.n_embd, cfg.n_embd, bias=False) | |
| self.dropout = cfg.dropout | |
| def forward(self, x, cos, sin): | |
| B, T, C = x.shape | |
| q, k, v = self.wqkv(x).split(C, dim=2) | |
| q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2) | |
| k = k.view(B, T, self.n_head, self.head_dim).transpose(1, 2) | |
| v = v.view(B, T, self.n_head, self.head_dim).transpose(1, 2) | |
| q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin) | |
| y = F.scaled_dot_product_attention( | |
| q, k, v, is_causal=True, | |
| dropout_p=self.dropout if self.training else 0.0, | |
| ) | |
| y = y.transpose(1, 2).contiguous().view(B, T, C) | |
| return self.wo(y) | |
| class SwiGLU(nn.Module): | |
| """Gated MLP: out = W2( silu(W_gate x) * (W_up x) ). | |
| Hidden dim chosen ~8/3 * d_model (rounded to 128) so total params match | |
| the classic 4x GELU MLP.""" | |
| def __init__(self, cfg: GPTConfig): | |
| super().__init__() | |
| hidden = int(8 * cfg.n_embd / 3) | |
| hidden = 128 * round(hidden / 128) | |
| self.w_gate = nn.Linear(cfg.n_embd, hidden, bias=False) | |
| self.w_up = nn.Linear(cfg.n_embd, hidden, bias=False) | |
| self.w_down = nn.Linear(hidden, cfg.n_embd, bias=False) | |
| def forward(self, x): | |
| return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x)) | |
| class Block(nn.Module): | |
| def __init__(self, cfg: GPTConfig): | |
| super().__init__() | |
| self.norm1 = RMSNorm(cfg.n_embd) | |
| self.attn = Attention(cfg) | |
| self.norm2 = RMSNorm(cfg.n_embd) | |
| self.mlp = SwiGLU(cfg) | |
| def forward(self, x, cos, sin): | |
| x = x + self.attn(self.norm1(x), cos, sin) | |
| x = x + self.mlp(self.norm2(x)) | |
| return x | |
| class ByteGPT(nn.Module): | |
| def __init__(self, cfg: GPTConfig): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.embed = nn.Embedding(cfg.vocab_size, cfg.n_embd) | |
| self.blocks = nn.ModuleList(Block(cfg) for _ in range(cfg.n_layer)) | |
| self.norm_f = RMSNorm(cfg.n_embd) | |
| self.lm_head = nn.Linear(cfg.n_embd, cfg.vocab_size, bias=False) | |
| self.lm_head.weight = self.embed.weight # weight tying | |
| cos, sin = precompute_rope(cfg.n_embd // cfg.n_head, cfg.block_size, cfg.rope_theta) | |
| self.register_buffer("rope_cos", cos, persistent=False) | |
| self.register_buffer("rope_sin", sin, persistent=False) | |
| self.apply(self._init_weights) | |
| # scale down residual-branch output projections (GPT-2 trick): | |
| # keeps the residual stream variance stable across depth | |
| for pn, p in self.named_parameters(): | |
| if pn.endswith("wo.weight") or pn.endswith("w_down.weight"): | |
| nn.init.normal_(p, std=0.02 / math.sqrt(2 * cfg.n_layer)) | |
| def _init_weights(self, m): | |
| if isinstance(m, nn.Linear): | |
| nn.init.normal_(m.weight, std=0.02) | |
| elif isinstance(m, nn.Embedding): | |
| nn.init.normal_(m.weight, std=0.02) | |
| def num_params(self): | |
| return sum(p.numel() for p in self.parameters()) | |
| def forward(self, idx, targets=None): | |
| x = self.embed(idx) | |
| for blk in self.blocks: | |
| x = blk(x, self.rope_cos, self.rope_sin) | |
| x = self.norm_f(x) | |
| if targets is not None: | |
| logits = self.lm_head(x) | |
| loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1)) | |
| return logits, loss | |
| # inference: only the last position is needed | |
| logits = self.lm_head(x[:, [-1], :]) | |
| return logits, None | |
| def configure_optimizer(self, weight_decay, lr, betas, device_type): | |
| # decay only matrices; norms and embeddings stay undecayed | |
| decay = [p for n, p in self.named_parameters() if p.requires_grad and p.dim() >= 2] | |
| nodecay = [p for n, p in self.named_parameters() if p.requires_grad and p.dim() < 2] | |
| groups = [ | |
| {"params": decay, "weight_decay": weight_decay}, | |
| {"params": nodecay, "weight_decay": 0.0}, | |
| ] | |
| fused = device_type == "cuda" | |
| return torch.optim.AdamW(groups, lr=lr, betas=betas, fused=fused) | |
| def generate(self, idx, max_new_bytes, temperature=1.0, top_k=None): | |
| for _ in range(max_new_bytes): | |
| idx_cond = idx[:, -self.cfg.block_size:] | |
| logits, _ = self(idx_cond) | |
| logits = logits[:, -1, :] / max(temperature, 1e-6) | |
| if top_k is not None: | |
| v, _ = torch.topk(logits, min(top_k, logits.size(-1))) | |
| logits[logits < v[:, [-1]]] = -float("inf") | |
| probs = F.softmax(logits, dim=-1) | |
| nxt = torch.multinomial(probs, num_samples=1) | |
| idx = torch.cat([idx, nxt], dim=1) | |
| return idx | |