"""A small GPT-style language model (character level).""" import torch import torch.nn as nn import torch.nn.functional as F class Block(nn.Module): """One transformer block: self-attention, then a small feed-forward net.""" def __init__(self, n_embd, n_head, dropout): super().__init__() self.n_head = n_head self.ln1 = nn.LayerNorm(n_embd) self.qkv = nn.Linear(n_embd, 3 * n_embd) self.proj = nn.Linear(n_embd, n_embd) self.ln2 = nn.LayerNorm(n_embd) self.mlp = nn.Sequential( nn.Linear(n_embd, 4 * n_embd), nn.GELU(), nn.Linear(4 * n_embd, n_embd) ) self.dropout = dropout def forward(self, x): B, T, C = x.shape q, k, v = self.qkv(self.ln1(x)).split(C, dim=2) q, k, v = (t.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) for t in (q, k, v)) # is_causal=True: each character may only look at the ones before it a = F.scaled_dot_product_attention( q, k, v, is_causal=True, dropout_p=self.dropout if self.training else 0.0 ) x = x + self.proj(a.transpose(1, 2).reshape(B, T, C)) return x + self.mlp(self.ln2(x)) class TinyGPT(nn.Module): def __init__(self, vocab_size, block_size=256, n_layer=6, n_head=8, n_embd=256, dropout=0.0): super().__init__() self.block_size = block_size self.tok_emb = nn.Embedding(vocab_size, n_embd) self.pos_emb = nn.Embedding(block_size, n_embd) self.blocks = nn.ModuleList(Block(n_embd, n_head, dropout) for _ in range(n_layer)) self.ln_f = nn.LayerNorm(n_embd) self.head = nn.Linear(n_embd, vocab_size) for m in self.modules(): # small random starting weights if isinstance(m, (nn.Linear, nn.Embedding)): nn.init.normal_(m.weight, std=0.02) if getattr(m, "bias", None) is not None: nn.init.zeros_(m.bias) for b in self.blocks: nn.init.normal_(b.proj.weight, std=0.02 / (2 * n_layer) ** 0.5) nn.init.normal_(b.mlp[2].weight, std=0.02 / (2 * n_layer) ** 0.5) def forward(self, idx, targets=None, mask=None): """mask (optional) picks which positions count towards the loss.""" T = idx.shape[1] x = self.tok_emb(idx) + self.pos_emb(torch.arange(T, device=idx.device)) for block in self.blocks: x = block(x) logits = self.head(self.ln_f(x)) loss = None if targets is not None: loss = F.cross_entropy(logits.float().view(-1, logits.size(-1)), targets.view(-1), reduction="none") if mask is None: loss = loss.mean() else: m = mask.view(-1).float() loss = (loss * m).sum() / m.sum().clamp(min=1) return logits, loss @torch.no_grad() def generate(self, idx, max_new_tokens, temperature=0.8, top_k=None): """Yield one new token id at a time, feeding each back in as input.""" for _ in range(max_new_tokens): logits, _ = self(idx[:, -self.block_size:]) logits = logits[:, -1, :].float() if temperature <= 0: # 0 means: always take the most likely character nxt = logits.argmax(-1, keepdim=True) else: logits = logits / temperature if top_k: kth = torch.topk(logits, min(top_k, logits.size(-1))).values[:, [-1]] logits[logits < kth] = -float("inf") nxt = torch.multinomial(F.softmax(logits, dim=-1), 1) idx = torch.cat([idx, nxt], dim=1) yield nxt.item()