File size: 5,435 Bytes
06513db 0247f2b 06513db 0247f2b 4aba285 0247f2b 06513db f2c7fb0 06513db f2c7fb0 06513db f2c7fb0 0247f2b f2c7fb0 0247f2b 06513db f2c7fb0 0247f2b f2c7fb0 06513db f2c7fb0 06513db f2c7fb0 06513db f2c7fb0 06513db 0247f2b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """
Tiny GPT-style transformer (~30M params target).
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PreTrainedModel, PretrainedConfig
# ========== CONFIG CLASS (embedded to avoid import issues) ==========
class GPTConfig(PretrainedConfig):
model_type = "tinybuddy"
def __init__(
self,
vocab_size: int = 50000,
block_size: int = 128,
n_layer: int = 6,
n_head: int = 8,
n_embd: int = 256,
mlp_ratio: int = 4,
dropout: float = 0.0,
tie_weights: bool = False,
**kwargs
):
super().__init__(**kwargs)
self.vocab_size = vocab_size
self.block_size = block_size
self.n_layer = n_layer
self.n_head = n_head
self.n_embd = n_embd
self.mlp_ratio = mlp_ratio
self.dropout = dropout
self.tie_weights = tie_weights
# ========== MODEL ARCHITECTURE ==========
class CausalSelfAttention(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
assert cfg.n_embd % cfg.n_head == 0
self.n_head = cfg.n_head
self.n_embd = cfg.n_embd
self.head_dim = cfg.n_embd // cfg.n_head
self.qkv = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=True)
self.proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=True)
self.drop = nn.Dropout(cfg.dropout)
mask = torch.tril(torch.ones(cfg.block_size, cfg.block_size)).bool()
self.register_buffer("mask", mask, persistent=False)
def forward(self, x):
B, T, C = x.shape
qkv = self.qkv(x)
q, k, v = qkv.split(self.n_embd, dim=2)
q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
k = k.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
v = v.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
y = F.scaled_dot_product_attention(q, k, v, is_causal=True,
dropout_p=self.drop.p if self.training else 0.0)
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.proj(y)
class MLP(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
hidden = cfg.mlp_ratio * cfg.n_embd
self.fc1 = nn.Linear(cfg.n_embd, hidden, bias=True)
self.fc2 = nn.Linear(hidden, cfg.n_embd, bias=True)
self.drop = nn.Dropout(cfg.dropout)
def forward(self, x):
return self.drop(self.fc2(F.gelu(self.fc1(x))))
class Block(nn.Module):
def __init__(self, cfg: GPTConfig):
super().__init__()
self.ln1 = nn.LayerNorm(cfg.n_embd)
self.attn = CausalSelfAttention(cfg)
self.ln2 = nn.LayerNorm(cfg.n_embd)
self.mlp = MLP(cfg)
def forward(self, x):
x = x + self.attn(self.ln1(x))
x = x + self.mlp(self.ln2(x))
return x
class TinyGPT(PreTrainedModel):
config_class = GPTConfig
def __init__(self, config: GPTConfig):
super().__init__(config)
self.config = config
self.tok_emb = nn.Embedding(config.vocab_size, config.n_embd)
self.pos_emb = nn.Embedding(config.block_size, config.n_embd)
self.drop = nn.Dropout(config.dropout)
self.blocks = nn.ModuleList([Block(config) for _ in range(config.n_layer)])
self.ln_f = nn.LayerNorm(config.n_embd)
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
if config.tie_weights:
self.lm_head.weight = self.tok_emb.weight
self.post_init()
def _init_weights(self, module):
if isinstance(module, nn.Linear):
module.weight.data.normal_(mean=0.0, std=0.02)
if module.bias is not None:
module.bias.data.zero_()
elif isinstance(module, nn.Embedding):
module.weight.data.normal_(mean=0.0, std=0.02)
def forward(self, input_ids, attention_mask=None, labels=None, **kwargs):
B, T = input_ids.shape
assert T <= self.config.block_size
pos = torch.arange(T, device=input_ids.device)
x = self.tok_emb(input_ids) + self.pos_emb(pos)[None, :, :]
x = self.drop(x)
for blk in self.blocks:
x = blk(x)
x = self.ln_f(x)
logits = self.lm_head(x)
loss = None
if labels is not None:
loss = F.cross_entropy(logits.view(-1, logits.size(-1)),
labels.view(-1), ignore_index=-100)
return (logits,) if loss is None else (logits, loss)
def generate(self, idx, max_new_tokens=100, temperature=1.0, top_k=None):
self.eval()
for _ in range(max_new_tokens):
idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:]
logits = self(idx_cond)[0]
logits = logits[:, -1, :] / max(temperature, 1e-6)
if top_k is not None:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[:, [-1]]] = -float("inf")
probs = F.softmax(logits, dim=-1)
next_id = torch.multinomial(probs, num_samples=1)
idx = torch.cat([idx, next_id], dim=1)
return idx
if __name__ == "__main__":
cfg = GPTConfig()
m = TinyGPT(cfg)
total = sum(p.numel() for p in m.parameters())
print(f"Total params: {total:,} (~{total/1e6:.2f}M)") |