MemoGPT-30M / model.py
memostarkbaba's picture
Upload folder using huggingface_hub
baa088a verified
Raw History Blame Contribute Delete
6.98 kB
"""
MemoGPT - 30M Mimari Geri Yukleme (8 Katman, 512 Boyut, ~29.5M Parametre)
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from dataclasses import dataclass
@dataclass
class MemoGPTConfig:
vocab_size: int = 8192
block_size: int = 512
n_layer: int = 8 # 30M icin 8 katman
n_head: int = 8 # 8 kafa
n_embd: int = 512 # 512 boyut
intermediate_size: int = 1376 # 30M orani
dropout: float = 0.1
bias: bool = False
norm_eps: float = 1e-5
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-5):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
rms = torch.sqrt(torch.mean(x ** 2, dim=-1, keepdim=True) + self.eps)
return x / rms * self.weight
def precompute_rope_frequencies(dim: int, max_seq_len: int, theta: float = 10000.0):
freqs = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim))
t = torch.arange(max_seq_len, dtype=torch.float)
freqs = torch.outer(t, freqs)
return torch.cos(freqs), torch.sin(freqs)
def apply_rotary_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
d = x.shape[-1]
x1 = x[..., :d // 2]
x2 = x[..., d // 2:]
cos = cos[:x1.shape[-2], :].unsqueeze(0).unsqueeze(0)
sin = sin[:x1.shape[-2], :].unsqueeze(0).unsqueeze(0)
out1 = x1 * cos - x2 * sin
out2 = x1 * sin + x2 * cos
return torch.cat([out1, out2], dim=-1)
class CausalSelfAttention(nn.Module):
def __init__(self, config: MemoGPTConfig):
super().__init__()
self.n_head = config.n_head
self.head_dim = config.n_embd // config.n_head
self.q_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
self.k_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
self.v_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
self.out_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
self.attn_dropout = nn.Dropout(config.dropout)
self.resid_dropout = nn.Dropout(config.dropout)
self.register_buffer(
"causal_mask",
torch.triu(torch.ones(config.block_size, config.block_size), diagonal=1).bool()
)
def forward(self, x: torch.Tensor, rope_cos: torch.Tensor, rope_sin: torch.Tensor) -> torch.Tensor:
B, T, C = x.shape
q = self.q_proj(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)
k = self.k_proj(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)
v = self.v_proj(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)
q = apply_rotary_emb(q, rope_cos, rope_sin)
k = apply_rotary_emb(k, rope_cos, rope_sin)
scale = 1.0 / math.sqrt(self.head_dim)
att = torch.matmul(q, k.transpose(-2, -1)) * scale
att = att.masked_fill(self.causal_mask[:T, :T], float('-inf'))
att = F.softmax(att, dim=-1)
att = self.attn_dropout(att)
out = torch.matmul(att, v)
out = out.transpose(1, 2).contiguous().view(B, T, C)
return self.resid_dropout(self.out_proj(out))
class SwiGLUFeedForward(nn.Module):
def __init__(self, config: MemoGPTConfig):
super().__init__()
self.gate_proj = nn.Linear(config.n_embd, config.intermediate_size, bias=config.bias)
self.up_proj = nn.Linear(config.n_embd, config.intermediate_size, bias=config.bias)
self.down_proj = nn.Linear(config.intermediate_size, config.n_embd, bias=config.bias)
self.dropout = nn.Dropout(config.dropout)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate = F.silu(self.gate_proj(x))
up = self.up_proj(x)
return self.dropout(self.down_proj(gate * up))
class TransformerBlock(nn.Module):
def __init__(self, config: MemoGPTConfig):
super().__init__()
self.attn_norm = RMSNorm(config.n_embd, eps=config.norm_eps)
self.attention = CausalSelfAttention(config)
self.ff_norm = RMSNorm(config.n_embd, eps=config.norm_eps)
self.feed_forward = SwiGLUFeedForward(config)
def forward(self, x: torch.Tensor, rope_cos: torch.Tensor, rope_sin: torch.Tensor) -> torch.Tensor:
x = x + self.attention(self.attn_norm(x), rope_cos, rope_sin)
x = x + self.feed_forward(self.ff_norm(x))
return x
class MemoGPT(nn.Module):
def __init__(self, config: MemoGPTConfig):
super().__init__()
self.config = config
self.token_embedding = nn.Embedding(config.vocab_size, config.n_embd)
self.layers = nn.ModuleList([TransformerBlock(config) for _ in range(config.n_layer)])
self.final_norm = RMSNorm(config.n_embd, eps=config.norm_eps)
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.token_embedding.weight = self.lm_head.weight
rope_cos, rope_sin = precompute_rope_frequencies(
dim=config.n_embd // config.n_head,
max_seq_len=config.block_size
)
self.register_buffer("rope_cos", rope_cos)
self.register_buffer("rope_sin", rope_sin)
self.apply(self._init_weights)
n_params = sum(p.numel() for p in self.parameters() if p.requires_grad)
print(f"MemoGPT basariyla olusturuldu! Toplam parametre: {n_params:,}")
def _init_weights(self, module):
if isinstance(module, nn.Linear):
nn.init.normal_(module.weight, mean=0.0, std=0.02)
if module.bias is not None:
nn.init.zeros_(module.bias)
elif isinstance(module, nn.Embedding):
nn.init.normal_(module.weight, mean=0.0, std=0.02)
def forward(self, idx: torch.Tensor, targets: torch.Tensor = None):
B, T = idx.shape
x = self.token_embedding(idx)
for layer in self.layers:
x = layer(x, self.rope_cos, self.rope_sin)
x = self.final_norm(x)
logits = self.lm_head(x)
loss = None
if targets is not None:
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
return logits, loss
@torch.no_grad()
def generate(self, idx: torch.Tensor, max_new_tokens: int, temperature: float = 0.7, top_k: int = 50, top_p: float = 0.9):
self.eval()
for _ in range(max_new_tokens):
idx_cond = idx[:, -self.config.block_size:]
logits, _ = self(idx_cond)
logits = logits[:, -1, :] / temperature
if top_k is not None and top_k > 0:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[:, [-1]]] = float('-inf')
probs = F.softmax(logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1)
idx = torch.cat([idx, next_token], dim=1)
return idx