""" MemoGPT - 30M Mimari Geri Yukleme (8 Katman, 512 Boyut, ~29.5M Parametre) """ import math import torch import torch.nn as nn import torch.nn.functional as F from dataclasses import dataclass @dataclass class MemoGPTConfig: vocab_size: int = 8192 block_size: int = 512 n_layer: int = 8 # 30M icin 8 katman n_head: int = 8 # 8 kafa n_embd: int = 512 # 512 boyut intermediate_size: int = 1376 # 30M orani dropout: float = 0.1 bias: bool = False norm_eps: float = 1e-5 class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-5): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: rms = torch.sqrt(torch.mean(x ** 2, dim=-1, keepdim=True) + self.eps) return x / rms * self.weight def precompute_rope_frequencies(dim: int, max_seq_len: int, theta: float = 10000.0): freqs = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim)) t = torch.arange(max_seq_len, dtype=torch.float) freqs = torch.outer(t, freqs) return torch.cos(freqs), torch.sin(freqs) def apply_rotary_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: d = x.shape[-1] x1 = x[..., :d // 2] x2 = x[..., d // 2:] cos = cos[:x1.shape[-2], :].unsqueeze(0).unsqueeze(0) sin = sin[:x1.shape[-2], :].unsqueeze(0).unsqueeze(0) out1 = x1 * cos - x2 * sin out2 = x1 * sin + x2 * cos return torch.cat([out1, out2], dim=-1) class CausalSelfAttention(nn.Module): def __init__(self, config: MemoGPTConfig): super().__init__() self.n_head = config.n_head self.head_dim = config.n_embd // config.n_head self.q_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias) self.k_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias) self.v_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias) self.out_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias) self.attn_dropout = nn.Dropout(config.dropout) self.resid_dropout = nn.Dropout(config.dropout) self.register_buffer( "causal_mask", torch.triu(torch.ones(config.block_size, config.block_size), diagonal=1).bool() ) def forward(self, x: torch.Tensor, rope_cos: torch.Tensor, rope_sin: torch.Tensor) -> torch.Tensor: B, T, C = x.shape q = self.q_proj(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2) k = self.k_proj(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2) v = self.v_proj(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2) q = apply_rotary_emb(q, rope_cos, rope_sin) k = apply_rotary_emb(k, rope_cos, rope_sin) scale = 1.0 / math.sqrt(self.head_dim) att = torch.matmul(q, k.transpose(-2, -1)) * scale att = att.masked_fill(self.causal_mask[:T, :T], float('-inf')) att = F.softmax(att, dim=-1) att = self.attn_dropout(att) out = torch.matmul(att, v) out = out.transpose(1, 2).contiguous().view(B, T, C) return self.resid_dropout(self.out_proj(out)) class SwiGLUFeedForward(nn.Module): def __init__(self, config: MemoGPTConfig): super().__init__() self.gate_proj = nn.Linear(config.n_embd, config.intermediate_size, bias=config.bias) self.up_proj = nn.Linear(config.n_embd, config.intermediate_size, bias=config.bias) self.down_proj = nn.Linear(config.intermediate_size, config.n_embd, bias=config.bias) self.dropout = nn.Dropout(config.dropout) def forward(self, x: torch.Tensor) -> torch.Tensor: gate = F.silu(self.gate_proj(x)) up = self.up_proj(x) return self.dropout(self.down_proj(gate * up)) class TransformerBlock(nn.Module): def __init__(self, config: MemoGPTConfig): super().__init__() self.attn_norm = RMSNorm(config.n_embd, eps=config.norm_eps) self.attention = CausalSelfAttention(config) self.ff_norm = RMSNorm(config.n_embd, eps=config.norm_eps) self.feed_forward = SwiGLUFeedForward(config) def forward(self, x: torch.Tensor, rope_cos: torch.Tensor, rope_sin: torch.Tensor) -> torch.Tensor: x = x + self.attention(self.attn_norm(x), rope_cos, rope_sin) x = x + self.feed_forward(self.ff_norm(x)) return x class MemoGPT(nn.Module): def __init__(self, config: MemoGPTConfig): super().__init__() self.config = config self.token_embedding = nn.Embedding(config.vocab_size, config.n_embd) self.layers = nn.ModuleList([TransformerBlock(config) for _ in range(config.n_layer)]) self.final_norm = RMSNorm(config.n_embd, eps=config.norm_eps) self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) self.token_embedding.weight = self.lm_head.weight rope_cos, rope_sin = precompute_rope_frequencies( dim=config.n_embd // config.n_head, max_seq_len=config.block_size ) self.register_buffer("rope_cos", rope_cos) self.register_buffer("rope_sin", rope_sin) self.apply(self._init_weights) n_params = sum(p.numel() for p in self.parameters() if p.requires_grad) print(f"MemoGPT basariyla olusturuldu! Toplam parametre: {n_params:,}") def _init_weights(self, module): if isinstance(module, nn.Linear): nn.init.normal_(module.weight, mean=0.0, std=0.02) if module.bias is not None: nn.init.zeros_(module.bias) elif isinstance(module, nn.Embedding): nn.init.normal_(module.weight, mean=0.0, std=0.02) def forward(self, idx: torch.Tensor, targets: torch.Tensor = None): B, T = idx.shape x = self.token_embedding(idx) for layer in self.layers: x = layer(x, self.rope_cos, self.rope_sin) x = self.final_norm(x) logits = self.lm_head(x) loss = None if targets is not None: loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1) return logits, loss @torch.no_grad() def generate(self, idx: torch.Tensor, max_new_tokens: int, temperature: float = 0.7, top_k: int = 50, top_p: float = 0.9): self.eval() for _ in range(max_new_tokens): idx_cond = idx[:, -self.config.block_size:] logits, _ = self(idx_cond) logits = logits[:, -1, :] / temperature if top_k is not None and top_k > 0: v, _ = torch.topk(logits, min(top_k, logits.size(-1))) logits[logits < v[:, [-1]]] = float('-inf') probs = F.softmax(logits, dim=-1) next_token = torch.multinomial(probs, num_samples=1) idx = torch.cat([idx, next_token], dim=1) return idx