import math import torch import torch.nn as nn import torch.nn.functional as F class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: variance = x.pow(2).mean(-1, keepdim=True) return x * torch.rsqrt(variance + self.eps) * self.weight def precompute_rope_freqs(head_dim: int, seq_len: int, theta: float = 10000.0, device="cpu"): freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim)) t = torch.arange(seq_len, device=device).float() freqs = torch.outer(t, freqs) return torch.cos(freqs), torch.sin(freqs) def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: # x: (B, num_heads, seq_len, head_dim) T = x.shape[2] cos = cos[:T].unsqueeze(0).unsqueeze(1) sin = sin[:T].unsqueeze(0).unsqueeze(1) x_real = x[..., 0::2] x_imag = x[..., 1::2] out_real = x_real * cos - x_imag * sin out_imag = x_real * sin + x_imag * cos return torch.stack([out_real, out_imag], dim=-1).flatten(-2) class QKVEGQAAttention(nn.Module): """ Attention z Value-State Gating (QKVE) i Grouped-Query Attention (GQA): V_mod = V * (1 + tanh(E)) num_heads = 12 (Query), num_kv_heads = 4 (Key, Value, Gate E) """ def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4): super().__init__() self.d_model = d_model self.num_heads = num_heads self.num_kv_heads = num_kv_heads self.head_dim = d_model // num_heads self.q_proj = nn.Linear(d_model, d_model, bias=False) self.k_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) self.v_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) self.e_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) # Bramka wartości dla KV self.out_proj = nn.Linear(d_model, d_model, bias=False) def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: B, T, C = x.shape q = self.q_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2) k = self.k_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2) v = self.v_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2) e = self.e_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2) q = apply_rope(q, cos, sin) k = apply_rope(k, cos, sin) # Modulacja wartości przez bramkę E: v_mod = v * (1.0 + torch.tanh(e)) # GQA: powielenie KV dla odpowiadających im głów Query if self.num_kv_heads != self.num_heads: repeats = self.num_heads // self.num_kv_heads k = k.repeat_interleave(repeats, dim=1) v_mod = v_mod.repeat_interleave(repeats, dim=1) # Wydajna atencja przyczynowa (PyTorch SDPA / FlashAttention) out = F.scaled_dot_product_attention(q, k, v_mod, is_causal=True) out = out.transpose(1, 2).contiguous().view(B, T, C) return self.out_proj(out) class SwiGLUFFN(nn.Module): def __init__(self, d_model: int, ffn_hidden: int): super().__init__() self.w_gate = nn.Linear(d_model, ffn_hidden, bias=False) self.w_up = nn.Linear(d_model, ffn_hidden, bias=False) self.w_down = nn.Linear(ffn_hidden, d_model, bias=False) def forward(self, x: torch.Tensor) -> torch.Tensor: return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x)) class QKVE30Block(nn.Module): """ Pojedynczy blok QKVE z GQA i 2 warstwami SwiGLU: 1 moduł uwagi QKVE-GQA + 2 warstwy SwiGLU FFN """ def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4, base_ffn: int = 512): super().__init__() self.attn_norm = RMSNorm(d_model) self.attn = QKVEGQAAttention(d_model, num_heads, num_kv_heads) # 2 warstwy SwiGLU z normami RMSNorm self.ffn_norms = nn.ModuleList([RMSNorm(d_model) for _ in range(2)]) self.ffn_layers = nn.ModuleList([ SwiGLUFFN(d_model, base_ffn), SwiGLUFFN(d_model, base_ffn) ]) def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: # Atencja z residual x = x + self.attn(self.attn_norm(x), cos, sin) # 2 warstwy SwiGLU z residual for norm, ffn in zip(self.ffn_norms, self.ffn_layers): x = x + ffn(norm(x)) return x class QKVE30Transformer(nn.Module): """ Model QKVE-61M-30block: 30 bloków transformera z Grouped-Query Attention (GQA) i Deep SwiGLU FFN (60 warstw FFN) Łącznie 61.03M parametrów. """ def __init__( self, vocab_size: int = 32128, d_model: int = 384, num_layers: int = 30, num_heads: int = 12, num_kv_heads: int = 4, base_ffn: int = 512, max_seq_len: int = 1024 ): super().__init__() self.vocab_size = vocab_size self.d_model = d_model self.max_seq_len = max_seq_len self.tok_embeddings = nn.Embedding(vocab_size, d_model) self.blocks = nn.ModuleList([ QKVE30Block(d_model, num_heads, num_kv_heads, base_ffn) for _ in range(num_layers) ]) self.norm = RMSNorm(d_model) # Wiązanie wag: lm_head współdzieli wagi z embeddingami self.lm_head = nn.Linear(d_model, vocab_size, bias=False) self.lm_head.weight = self.tok_embeddings.weight # Precompute RoPE frequencies cos, sin = precompute_rope_freqs(d_model // num_heads, max_seq_len) self.register_buffer("rope_cos", cos) self.register_buffer("rope_sin", sin) # Inicjalizacja wag self.apply(self._init_weights) def _init_weights(self, module): if isinstance(module, nn.Linear): torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) elif isinstance(module, nn.Embedding): torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) def forward(self, input_ids: torch.Tensor, targets: torch.Tensor = None): B, T = input_ids.shape x = self.tok_embeddings(input_ids) cos = self.rope_cos[:T] sin = self.rope_sin[:T] for block in self.blocks: x = block(x, cos, sin) x = self.norm(x) logits = self.lm_head(x) loss = None if targets is not None: loss = F.cross_entropy(logits.view(-1, self.vocab_size), targets.view(-1)) return logits, loss @torch.no_grad() def generate( self, input_ids: torch.Tensor, max_new_tokens: int = 100, temperature: float = 0.8, top_p: float = 0.9, eos_token_id: int = 2 ): self.eval() for _ in range(max_new_tokens): idx_cond = input_ids if input_ids.size(1) <= self.max_seq_len else input_ids[:, -self.max_seq_len:] logits, _ = self(idx_cond) logits = logits[:, -1, :] / max(temperature, 1e-5) if top_p < 1.0: sorted_logits, sorted_indices = torch.sort(logits, descending=True) cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) sorted_indices_to_remove = cumulative_probs > top_p sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() sorted_indices_to_remove[..., 0] = 0 indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) logits[indices_to_remove] = -float('Inf') probs = F.softmax(logits, dim=-1) next_token = torch.multinomial(probs, num_samples=1) input_ids = torch.cat((input_ids, next_token), dim=1) if eos_token_id is not None and (next_token == eos_token_id).all(): break return input_ids def count_parameters(model): total = sum(p.numel() for p in model.parameters()) trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) return total, trainable if __name__ == "__main__": model = QKVE30Transformer() total, trainable = count_parameters(model) print(f"✨ Model QKVE-61M-30block zainicjalizowany pomyślnie!") print(f" Liczba parametrów (total): {total:,} ({total/1e6:.2f}M)") print(f" Liczba parametrów (trainable): {trainable:,} ({trainable/1e6:.2f}M)")