QKVE-61M / model.py
Sticzu's picture
Initial release: QKVE-61M architecture, weights, ONNX and documentation
29d2351 verified
Raw History Blame Contribute Delete
8.68 kB
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
variance = x.pow(2).mean(-1, keepdim=True)
return x * torch.rsqrt(variance + self.eps) * self.weight
def precompute_rope_freqs(head_dim: int, seq_len: int, theta: float = 10000.0, device="cpu"):
freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim))
t = torch.arange(seq_len, device=device).float()
freqs = torch.outer(t, freqs)
return torch.cos(freqs), torch.sin(freqs)
def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
# x: (B, num_heads, seq_len, head_dim)
T = x.shape[2]
cos = cos[:T].unsqueeze(0).unsqueeze(1)
sin = sin[:T].unsqueeze(0).unsqueeze(1)
x_real = x[..., 0::2]
x_imag = x[..., 1::2]
out_real = x_real * cos - x_imag * sin
out_imag = x_real * sin + x_imag * cos
return torch.stack([out_real, out_imag], dim=-1).flatten(-2)
class QKVEGQAAttention(nn.Module):
"""
Attention z Value-State Gating (QKVE) i Grouped-Query Attention (GQA):
V_mod = V * (1 + tanh(E))
num_heads = 12 (Query), num_kv_heads = 4 (Key, Value, Gate E)
"""
def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4):
super().__init__()
self.d_model = d_model
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_dim = d_model // num_heads
self.q_proj = nn.Linear(d_model, d_model, bias=False)
self.k_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False)
self.v_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False)
self.e_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) # Bramka wartości dla KV
self.out_proj = nn.Linear(d_model, d_model, bias=False)
def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
B, T, C = x.shape
q = self.q_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
k = self.k_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)
v = self.v_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)
e = self.e_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)
q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
# Modulacja wartości przez bramkę E:
v_mod = v * (1.0 + torch.tanh(e))
# GQA: powielenie KV dla odpowiadających im głów Query
if self.num_kv_heads != self.num_heads:
repeats = self.num_heads // self.num_kv_heads
k = k.repeat_interleave(repeats, dim=1)
v_mod = v_mod.repeat_interleave(repeats, dim=1)
# Wydajna atencja przyczynowa (PyTorch SDPA / FlashAttention)
out = F.scaled_dot_product_attention(q, k, v_mod, is_causal=True)
out = out.transpose(1, 2).contiguous().view(B, T, C)
return self.out_proj(out)
class SwiGLUFFN(nn.Module):
def __init__(self, d_model: int, ffn_hidden: int):
super().__init__()
self.w_gate = nn.Linear(d_model, ffn_hidden, bias=False)
self.w_up = nn.Linear(d_model, ffn_hidden, bias=False)
self.w_down = nn.Linear(ffn_hidden, d_model, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x))
class QKVE30Block(nn.Module):
"""
Pojedynczy blok QKVE z GQA i 2 warstwami SwiGLU:
1 moduł uwagi QKVE-GQA + 2 warstwy SwiGLU FFN
"""
def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4, base_ffn: int = 512):
super().__init__()
self.attn_norm = RMSNorm(d_model)
self.attn = QKVEGQAAttention(d_model, num_heads, num_kv_heads)
# 2 warstwy SwiGLU z normami RMSNorm
self.ffn_norms = nn.ModuleList([RMSNorm(d_model) for _ in range(2)])
self.ffn_layers = nn.ModuleList([
SwiGLUFFN(d_model, base_ffn),
SwiGLUFFN(d_model, base_ffn)
])
def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
# Atencja z residual
x = x + self.attn(self.attn_norm(x), cos, sin)
# 2 warstwy SwiGLU z residual
for norm, ffn in zip(self.ffn_norms, self.ffn_layers):
x = x + ffn(norm(x))
return x
class QKVE30Transformer(nn.Module):
"""
Model QKVE-61M-30block:
30 bloków transformera z Grouped-Query Attention (GQA) i Deep SwiGLU FFN (60 warstw FFN)
Łącznie 61.03M parametrów.
"""
def __init__(
self,
vocab_size: int = 32128,
d_model: int = 384,
num_layers: int = 30,
num_heads: int = 12,
num_kv_heads: int = 4,
base_ffn: int = 512,
max_seq_len: int = 1024
):
super().__init__()
self.vocab_size = vocab_size
self.d_model = d_model
self.max_seq_len = max_seq_len
self.tok_embeddings = nn.Embedding(vocab_size, d_model)
self.blocks = nn.ModuleList([
QKVE30Block(d_model, num_heads, num_kv_heads, base_ffn) for _ in range(num_layers)
])
self.norm = RMSNorm(d_model)
# Wiązanie wag: lm_head współdzieli wagi z embeddingami
self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
self.lm_head.weight = self.tok_embeddings.weight
# Precompute RoPE frequencies
cos, sin = precompute_rope_freqs(d_model // num_heads, max_seq_len)
self.register_buffer("rope_cos", cos)
self.register_buffer("rope_sin", sin)
# Inicjalizacja wag
self.apply(self._init_weights)
def _init_weights(self, module):
if isinstance(module, nn.Linear):
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
elif isinstance(module, nn.Embedding):
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
def forward(self, input_ids: torch.Tensor, targets: torch.Tensor = None):
B, T = input_ids.shape
x = self.tok_embeddings(input_ids)
cos = self.rope_cos[:T]
sin = self.rope_sin[:T]
for block in self.blocks:
x = block(x, cos, sin)
x = self.norm(x)
logits = self.lm_head(x)
loss = None
if targets is not None:
loss = F.cross_entropy(logits.view(-1, self.vocab_size), targets.view(-1))
return logits, loss
@torch.no_grad()
def generate(
self,
input_ids: torch.Tensor,
max_new_tokens: int = 100,
temperature: float = 0.8,
top_p: float = 0.9,
eos_token_id: int = 2
):
self.eval()
for _ in range(max_new_tokens):
idx_cond = input_ids if input_ids.size(1) <= self.max_seq_len else input_ids[:, -self.max_seq_len:]
logits, _ = self(idx_cond)
logits = logits[:, -1, :] / max(temperature, 1e-5)
if top_p < 1.0:
sorted_logits, sorted_indices = torch.sort(logits, descending=True)
cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
sorted_indices_to_remove = cumulative_probs > top_p
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
sorted_indices_to_remove[..., 0] = 0
indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
logits[indices_to_remove] = -float('Inf')
probs = F.softmax(logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1)
input_ids = torch.cat((input_ids, next_token), dim=1)
if eos_token_id is not None and (next_token == eos_token_id).all():
break
return input_ids
def count_parameters(model):
total = sum(p.numel() for p in model.parameters())
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
return total, trainable
if __name__ == "__main__":
model = QKVE30Transformer()
total, trainable = count_parameters(model)
print(f"✨ Model QKVE-61M-30block zainicjalizowany pomyślnie!")
print(f" Liczba parametrów (total): {total:,} ({total/1e6:.2f}M)")
print(f" Liczba parametrów (trainable): {trainable:,} ({trainable/1e6:.2f}M)")