Download model.py from Sticzu/QKVE-61M: direct link, hf CLI and curl.
- Browser
- Download file 8.68 kB
-
https://huggingface.co/Sticzu/QKVE-61M/resolve/main/model.py
- Command line
-
hf download hf://Sticzu/QKVE-61M/model.py
-
curl -L -o model.py https://huggingface.co/Sticzu/QKVE-61M/resolve/main/model.py
8.68 kB
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim: int, eps: float = 1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| variance = x.pow(2).mean(-1, keepdim=True) | |
| return x * torch.rsqrt(variance + self.eps) * self.weight | |
| def precompute_rope_freqs(head_dim: int, seq_len: int, theta: float = 10000.0, device="cpu"): | |
| freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim)) | |
| t = torch.arange(seq_len, device=device).float() | |
| freqs = torch.outer(t, freqs) | |
| return torch.cos(freqs), torch.sin(freqs) | |
| def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| # x: (B, num_heads, seq_len, head_dim) | |
| T = x.shape[2] | |
| cos = cos[:T].unsqueeze(0).unsqueeze(1) | |
| sin = sin[:T].unsqueeze(0).unsqueeze(1) | |
| x_real = x[..., 0::2] | |
| x_imag = x[..., 1::2] | |
| out_real = x_real * cos - x_imag * sin | |
| out_imag = x_real * sin + x_imag * cos | |
| return torch.stack([out_real, out_imag], dim=-1).flatten(-2) | |
| class QKVEGQAAttention(nn.Module): | |
| """ | |
| Attention z Value-State Gating (QKVE) i Grouped-Query Attention (GQA): | |
| V_mod = V * (1 + tanh(E)) | |
| num_heads = 12 (Query), num_kv_heads = 4 (Key, Value, Gate E) | |
| """ | |
| def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.num_heads = num_heads | |
| self.num_kv_heads = num_kv_heads | |
| self.head_dim = d_model // num_heads | |
| self.q_proj = nn.Linear(d_model, d_model, bias=False) | |
| self.k_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) | |
| self.v_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) | |
| self.e_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) # Bramka wartości dla KV | |
| self.out_proj = nn.Linear(d_model, d_model, bias=False) | |
| def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| B, T, C = x.shape | |
| q = self.q_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2) | |
| k = self.k_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2) | |
| v = self.v_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2) | |
| e = self.e_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2) | |
| q = apply_rope(q, cos, sin) | |
| k = apply_rope(k, cos, sin) | |
| # Modulacja wartości przez bramkę E: | |
| v_mod = v * (1.0 + torch.tanh(e)) | |
| # GQA: powielenie KV dla odpowiadających im głów Query | |
| if self.num_kv_heads != self.num_heads: | |
| repeats = self.num_heads // self.num_kv_heads | |
| k = k.repeat_interleave(repeats, dim=1) | |
| v_mod = v_mod.repeat_interleave(repeats, dim=1) | |
| # Wydajna atencja przyczynowa (PyTorch SDPA / FlashAttention) | |
| out = F.scaled_dot_product_attention(q, k, v_mod, is_causal=True) | |
| out = out.transpose(1, 2).contiguous().view(B, T, C) | |
| return self.out_proj(out) | |
| class SwiGLUFFN(nn.Module): | |
| def __init__(self, d_model: int, ffn_hidden: int): | |
| super().__init__() | |
| self.w_gate = nn.Linear(d_model, ffn_hidden, bias=False) | |
| self.w_up = nn.Linear(d_model, ffn_hidden, bias=False) | |
| self.w_down = nn.Linear(ffn_hidden, d_model, bias=False) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x)) | |
| class QKVE30Block(nn.Module): | |
| """ | |
| Pojedynczy blok QKVE z GQA i 2 warstwami SwiGLU: | |
| 1 moduł uwagi QKVE-GQA + 2 warstwy SwiGLU FFN | |
| """ | |
| def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4, base_ffn: int = 512): | |
| super().__init__() | |
| self.attn_norm = RMSNorm(d_model) | |
| self.attn = QKVEGQAAttention(d_model, num_heads, num_kv_heads) | |
| # 2 warstwy SwiGLU z normami RMSNorm | |
| self.ffn_norms = nn.ModuleList([RMSNorm(d_model) for _ in range(2)]) | |
| self.ffn_layers = nn.ModuleList([ | |
| SwiGLUFFN(d_model, base_ffn), | |
| SwiGLUFFN(d_model, base_ffn) | |
| ]) | |
| def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| # Atencja z residual | |
| x = x + self.attn(self.attn_norm(x), cos, sin) | |
| # 2 warstwy SwiGLU z residual | |
| for norm, ffn in zip(self.ffn_norms, self.ffn_layers): | |
| x = x + ffn(norm(x)) | |
| return x | |
| class QKVE30Transformer(nn.Module): | |
| """ | |
| Model QKVE-61M-30block: | |
| 30 bloków transformera z Grouped-Query Attention (GQA) i Deep SwiGLU FFN (60 warstw FFN) | |
| Łącznie 61.03M parametrów. | |
| """ | |
| def __init__( | |
| self, | |
| vocab_size: int = 32128, | |
| d_model: int = 384, | |
| num_layers: int = 30, | |
| num_heads: int = 12, | |
| num_kv_heads: int = 4, | |
| base_ffn: int = 512, | |
| max_seq_len: int = 1024 | |
| ): | |
| super().__init__() | |
| self.vocab_size = vocab_size | |
| self.d_model = d_model | |
| self.max_seq_len = max_seq_len | |
| self.tok_embeddings = nn.Embedding(vocab_size, d_model) | |
| self.blocks = nn.ModuleList([ | |
| QKVE30Block(d_model, num_heads, num_kv_heads, base_ffn) for _ in range(num_layers) | |
| ]) | |
| self.norm = RMSNorm(d_model) | |
| # Wiązanie wag: lm_head współdzieli wagi z embeddingami | |
| self.lm_head = nn.Linear(d_model, vocab_size, bias=False) | |
| self.lm_head.weight = self.tok_embeddings.weight | |
| # Precompute RoPE frequencies | |
| cos, sin = precompute_rope_freqs(d_model // num_heads, max_seq_len) | |
| self.register_buffer("rope_cos", cos) | |
| self.register_buffer("rope_sin", sin) | |
| # Inicjalizacja wag | |
| self.apply(self._init_weights) | |
| def _init_weights(self, module): | |
| if isinstance(module, nn.Linear): | |
| torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) | |
| elif isinstance(module, nn.Embedding): | |
| torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) | |
| def forward(self, input_ids: torch.Tensor, targets: torch.Tensor = None): | |
| B, T = input_ids.shape | |
| x = self.tok_embeddings(input_ids) | |
| cos = self.rope_cos[:T] | |
| sin = self.rope_sin[:T] | |
| for block in self.blocks: | |
| x = block(x, cos, sin) | |
| x = self.norm(x) | |
| logits = self.lm_head(x) | |
| loss = None | |
| if targets is not None: | |
| loss = F.cross_entropy(logits.view(-1, self.vocab_size), targets.view(-1)) | |
| return logits, loss | |
| def generate( | |
| self, | |
| input_ids: torch.Tensor, | |
| max_new_tokens: int = 100, | |
| temperature: float = 0.8, | |
| top_p: float = 0.9, | |
| eos_token_id: int = 2 | |
| ): | |
| self.eval() | |
| for _ in range(max_new_tokens): | |
| idx_cond = input_ids if input_ids.size(1) <= self.max_seq_len else input_ids[:, -self.max_seq_len:] | |
| logits, _ = self(idx_cond) | |
| logits = logits[:, -1, :] / max(temperature, 1e-5) | |
| if top_p < 1.0: | |
| sorted_logits, sorted_indices = torch.sort(logits, descending=True) | |
| cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) | |
| sorted_indices_to_remove = cumulative_probs > top_p | |
| sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() | |
| sorted_indices_to_remove[..., 0] = 0 | |
| indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) | |
| logits[indices_to_remove] = -float('Inf') | |
| probs = F.softmax(logits, dim=-1) | |
| next_token = torch.multinomial(probs, num_samples=1) | |
| input_ids = torch.cat((input_ids, next_token), dim=1) | |
| if eos_token_id is not None and (next_token == eos_token_id).all(): | |
| break | |
| return input_ids | |
| def count_parameters(model): | |
| total = sum(p.numel() for p in model.parameters()) | |
| trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) | |
| return total, trainable | |
| if __name__ == "__main__": | |
| model = QKVE30Transformer() | |
| total, trainable = count_parameters(model) | |
| print(f"✨ Model QKVE-61M-30block zainicjalizowany pomyślnie!") | |
| print(f" Liczba parametrów (total): {total:,} ({total/1e6:.2f}M)") | |
| print(f" Liczba parametrów (trainable): {trainable:,} ({trainable/1e6:.2f}M)") | |