Delete gpt2

gpt2/JiRack_H12_L12_V50257_D768_MSL8192_FF768x4.py

@@ -1,320 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-from pathlib import Path
-
-# ========================================
-# Model Configuration (GPT-2 Base Style)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 12
-NUM_LAYERS = 12
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS # 768 / 12 = 64
-
-# ИСПРАВЛЕНИЕ: Проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        # УДАЛЕНА Python-проверка для JIT-совместимости, если требуется
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш
-        seqlen_k_new = k.size(2)
-        pos_offset = 0
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2)
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. Маскирование
-        if T == seqlen_k_new:
-            causal_mask_present = torch.tensor(seqlen_k > 0, dtype=torch.bool, device=x.device)
-            if causal_mask_present.item():
-                mask = torch.full((T, seqlen_k), float("-inf"), device=x.device, dtype=attn.dtype)
-                current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-                
-                mask[:, :pos_offset] = 0.0
-                mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-                
-                attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (L=12, H=12)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # 🎯 ИСПРАВЛЕНИЕ: Инициализация кеша (если T > 1 ИЛИ кеш передан)
-        if T > 1 or past_kv is not None:
-            new_kv_cache = []
-        else:
-            new_kv_cache = None
-            
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                # Первый проход: обрабатываем весь вход
-                input_for_model = current_ids
-            else:
-                # Инкрементальные проходы: обрабатываем только последний сгенерированный токен
-                input_for_model = current_ids[:, -1].unsqueeze(-1) # 🚀 ВОССТАНОВЛЕНА ОБОРОРВАННАЯ ЧАСТЬ
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :] # Логиты только для последнего токена
-
-            # Применение температуры
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Применение Repetition Penalty
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                
-                # Сдвигаем назад, чтобы сохранить первый токен, превысивший top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование или жадный выбор
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~200.75M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        
-        # КРИТИЧЕСКАЯ ПРОВЕРКА: kv_cache_50 не должен быть None
-        assert kv_cache_50 is not None and len(kv_cache_50) == NUM_LAYERS
-        
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)} (Head Dim: {kv_cache_50[0][0].size(3)})")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_H12_L12_V50257_D768_MSL8192_FF768x4.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H12_L18_V50257_D768_MSL8192_FF768x4.py

@@ -1,318 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (GPT-2 Medium Style)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 12
-NUM_LAYERS = 18       # Increased depth
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            # Проверка на выход за пределы MAX_SEQ_LEN
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T x seqlen_k
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style): Input -> Attn(Norm(Input)) + Input -> FFN(Norm(Result)) + Result
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (18 слоев)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети (TFixup/ReZero style)
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # ИСПРАВЛЕНИЕ: Инициализируем новый кеш
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        # ИСПРАВЛЕНИЕ: Инициализируем KV cache как список None кортежей
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~167.33M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)}")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L18_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H12_L24_V50257_D768_MSL8192_FF768x4.py

@@ -1,317 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (GPT-2 Large Style)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 12
-NUM_LAYERS = 24       # Increased depth (GPT-2 Large equivalent)
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T x seqlen_k
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (24 слоя)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети (TFixup/ReZero style)
-            # Стандартное отклонение уменьшается с ростом числа слоев
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~221.75M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)}")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L24_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H12_L32_V50257_D768_MSL8192_FF768x4.py

@@ -1,317 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (GPT-2 XL Style)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 12
-NUM_LAYERS = 32       # Increased depth (GPT-2 XL equivalent)
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T x seqlen_k
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (32 слоя)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети (TFixup/ReZero style). 
-            # Critical for NUM_LAYERS = 32
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~295.1M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)}")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L32_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H12_L6_V50257_D768_MSL8192_FF768x4.py

@@ -1,316 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (GPT-2 Small Style)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 12
-NUM_LAYERS = 6        # Back to 6 layers
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T x seqlen_k
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (6 слоев)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~124.44M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)}")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L6_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H16_L24_V50257_D768_MSL8192_FF768x4.py

@@ -1,318 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (L=24, H=16)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 16        # Changed to 16
-NUM_LAYERS = 24       # Changed to 24
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS # Recalculated: 768 / 16 = 48
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        # NOTE: D is MODEL_DIM
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T (query length) x seqlen_k (key length)
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (L=24, H=16)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~221.75M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        # Проверяем, что размерность головы HEAD_DIM (48) соблюдена
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)} (Head Dim: {kv_cache_50[0][0].size(3)})")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L24_H16_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H16_L32_V50257_D768_MSL8192_FF768x4.py

@@ -1,318 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (L=32, H=16, D=768)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 16
-NUM_LAYERS = 32       # Deepest version yet
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS # 768 / 16 = 48
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T (query length) x seqlen_k (key length)
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (L=32, H=16)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети.
-            # Critical for NUM_LAYERS = 32
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~295.1M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        # Проверяем, что размерность головы HEAD_DIM (48) соблюдена
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)} (Head Dim: {kv_cache_50[0][0].size(3)})")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L32_H16_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H6_L6_V50257_D768_MSL8192_FF768x4.py

@@ -1,317 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (L=6, H=6, D=768)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 6         # Changed to 6
-NUM_LAYERS = 6        # Set to 6 layers
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS # 768 / 6 = 128
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T (query length) x seqlen_k (key length)
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (L=6, H=6)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети.
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~124.44M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        # Проверяем, что размерность головы HEAD_DIM (128) соблюдена
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)} (Head Dim: {kv_cache_50[0][0].size(3)})")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L6_H6_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H8_L6_V50257_D768_MSL8192_FF768x4.py

@@ -1,317 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (L=6, H=8, D=768)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 8         # Set to 8
-NUM_LAYERS = 6
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS # Recalculated: 768 / 8 = 96
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T (query length) x seqlen_k (key length)
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (L=6, H=8)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети.
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~124.44M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        # Проверяем, что размерность головы HEAD_DIM (96) соблюдена
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)} (Head Dim: {kv_cache_50[0][0].size(3)})")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L6_H8_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")

gpt2/JiRack_H8_L8_V50257_D768_MSL8192_FF768x4.py

@@ -1,317 +0,0 @@
-# Copyright (c) 2025 CMS Manhattan
-# All rights reserved.
-# Author: Konstantin Vladimirovich Grabko
-# Email: grabko@cmsmanhattan.com
-# Phone: +1(516)777-0945
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-# Additional terms:
-# Any commercial use or distribution of this software or derivative works
-# requires explicit written permission from the copyright holder.
-
-import os
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from typing import Optional, Tuple, List
-import math
-
-# ========================================
-# Model Configuration (L=8, H=8, D=768)
-# ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 768
-NUM_HEADS = 8
-NUM_LAYERS = 8        # Set to 8 layers
-MAX_SEQ_LEN = 8192
-FFN_HIDDEN_DIM = 4 * MODEL_DIM
-HEAD_DIM = MODEL_DIM // NUM_HEADS # 768 / 8 = 96
-
-# ИСПРАВЛЕНИЕ: ROCm/HIP-совместимая проверка устройства
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-elif hasattr(torch, 'hip') and torch.hip.is_available():
-    device = torch.device("cuda") 
-else:
-    device = torch.device("cpu")
-
-# -------------------------------
-# Learned Positional Embedding
-# -------------------------------
-class LearnedPositionalEmbedding(nn.Module):
-    def __init__(self, max_seq_len: int, embed_dim: int):
-        super().__init__()
-        self.pos_emb = nn.Parameter(torch.zeros(max_seq_len, embed_dim))
-
-    def forward(self, x: torch.Tensor, pos_offset: int = 0) -> torch.Tensor:
-        seq_len = x.size(1)
-        if pos_offset + seq_len > self.pos_emb.size(0):
-            raise ValueError("Sequence length exceeds MAX_SEQ_LEN defined in position embedding.")
-        
-        pos = self.pos_emb[pos_offset : pos_offset + seq_len]
-        return x + pos.unsqueeze(0)
-
-
-# -------------------------------
-# MultiHeadAttention (MHA)
-# -------------------------------
-class MultiHeadAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.q_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.k_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.v_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.out_proj = nn.Linear(MODEL_DIM, MODEL_DIM, bias=False)
-        self.scale = HEAD_DIM ** -0.5
-
-    def forward(self, x: torch.Tensor, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        B, T, D = x.shape
-        
-        q = self.q_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        k = self.k_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-        v = self.v_proj(x).view(B, T, NUM_HEADS, HEAD_DIM).transpose(1, 2)
-
-        # 1. KV-кеш и определение смещения
-        pos_offset = 0
-        seqlen_k_new = k.size(2)
-        if past_kv is not None:
-            past_k, past_v = past_kv
-            k = torch.cat([past_k, k], dim=2)
-            v = torch.cat([past_v, v], dim=2)
-            pos_offset = past_k.size(2)
-        
-        seqlen_k = k.size(2) # Общая длина K
-        new_kv = (k, v)
-
-        # 2. Расчет внимания
-        attn = torch.matmul(q, k.transpose(-2, -1)) * self.scale
-
-        # 3. КРИТИЧЕСКОЕ ИСПРАВЛЕНИЕ МАСКИРОВАНИЯ (Causal Mask)
-        if T == seqlen_k_new and seqlen_k > 0:
-            # Создаем маску T (query length) x seqlen_k (key length)
-            mask = torch.full((T, seqlen_k), 
-                              float("-inf"), 
-                              device=x.device, 
-                              dtype=attn.dtype)
-            
-            # Разрешаем всем новым токенам видеть все старые токены (past_kv)
-            mask[:, :pos_offset] = 0.0
-
-            # Применяем треугольную маску для текущих T токенов
-            current_causal_mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
-            mask[:, pos_offset : pos_offset + T].masked_fill_(~current_causal_mask, float('-inf'))
-            
-            # Применяем маску к весам внимания
-            attn = attn + mask[None, None, :, :]
-
-        # 4. Выход
-        attn = F.softmax(attn, dim=-1)
-        out = torch.matmul(attn, v)
-        out = out.transpose(1, 2).contiguous().view(B, T, D)
-        out = self.out_proj(out)
-
-        return out, new_kv
-
-
-# -------------------------------
-# FeedForward (GELU, GPT-style)
-# -------------------------------
-class FeedForward(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.c_fc = nn.Linear(MODEL_DIM, FFN_HIDDEN_DIM, bias=False)
-        self.c_proj = nn.Linear(FFN_HIDDEN_DIM, MODEL_DIM, bias=False)
-
-    def forward(self, x):
-        return self.c_proj(F.gelu(self.c_fc(x), approximate='tanh'))
-
-
-# -------------------------------
-# Transformer Block (Post-Norm, GPT-style)
-# -------------------------------
-class TransformerBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = MultiHeadAttention()
-        self.ffn = FeedForward()
-        # ИСПРАВЛЕНИЕ: Добавлен стандартный eps
-        self.norm1 = nn.LayerNorm(MODEL_DIM, eps=1e-5) 
-        self.norm2 = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-
-    def forward(self, x, past_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
-        # Post-Normalization (GPT Style)
-        attn_out, new_kv = self.attn(self.norm1(x), past_kv)
-        x = x + attn_out
-        x = x + self.ffn(self.norm2(x))
-        return x, new_kv
-
-
-# -------------------------------
-# Главная модель GPTPyTorch (L=8, H=8)
-# -------------------------------
-class GPTPyTorch(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.token_emb = nn.Embedding(VOCAB_SIZE, MODEL_DIM)
-        self.pos_emb = LearnedPositionalEmbedding(MAX_SEQ_LEN, MODEL_DIM)
-        self.blocks = nn.ModuleList([TransformerBlock() for _ in range(NUM_LAYERS)])
-        self.ln_f = nn.LayerNorm(MODEL_DIM, eps=1e-5)
-        self.lm_head = nn.Linear(MODEL_DIM, VOCAB_SIZE, bias=False)
-        
-        signature = "Konstantin V Gbabko .  original author © 2025"
-        bytes_tensor = torch.tensor([ord(c) for c in signature], dtype=torch.uint8)
-        self.register_buffer("konstantin_gbabko_proof_of_authorship", bytes_tensor)
-        self.register_buffer("konstantin_gbabko_birth_date", torch.tensor([20251126], dtype=torch.int64))
-
-        self.lm_head.weight = self.token_emb.weight
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            # ИСПРАВЛЕНИЕ: Инициализация, масштабированная по глубине сети.
-            std = 0.02 / math.sqrt(2 * NUM_LAYERS) if isinstance(module, nn.Linear) and module.out_features == MODEL_DIM else 0.02
-            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.zeros_(module.bias)
-            nn.init.ones_(module.weight)
-
-    def forward(self, input_ids, past_kv: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None):
-        B, T = input_ids.shape
-        x = self.token_emb(input_ids)
-
-        pos_offset = 0
-        if past_kv is not None and past_kv[0] is not None:
-            pos_offset = past_kv[0][0].size(2)
-            
-        x = self.pos_emb(x, pos_offset=pos_offset)
-
-        # Инициализация нового кеша
-        new_kv_cache = [] if past_kv is not None or T > 1 else None
-        current_past = past_kv
-
-        for i, block in enumerate(self.blocks):
-            layer_past = current_past[i] if (current_past and i < len(current_past)) else None
-            x, layer_kv = block(x, layer_past)
-            
-            if new_kv_cache is not None:
-                new_kv_cache.append(layer_kv)
-
-        x = self.ln_f(x)
-        logits = self.lm_head(x)
-        return logits, new_kv_cache
-
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor,
-        max_new_tokens: int = 100,
-        temperature: float = 0.8,
-        top_p: float = 0.95,
-        repetition_penalty: float = 1.0,
-        do_sample: bool = True,
-        eos_token_id: int = 50256
-    ) -> torch.Tensor:
-        kv_cache = [None] * NUM_LAYERS
-        current_ids = input_ids.clone()
-
-        for step in range(max_new_tokens):
-            if step == 0:
-                input_for_model = current_ids
-            else:
-                input_for_model = current_ids[:, -1].unsqueeze(-1)
-
-            logits, kv_cache = self(input_for_model, kv_cache)
-            next_token_logits = logits[:, -1, :]
-
-            if temperature > 0:
-                next_token_logits = next_token_logits / temperature
-
-            # Repetition Penalty (логика сохранена)
-            if repetition_penalty != 1.0:
-                for i in range(current_ids.shape[0]):
-                    unique_tokens = torch.unique(current_ids[i]).tolist()
-                    for token_id in unique_tokens:
-                        score = next_token_logits[i, token_id]
-                        if score < 0:
-                            next_token_logits[i, token_id] = score * repetition_penalty
-                        else:
-                            next_token_logits[i, token_id] = score / repetition_penalty
-
-            # Top-P сэмплирование (логика сохранена)
-            if do_sample and top_p < 1.0:
-                sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
-                cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
-                sorted_indices_to_remove = cumulative_probs > top_p
-                sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone()
-                sorted_indices_to_remove[:, 0] = False
-                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-                next_token_logits = next_token_logits.masked_fill(indices_to_remove, float('-inf'))
-
-            # Сэмплирование
-            if do_sample and temperature > 0:
-                probs = torch.softmax(next_token_logits, dim=-1)
-                if torch.isnan(probs).any() or torch.isinf(probs).any():
-                    next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-                else:
-                    next_token = torch.multinomial(probs, num_samples=1)
-            else:
-                next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
-
-            if next_token.item() == eos_token_id:
-                break
-
-            current_ids = torch.cat([current_ids, next_token], dim=1)
-
-        return current_ids
-
-
-if __name__ == "__main__":
-    os.makedirs("models", exist_ok=True)
-
-    model = GPTPyTorch().to(device)
-    model.eval()
-
-    total_params = sum(p.numel() for p in model.parameters())
-    print(f"Device: {device}")
-    print(f"Total parameters: {total_params / 1e6:.2f}M") # ~157.14M
-
-    # 1. Проверка первого прохода (T=50)
-    input_ids_T50 = torch.randint(0, VOCAB_SIZE, (1, 50), device=device)
-    with torch.no_grad():
-        logits_50, kv_cache_50 = model(input_ids_T50)
-        expected_k_shape = (1, NUM_HEADS, 50, HEAD_DIM)
-        # Проверяем, что размерность головы HEAD_DIM (96) соблюдена
-        assert kv_cache_50[0][0].shape == expected_k_shape
-        print(f"Initial logits shape: {logits_50.shape}")
-        print(f"Initial KV-cache seqlen: {kv_cache_50[0][0].size(2)} (Head Dim: {kv_cache_50[0][0].size(3)})")
-
-    # 2. Проверка инкрементального прохода (T=1)
-    input_ids_T1 = torch.randint(0, VOCAB_SIZE, (1, 1), device=device)
-    with torch.no_grad():
-        logits_51, kv_cache_51 = model(input_ids_T1, past_kv=kv_cache_50)
-
-        # Проверка длины кеша: 50 + 1 = 51
-        assert kv_cache_51[0][0].size(2) == 51
-        print(f"Incremental logits shape: {logits_51.shape}")
-        print(f"Incremental KV-cache seqlen: {kv_cache_51[0][0].size(2)}")
-
-    # 3. Проверка функции generate
-    generated = model.generate(input_ids_T50, max_new_tokens=5, temperature=0.8, top_p=0.9)
-    print(f"Generated sequence length (50 + 5): {generated.shape[1]}")
-
-    save_path = "models/JiRack_GPT_L8_H8_PostNorm_fixed.pt"
-    torch.save(model.state_dict(), save_path)
-    print(f"Model successfully saved to {save_path}")