Update README.md

README.md

@@ -72,27 +72,29 @@ My LLMs
 # ========================================
 # Model Configuration (1B-class model)
 # ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 2048
-NUM_HEADS = 32
-NUM_LAYERS = 16
-MAX_SEQ_LEN = 2048
-# POS_EMB_MAX_LEN больше не используется, RoPE использует MAX_SEQ_LEN
-FFN_HIDDEN_DIM = int(MODEL_DIM * 4)
-HEAD_DIM = MODEL_DIM // NUM_HEADS  # 128
+- VOCAB_SIZE = 50257
+- MODEL_DIM = 2048
+- NUM_HEADS = 32
+- NUM_LAYERS = 16
+- MAX_SEQ_LEN = 2048
+# RoPE
+FFN_HIDDEN_DIM = int(MODEL_DIM * 4)     # Нестандартный FFN (4D)
+HEAD_DIM = MODEL_DIM // NUM_HEADS       # 64
+EPSILON = 1e-6
 ---
 
 # ========================================
 # Model Configuration 31B-class model)
 # ========================================
-VOCAB_SIZE = 50257
-MODEL_DIM = 2560
-NUM_HEADS = 32
-NUM_LAYERS = 32
-MAX_SEQ_LEN = 2048
-# POS_EMB_MAX_LEN больше не используется, RoPE использует MAX_SEQ_LEN
-FFN_HIDDEN_DIM = int(MODEL_DIM * 4)
-HEAD_DIM = MODEL_DIM // NUM_HEADS  # 128
+- VOCAB_SIZE = 50257
+- MODEL_DIM = 8192        # Большая размерность (как Llama 2 70B)
+- NUM_HEADS = 64
+- NUM_LAYERS = 32
+- MAX_SEQ_LEN = 8192      # Большая длина контекста
+# RoPE
+- FFN_HIDDEN_DIM = int(MODEL_DIM * 4)     # Нестандартный FFN (4D) - 32768
+- HEAD_DIM = MODEL_DIM // NUM_HEADS       # 128
+- EPSILON = 1e-6
 
 ---
 
@@ -100,27 +102,44 @@ HEAD_DIM = MODEL_DIM // NUM_HEADS  # 128
 # Model Configuration (8B-class model)
 # ========================================
 - VOCAB_SIZE = 50257
-- MODEL_DIM = 2048
+- MODEL_DIM = 4096        # Увеличен для 8.5B-класса (Стандартный, высокоэффективный)
+- NUM_HEADS = 32
+- NUM_LAYERS = 40         # Увеличен до 40 (как у Llama 13B)
+- MAX_SEQ_LEN = 2048
+# RoPE
+- FFN_HIDDEN_DIM = int(MODEL_DIM * 8 / 3) # 10922 (стандарт Llama)
+- HEAD_DIM = MODEL_DIM // NUM_HEADS       # 128
+- EPSILON = 1e-6
+
+---
+
+# ========================================
+# Model Configuration (10B-class model)
+# ========================================
+- VOCAB_SIZE = 50257
+- MODEL_DIM = 4096
 - NUM_HEADS = 32
-- NUM_LAYERS = 24
+- NUM_LAYERS = 48       # Увеличена глубина
 - MAX_SEQ_LEN = 2048
-# POS_EMB_MAX_LEN больше не используется, RoPE использует MAX_SEQ_LEN
-- FFN_HIDDEN_DIM = int(MODEL_DIM * 8 / 3)
-- HEAD_DIM = MODEL_DIM // NUM_HEADS  # 128
+# RoPE
+- FFN_HIDDEN_DIM = int(MODEL_DIM * 8 / 3) # 10922 (стандарт Llama)
+- HEAD_DIM = MODEL_DIM // NUM_HEADS       # 128
+- EPSILON = 1e-6
 
 ---
 
 # =====================================================================
-# Model Configuration (33B-class model) that available by request , 135 Gb
+# Model Configuration (33B-class model) that available by request 
 # =====================================================================
 - VOCAB_SIZE = 50257
-- MODEL_DIM = 8192
+- MODEL_DIM = 8192        # Большая размерность (как Llama 2 70B)
 - NUM_HEADS = 64
 - NUM_LAYERS = 32
-- MAX_SEQ_LEN = 8192
-- POS_EMB_MAX_LEN = 32768
-- FFN_HIDDEN_DIM = 4 * MODEL_DIM
-- HEAD_DIM = MODEL_DIM // NUM_HEADS  # 128
+- MAX_SEQ_LEN = 8192      # Большая длина контекста
+# RoPE
+- FFN_HIDDEN_DIM = int(MODEL_DIM * 4)     # Нестандартный FFN (4D) - 32768
+- HEAD_DIM = MODEL_DIM // NUM_HEADS       # 128
+- EPSILON = 1e-6
 
 ---
 
@@ -129,13 +148,16 @@ HEAD_DIM = MODEL_DIM // NUM_HEADS  # 128
 # =======================================================================
 - VOCAB_SIZE      = 50257
 - MODEL_DIM       = 8192          # Hidden size (d_model)
-- NUM_HEADS       = 64            # Attention heads → head_dim = 128
-- NUM_KV_HEADS    = 8             # GQA: 8 KV heads (like LLaMA-70B), 64 Q heads
-- NUM_LAYERS      = 80            # 80 layers → ~71B params
-- MAX_SEQ_LEN     = 8192          # Training context
-- POS_EMB_MAX_LEN = 32768         # Safe for long generation
-- FFN_HIDDEN_DIM  = 32768         # 4 × MODEL_DIM (32,768) → matches LLaMA-70B exactly
+- NUM_HEADS       = 64            # Q Heads
+- NUM_KV_HEADS    = 8             # KV Heads (GQA ratio = 8)
+- NUM_LAYERS      = 80            # 80 layers
+- MAX_SEQ_LEN     = 8192          # Max context (RoPE)
+# FFN LLaMA-70B Hidden Dim: 28672 (32768 * 2/3 + 32768 * 1/3 * 2/3 * 0.95, roughly 28672)
+# Точное значение для LLaMA: 2 * (D * 2/3) + D * 2/3 * (1 - 2/3) * ~1.2 (для 70B)
+# Используем стандартный FFN LLaMA-70B для точности
+- FFN_HIDDEN_DIM  = 28672         
 - HEAD_DIM        = MODEL_DIM // NUM_HEADS
+- EPSILON         = 1e-6
 
 ---
 #
@@ -143,17 +165,17 @@ HEAD_DIM = MODEL_DIM // NUM_HEADS  # 128
 #  It was Designed military design and Discover worlds and learn space and science goals
 #
 # =======================================================================
-# 120B Configuration (real numbers) that available by request , JiRack Super Brain
+# 140B Configuration (real numbers) that available by request , JiRack Super Brain
 # =======================================================================
-- VOCAB_SIZE       = 32000        # Modern tokenizer size (you can change later)
-- MODEL_DIM        = 12288        # d_model = 12288 → matches 120B+ scale
+- VOCAB_SIZE       = 32000        
+- MODEL_DIM        = 12288        # d_model
 - NUM_HEADS        = 96           # Query heads
-- NUM_KV_HEADS     = 12           # GQA: 8× groups (12 KV heads → 96/12 = 8)
-- NUM_LAYERS       = 80           # 80 layers
+- NUM_KV_HEADS     = 12           # GQA: 8× groups
+- NUM_LAYERS       = 80           
 - HEAD_DIM         = MODEL_DIM // NUM_HEADS          # 128
-- FFN_HIDDEN_DIM   = int(4 * MODEL_DIM * 1.3)        # ~4.3× expansion (DeepSeek/Qwen style) → 53248
-- MAX_SEQ_LEN      = 131072       # Training on 128k context
-- POS_EMB_MAX_LEN  = 262144       # Generation up to 256k+ tokens safely
+- FFN_HIDDEN_DIM   = int(4 * MODEL_DIM * 1.3)        # 53248
+- MAX_SEQ_LEN      = 131072       # Max context
+- EPSILON          = 1e-6