Fixes

Model_Architecture/config.json

@@ -4,7 +4,7 @@
         "max_seq_len": 2048,
         "dtype": "bf16",
         "scale_fmt": null,
-        "vocab_size": 102400,
+        "vocab_size": 32768,
         "dim": 1024,
         "inter_dim": 4096,
         "moe_inter_dim": 1024,
@@ -27,7 +27,7 @@
         "beta_fast": 32,
         "beta_slow": 1,
         "mscale": 1.0,
-        "tokenizer_name": "gpt2"
+        "tokenizer_name": "turkish"
     },
     "training": {
         "learning_rate": 3e-4,

Model_Architecture/generation.py

@@ -164,7 +164,7 @@ if __name__ == "__main__":
     from pathlib import Path
 
     # Configuration: Set to True to use Turkish tokenizer, False for tiktoken
-    USE_TURKISH_TOKENIZER = False  # Change this to True for Turkish text generation
+    USE_TURKISH_TOKENIZER = True  # Change this to False for English text generation
 
     # Example configuration - smaller model for testing
     config_path = Path("config.json")
@@ -179,15 +179,20 @@ if __name__ == "__main__":
 
     # Initialize tokenizer
     tokenizer_name = getattr(args, "tokenizer_name", "gpt2")
+    # Auto-detect Turkish tokenizer from config
+    use_turkish = (tokenizer_name.lower() == "turkish") or USE_TURKISH_TOKENIZER
+
     tokenizer = get_tokenizer(
-        use_turkish=USE_TURKISH_TOKENIZER,
-        tokenizer_name=tokenizer_name
+        use_turkish=use_turkish,
+        tokenizer_name="gpt2" if use_turkish else tokenizer_name
     )
 
     # Update vocab size if using Turkish tokenizer
-    if USE_TURKISH_TOKENIZER and isinstance(tokenizer, TurkishTokenizerWrapper):
-        args.vocab_size = tokenizer.n_vocab
-        print(f"📊 Updated vocab_size to {args.vocab_size:,} for Turkish tokenizer")
+    if use_turkish and isinstance(tokenizer, TurkishTokenizerWrapper):
+        if args.vocab_size != tokenizer.n_vocab:
+            print(f"⚠️  Config vocab_size ({args.vocab_size:,}) doesn't match tokenizer ({tokenizer.n_vocab:,})")
+            args.vocab_size = tokenizer.n_vocab
+            print(f"📊 Updated vocab_size to {args.vocab_size:,} for Turkish tokenizer")
 
     # Initialize model
     print("Initializing model...")

Model_Architecture/model.py

@@ -190,7 +190,8 @@ class RMSNorm(nn.Module):
         self.weight = nn.Parameter(torch.ones(dim))
 
     def forward(self, x: torch.Tensor):
-        return F.rms_norm(x, (self.dim,), self.weight, self.eps)
+        output = F.rms_norm(x, (self.dim,), self.weight, self.eps)
+        return output.to(x.dtype)
 
 
 #####################################
@@ -228,8 +229,8 @@ class MultiHeadLatentAttention(nn.Module):
             self.softmax_scale = self.softmax_scale * mscale * mscale
 
 
-        self.register_buffer("kv_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.kv_lora_rank), persistent=False)
-        self.register_buffer("pe_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.qk_rope_head_dim), persistent=False)
+        self.register_buffer("kv_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.kv_lora_rank, dtype=Linear.dtype), persistent=False)
+        self.register_buffer("pe_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.qk_rope_head_dim, dtype=Linear.dtype), persistent=False)
 
     def forward(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor, mask: Optional[torch.Tensor]):
 
@@ -280,7 +281,7 @@ class Gate(nn.Module):
         self.route_scale = args.route_scale
 
         # Gate weight
-        self.weight = nn.Parameter(torch.empty(args.n_routed_experts, args.dim))
+        self.weight = nn.Parameter(torch.empty(args.n_routed_experts, args.dim, dtype=Linear.dtype))
 
         # Optional routing bias for fine-tuning expert selection
         if args.use_routing_bias:
@@ -509,7 +510,7 @@ class ismail(nn.Module):
 
     def forward(self, tokens: torch.Tensor, start_pos: int = 0) -> torch.Tensor:
         bsz, seqlen = tokens.shape
-        h = self.tok_embeddings(tokens)
+        h = self.tok_embeddings(tokens).to(Linear.dtype)
         freqs_cis = self.freqs_cis[start_pos:start_pos + seqlen]
 
         # Create causal mask