Upgrade: Scaled up to ~15M parameters and 256 block size.

modeling_syko.py

@@ -11,10 +11,10 @@ class SykoConfig(PretrainedConfig):
     def __init__(
         self,
         vocab_size=4096,
-        n_embd=256,
-        n_layer=6,
-        n_head=8,
-        block_size=64,
+        n_embd=384,      # ARTIRILDI (Eskisi 256)
+        n_layer=8,       # ARTIRILDI (Eskisi 6)
+        n_head=6,        # AYARLANDI (384 / 64 = 6)
+        block_size=256,  # ARTIRILDI (Eskisi 64) -> Daha uzun hafıza
         dropout=0.2,
         **kwargs
     ):
@@ -45,6 +45,7 @@ class Head(nn.Module):
         k = self.key(x)
         q = self.query(x)
         wei = q @ k.transpose(-2, -1) * (C ** -0.5)
+        # Maskeleme dinamik olmalı (gelen T kadar)
         wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))
         wei = F.softmax(wei, dim=-1)
         wei = self.dropout(wei)
@@ -103,7 +104,6 @@ class SykoForCausalLM(PreTrainedModel):
         self.n_layer = config.n_layer
         self.dropout = config.dropout
         
-        # Embedding katmanının adı 'token_embedding_table'
         self.token_embedding_table = nn.Embedding(self.vocab_size, self.n_embd)
         self.position_embedding_table = nn.Embedding(self.block_size, self.n_embd)
         self.blocks = nn.Sequential(*[Block(self.n_embd, self.n_head, self.block_size, self.dropout) for _ in range(self.n_layer)])
@@ -112,13 +112,11 @@ class SykoForCausalLM(PreTrainedModel):
         
         self.apply(self._init_weights)
 
-    # --- YENİ EKLENEN KISIM: HF BU FONKSİYONLARI ARIYOR ---
     def get_input_embeddings(self):
         return self.token_embedding_table
 
     def set_input_embeddings(self, new_embeddings):
         self.token_embedding_table = new_embeddings
-    # -----------------------------------------------------
 
     def _init_weights(self, module):
         if isinstance(module, nn.Linear):
@@ -133,16 +131,25 @@ class SykoForCausalLM(PreTrainedModel):
         B, T = idx.shape
         device = idx.device
         
+        # Eğer context (T), block_size'dan büyükse kırp (Safety check)
+        if T > self.block_size:
+            idx = idx[:, -self.block_size:]
+            T = self.block_size
+            
         pos_emb = self.position_embedding_table(torch.arange(T, device=device))
         tok_emb = self.token_embedding_table(idx)
         x = tok_emb + pos_emb
         
         x = self.blocks(x)
-        x = self.ln_f(x)
+        x = self.ln1_f(x) if hasattr(self, 'ln1_f') else self.ln_f(x)
         logits = self.lm_head(x)
 
         loss = None
         if labels is not None:
+            # Labels da kırpılmalı eğer idx kırpıldıysa
+            if labels.shape[1] > T:
+                 labels = labels[:, -T:]
+                 
             B, T, C = logits.shape
             logits_reshaped = logits.view(B*T, C)
             labels_reshaped = labels.view(B*T)