Upload folder using huggingface_hub

config.json

@@ -6,17 +6,17 @@
   "vocab_size": 16000,
   "context_len": 1024,
   "tie_word_embeddings": true,
+  "auto_map": {
+    "AutoConfig": "modeling_ivme.IvmeConfig",
+    "AutoModelForCausalLM": "modeling_ivme.IvmeConversateV2HF"
+  },
+  "transformers_version": "4.41.0",
   "hidden_dim": 384,
   "n_layers": 10,
   "n_heads": 6,
   "dropout": 0.0,
-  "ffn_mult": 1.0,
+  "ffn_mult": 4.0,
   "norm_eps": 1e-05,
   "rope_theta": 10000.0,
-  "head_dim": 64,
-  "auto_map": {
-    "AutoConfig": "modeling_ivme.IvmeConfig",
-    "AutoModelForCausalLM": "modeling_ivme.IvmeConversateV2HF"
-  },
-  "transformers_version": "4.41.0"
-}
+  "head_dim": 64
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db78dcec9aa219862c3b3ca36372af576ff02696e17c3389e121f3653c4cf591
-size 95396368
+oid sha256:eb24ff17d2145c4966b780c31dd0ff124f7ab8ec88fdd912ebc50981525a9d96
+size 119972832

modeling_ivme.py

@@ -1,174 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from transformers import PretrainedConfig, PreTrainedModel
-from transformers.modeling_outputs import CausalLMOutputWithPast
-
-# ==========================================
-# 1. RESMİ HUGGING FACE CONFIG SIFINFI
-# ==========================================
-class IvmeConfig(PretrainedConfig):
-    model_type = "ivme"
-    
-    def __init__(
-        self, 
-        vocab_size=16000, 
-        context_len=1024, 
-        tie_word_embeddings=True,
-        hidden_dim=384,
-        n_layers=10,
-        n_heads=6,
-        dropout=0.0,
-        ffn_mult=1.0,
-        norm_eps=1e-5,
-        rope_theta=10000.0,
-        head_dim=64,
-        **kwargs
-    ):
-        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
-        self.vocab_size = vocab_size
-        self.context_len = context_len
-        self.hidden_dim = hidden_dim
-        self.n_layers = n_layers
-        self.n_heads = n_heads
-        self.dropout = dropout
-        self.ffn_mult = ffn_mult
-        self.norm_eps = norm_eps
-        self.rope_theta = rope_theta
-        self.head_dim = head_dim
-
-# ==========================================
-# 2. SİZİN MODELİNİZİN ORİJİNAL MATEMATİKSEL KATMANLARI
-# ==========================================
-class RMSNorm(nn.Module):
-    def __init__(self, dim: int, eps: float = 1e-5):
-        super().__init__()
-        self.eps = eps
-        self.weight = nn.Parameter(torch.ones(dim))
-    def forward(self, x):
-        pow_x = x.pow(2).mean(-1, keepdim=True)
-        return x * torch.rsqrt(pow_x + self.eps) * self.weight
-
-def precompute_rope_freqs(dim: int, max_seq_len: int, theta: float = 10000.0):
-    inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim))
-    t = torch.arange(max_seq_len, dtype=torch.float32)
-    freqs = torch.outer(t, inv_freq)
-    return torch.polar(torch.ones_like(freqs), freqs)
-
-class CausalSelfAttention(nn.Module):
-    def __init__(self, hidden_dim: int, n_heads: int, dropout: float = 0.0):
-        super().__init__()
-        self.n_heads = n_heads
-        self.head_dim = hidden_dim // n_heads
-        self.wq = nn.Linear(hidden_dim, hidden_dim, bias=False)
-        self.wk = nn.Linear(hidden_dim, hidden_dim, bias=False)
-        self.wv = nn.Linear(hidden_dim, hidden_dim, bias=False)
-        self.wo = nn.Linear(hidden_dim, hidden_dim, bias=False)
-        self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
-
-    def forward(self, x, rope_freqs):
-        B, T, C = x.shape
-        q, k, v = self.wq(x), self.wk(x), self.wv(x)
-        q = q.view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
-        k = k.view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
-        v = v.view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
-        
-        # Dtype Uyuşmazlığını Çözen Güvenli RoPE Hesaplaması
-        q_complex = torch.view_as_complex(q.float().reshape(*q.shape[:-1], -1, 2))
-        k_complex = torch.view_as_complex(k.float().reshape(*k.shape[:-1], -1, 2))
-        freqs = rope_freqs[:T].view(1, 1, T, -1).to(q_complex.device)
-        
-        # Girdinin orijinal tipine (bfloat16) güvenli geri dönüş
-        q = torch.view_as_real(q_complex * freqs).flatten(3).to(x.dtype)
-        k = torch.view_as_real(k_complex * freqs).flatten(3).to(x.dtype)
-        v = v.to(x.dtype)
-        
-        scores = torch.matmul(q, k.transpose(-2, -1)) / (self.head_dim ** 0.5)
-        mask = torch.full((T, T), float("-inf"), device=x.device).triu(1)
-        scores = scores + mask
-        probs = torch.softmax(scores, dim=-1).to(x.dtype)
-        probs = self.dropout(probs)
-        
-        output = torch.matmul(probs, v)
-        output = output.transpose(1, 2).contiguous().view(B, T, C)
-        return self.wo(output)
-
-class SwiGLU(nn.Module):
-    def __init__(self, hidden_dim: int, ffn_mult: float = 1.0):
-        super().__init__()
-        hidden_features = int(2 * hidden_dim * 4 / 3)
-        hidden_features = int(ffn_mult * hidden_features)
-        self.w1 = nn.Linear(hidden_dim, hidden_features, bias=False)
-        self.w2 = nn.Linear(hidden_features, hidden_dim, bias=False)
-        self.w3 = nn.Linear(hidden_dim, hidden_features, bias=False)
-    def forward(self, x):
-        return self.w2(F.silu(self.w1(x)) * self.w3(x))
-
-class TransformerBlock(nn.Module):
-    def __init__(self, cfg):
-        super().__init__()
-        self.attn_norm = RMSNorm(cfg.hidden_dim, eps=cfg.norm_eps)
-        self.attn = CausalSelfAttention(cfg.hidden_dim, cfg.n_heads, cfg.dropout)
-        self.ffn_norm = RMSNorm(cfg.hidden_dim, eps=cfg.norm_eps)
-        self.ffn = SwiGLU(cfg.hidden_dim, cfg.ffn_mult)
-    def forward(self, x, rope_freqs):
-        x = x + self.attn(self.attn_norm(x), rope_freqs)
-        x = x + self.ffn(self.ffn_norm(x))
-        return x
-
-# ==========================================
-# 3. RESMİ HUGGING FACE CAUSAL LM MODEL SINIFI
-# ==========================================
-class IvmeConversateV2HF(PreTrainedModel):
-    config_class = IvmeConfig
-    base_model_prefix = "model"
-    
-    def __init__(self, config):
-        super().__init__(config)
-        self.config = config
-        
-        self.tok_embed = nn.Embedding(config.vocab_size, config.hidden_dim)
-        self.blocks = nn.ModuleList([TransformerBlock(config) for _ in range(config.n_layers)])
-        self.final_norm = RMSNorm(config.hidden_dim, eps=config.norm_eps)
-        self.lm_head = nn.Linear(config.hidden_dim, config.vocab_size, bias=False)
-        
-        if config.tie_word_embeddings:
-            self.lm_head.weight = self.tok_embed.weight
-            
-        rope_freqs = precompute_rope_freqs(config.hidden_dim // config.n_heads, config.context_len, config.rope_theta)
-        self.register_buffer("rope_freqs", rope_freqs, persistent=False)
-        
-        self.post_init()
-
-    def load_state_dict(self, state_dict, strict=False):
-        """Sözlükteki katman isim uyuşmazlıklarını (Qwen/Llama türevleri) otomatik olarak 
-        sizin orijinal Ivme katmanlarınıza bağlayan can kurtaran akıllı haritalama."""
-        clean_state_dict = {}
-        for k, v in state_dict.items():
-            new_key = k
-            # Attention Katman Haritalaması
-            new_key = new_key.replace(".attn.q_proj.", ".attn.wq.")
-            new_key = new_key.replace(".attn.k_proj.", ".attn.wk.")
-            new_key = new_key.replace(".attn.v_proj.", ".attn.wv.")
-            new_key = new_key.replace(".attn.out_proj.", ".attn.wo.")
-            # FFN Katman Haritalaması
-            new_key = new_key.replace(".ffn.gate_proj.", ".ffn.w1.")
-            new_key = new_key.replace(".ffn.down_proj.", ".ffn.w2.")
-            new_key = new_key.replace(".ffn.up_proj.", ".ffn.w3.")
-            clean_state_dict[new_key] = v
-            
-        return super().load_state_dict(clean_state_dict, strict=False)
-
-    def forward(self, input_ids=None, labels=None, **kwargs):
-        B, T = input_ids.shape
-        x = self.tok_embed(input_ids)
-        for block in self.blocks:
-            x = block(x, self.rope_freqs)
-        x = self.final_norm(x)
-        logits = self.lm_head(x)
-        
-        loss = None
-        if labels is not None:
-            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1), ignore_index=-1)
-            
-        return CausalLMOutputWithPast(loss=loss, logits=logits)

tokenizer_config.json

@@ -1,7 +1 @@
-{
-  "add_bos_token": false,
-  "add_eos_token": false,
-  "model_max_length": 1024,
-  "tokenizer_class": "PreTrainedTokenizerFast",
-  "clean_up_tokenization_spaces": true
-}
+{"add_bos_token": false, "add_eos_token": false, "model_max_length": 1024, "tokenizer_class": "PreTrainedTokenizerFast"}