total repo overhaul

myolmoe/__init__.py

@@ -0,0 +1 @@
+from .modeling_olmoe import OLMoEForCausalLM

myolmoe/config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "OlmoeForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "clip_qkv": null,
+  "eos_token_id": 50279,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "model_type": "olmoe",
+  "norm_topk_prob": false,
+  "num_attention_heads": 16,
+  "num_experts": 64,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "output_router_logits": false,
+  "pad_token_id": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.01,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 50304
+}

myolmoe/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 50279,
+  "pad_token_id": 1,
+  "transformers_version": "4.52.4"
+}

myolmoe/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e3cff7e367794685c241169072c940d200918617d5e2813f1c387dff52d845e
+size 4997744872

myolmoe/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15ef5c730ee3cfed7199498788cd2faf337203fc74b529625e7502cdd759f4a7
+size 4997235176

myolmoe/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9abac4ac1b55c9adabac721a02fa39971f103eea9a65c310972b1246de76e04
+size 3843741912

modeling_myolmoe.py → myolmoe/modeling_myolmoe.py

@@ -1,26 +1,25 @@
-"""PyTorch MyOLMoE model with custom routing mechanisms."""
-
 import math
 from typing import List, Optional, Tuple, Union
-
 import torch
 import torch.nn.functional as F
 import torch.utils.checkpoint
 from torch import nn
 from torch.distributions import Categorical
-
 from transformers.activations import ACT2FN
 from transformers.cache_utils import Cache, DynamicCache
 from transformers.modeling_attn_mask_utils import AttentionMaskConverter
-from transformers.modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast
+from transformers.modeling_outputs import (
+    MoeCausalLMOutputWithPast,
+    MoeModelOutputWithPast,
+)
 from transformers.modeling_utils import PreTrainedModel
 from transformers.models.olmoe.configuration_olmoe import OlmoeConfig
 from transformers.utils import logging
 from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
 
-
 logger = logging.get_logger(__name__)
 
+
 def load_balancing_loss_func(
     gate_logits: Union[torch.Tensor, Tuple[torch.Tensor], None],
     num_experts: Optional[int] = None,
@@ -29,77 +28,195 @@ def load_balancing_loss_func(
 ) -> Union[torch.Tensor, int]:
     if gate_logits is None or not isinstance(gate_logits, tuple):
         return 0
-
     if isinstance(gate_logits, tuple):
         compute_device = gate_logits[0].device
-        concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0)
-
+        concatenated_gate_logits = torch.cat(
+            [layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0
+        )
     routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1)
-
     _, selected_experts = torch.topk(routing_weights, top_k, dim=-1)
-
     expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts)
-
     if attention_mask is None:
-        # Compute the percentage of tokens routed to each experts
         tokens_per_expert = torch.mean(expert_mask.float(), dim=0)
-
-        # Compute the average probability of routing to these experts
         router_prob_per_expert = torch.mean(routing_weights, dim=0)
     else:
         batch_size, sequence_length = attention_mask.shape
-        num_hidden_layers = concatenated_gate_logits.shape[0] // (batch_size * sequence_length)
-
-        # Compute the mask that masks all padding tokens as 0 with the same shape of expert_mask
+        num_hidden_layers = concatenated_gate_logits.shape[0] // (
+            batch_size * sequence_length
+        )
         expert_attention_mask = (
             attention_mask[None, :, :, None, None]
-            .expand((num_hidden_layers, batch_size, sequence_length, top_k, num_experts))
+            .expand(
+                (num_hidden_layers, batch_size, sequence_length, top_k, num_experts)
+            )
             .reshape(-1, top_k, num_experts)
             .to(compute_device)
         )
-
-        # Compute the percentage of tokens routed to each experts
-        tokens_per_expert = torch.sum(expert_mask.float() * expert_attention_mask, dim=0) / torch.sum(
-            expert_attention_mask, dim=0
-        )
-
-        # Compute the mask that masks all padding tokens as 0 with the same shape of tokens_per_expert
+        tokens_per_expert = torch.sum(
+            expert_mask.float() * expert_attention_mask, dim=0
+        ) / torch.sum(expert_attention_mask, dim=0)
         router_per_expert_attention_mask = (
             attention_mask[None, :, :, None]
             .expand((num_hidden_layers, batch_size, sequence_length, num_experts))
             .reshape(-1, num_experts)
             .to(compute_device)
         )
+        router_prob_per_expert = torch.sum(
+            routing_weights * router_per_expert_attention_mask, dim=0
+        ) / torch.sum(router_per_expert_attention_mask, dim=0)
+    overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0))
+    return overall_loss * num_experts
+
+
+class OlmoeRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-5):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+ALL_LAYERNORM_LAYERS.append(OlmoeRMSNorm)
+
+
+class OlmoeRotaryEmbedding(nn.Module):
+    def __init__(self, config: OlmoeConfig, device=None):
+        super().__init__()
+        if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
+            self.rope_type = config.rope_scaling.get(
+                "rope_type", config.rope_scaling.get("type")
+            )
+        else:
+            self.rope_type = "default"
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+        self.config = config
+        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.original_inv_freq = self.inv_freq
 
-        # Compute the average probability of routing to these experts
-        router_prob_per_expert = torch.sum(routing_weights * router_per_expert_attention_mask, dim=0) / torch.sum(
-            router_per_expert_attention_mask, dim=0
+    @torch.no_grad()
+    @dynamic_rope_update
+    def forward(self, x, position_ids):
+        inv_freq_expanded = (
+            self.inv_freq[None, :, None]
+            .float()
+            .expand(position_ids.shape[0], -1, 1)
+            .to(x.device)
         )
+        position_ids_expanded = position_ids[:, None, :].float()
+        device_type = (
+            x.device.type
+            if isinstance(x.device.type, str) and x.device.type != "mps"
+            else "cpu"
+        )
+        with torch.autocast(device_type=device_type, enabled=False):
+            freqs = (
+                inv_freq_expanded.float() @ position_ids_expanded.float()
+            ).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+def rotate_half(x):
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class OlmoeMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x):
+        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        return down_proj
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(
+        batch, num_key_value_heads, n_rep, slen, head_dim
+    )
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
 
-    overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0))
-    return overall_loss * num_experts
 
 class OlmoeAttention(nn.Module):
-    """Multi-headed attention from 'Attention Is All You Need' paper"""
-    
     def __init__(self, config: OlmoeConfig, layer_idx: Optional[int] = None):
         super().__init__()
         self.config = config
         self.layer_idx = layer_idx
+        if layer_idx is None:
+            logger.warning_once(
+                f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will "
+                "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` "
+                "when creating this class."
+            )
+        self.attention_dropout = config.attention_dropout
         self.hidden_size = config.hidden_size
         self.num_heads = config.num_attention_heads
         self.head_dim = self.hidden_size // self.num_heads
         self.num_key_value_heads = config.num_key_value_heads
         self.num_key_value_groups = self.num_heads // self.num_key_value_heads
-        self.attention_dropout = config.attention_dropout
-        
-        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias)
-        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
-        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
-        self.o_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=config.attention_bias)
+        self.max_position_embeddings = config.max_position_embeddings
+        self.rope_theta = config.rope_theta
+        self.is_causal = True
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+        self.q_proj = nn.Linear(
+            self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.k_proj = nn.Linear(
+            self.hidden_size,
+            self.num_key_value_heads * self.head_dim,
+            bias=config.attention_bias,
+        )
+        self.v_proj = nn.Linear(
+            self.hidden_size,
+            self.num_key_value_heads * self.head_dim,
+            bias=config.attention_bias,
+        )
+        self.o_proj = nn.Linear(
+            self.hidden_size, self.hidden_size, bias=config.attention_bias
+        )
         self.q_norm = OlmoeRMSNorm(self.hidden_size, eps=config.rms_norm_eps)
-        self.k_norm = OlmoeRMSNorm((self.hidden_size // self.num_heads) * self.num_key_value_heads, eps=config.rms_norm_eps)
-        
+        self.k_norm = OlmoeRMSNorm(
+            (self.hidden_size // self.num_heads) * self.num_key_value_heads,
+            eps=config.rms_norm_eps,
+        )
+
     def forward(
         self,
         hidden_states: torch.Tensor,
@@ -113,274 +230,283 @@ class OlmoeAttention(nn.Module):
         **kwargs,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
         bsz, q_len, _ = hidden_states.size()
-
         query_states = self.q_norm(self.q_proj(hidden_states))
         key_states = self.k_norm(self.k_proj(hidden_states))
         value_states = self.v_proj(hidden_states)
-
-        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
-        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
-        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
-
+        if self.config.clip_qkv is not None:
+            query_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+            key_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+            value_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+        query_states = query_states.view(
+            bsz, q_len, self.num_heads, self.head_dim
+        ).transpose(1, 2)
+        key_states = key_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
+        value_states = value_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
         cos, sin = position_embeddings
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
-
+        query_states, key_states = apply_rotary_pos_emb(
+            query_states, key_states, cos, sin
+        )
         if past_key_value is not None:
             cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
-            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
-
+            key_states, value_states = past_key_value.update(
+                key_states, value_states, self.layer_idx, cache_kwargs
+            )
         key_states = repeat_kv(key_states, self.num_key_value_groups)
         value_states = repeat_kv(value_states, self.num_key_value_groups)
-
-        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
-
+        attn_weights = torch.matmul(
+            query_states, key_states.transpose(2, 3)
+        ) / math.sqrt(self.head_dim)
         if attention_mask is not None:
-            attn_weights = attn_weights + attention_mask
-
-        # upcast attention to fp32
-        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
-        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+            causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+            attn_weights = attn_weights + causal_mask
+        attn_weights = nn.functional.softmax(
+            attn_weights, dim=-1, dtype=torch.float32
+        ).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(
+            attn_weights, p=self.attention_dropout, training=self.training
+        )
         attn_output = torch.matmul(attn_weights, value_states)
-
         if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
             raise ValueError(
                 f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
                 f" {attn_output.size()}"
             )
-
         attn_output = attn_output.transpose(1, 2).contiguous()
         attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
         attn_output = self.o_proj(attn_output)
-
         if not output_attentions:
             attn_weights = None
+        return attn_output, attn_weights, past_key_value
+
 
+class OlmoeFlashAttention2(OlmoeAttention):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        cache_position: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        output_attentions = False
+        bsz, q_len, _ = hidden_states.size()
+        query_states = self.q_norm(self.q_proj(hidden_states))
+        key_states = self.k_norm(self.k_proj(hidden_states))
+        value_states = self.v_proj(hidden_states)
+        if self.config.clip_qkv is not None:
+            query_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+            key_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+            value_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+        query_states = query_states.view(
+            bsz, q_len, self.num_heads, self.head_dim
+        ).transpose(1, 2)
+        key_states = key_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
+        value_states = value_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(
+            query_states, key_states, cos, sin
+        )
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            key_states, value_states = past_key_value.update(
+                key_states, value_states, self.layer_idx, cache_kwargs
+            )
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+        dropout_rate = self.attention_dropout if self.training else 0.0
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+        attn_output = _flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            q_len,
+            dropout=dropout_rate,
+            use_top_left_mask=self._flash_attn_uses_top_left_mask,
+            is_causal=self.is_causal,
+        )
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+        attn_output = self.o_proj(attn_output)
+        if not output_attentions:
+            attn_weights = None
         return attn_output, attn_weights, past_key_value
 
-def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
-    """Repeat key/value heads for grouped query attention"""
-    batch, num_key_value_heads, seq_len, head_dim = hidden_states.shape
-    if n_rep == 1:
-        return hidden_states
-    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, seq_len, head_dim)
-    return hidden_states.reshape(batch, num_key_value_heads * n_rep, seq_len, head_dim)
 
-def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None):
-    """Apply rotary position embedding to query and key tensors"""
-    cos = cos.unsqueeze(1)
-    sin = sin.unsqueeze(1)
-    q_embed = (q * cos) + (rotate_half(q) * sin)
-    k_embed = (k * cos) + (rotate_half(k) * sin)
-    return q_embed, k_embed
+class OlmoeSdpaAttention(OlmoeAttention):
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        cache_position: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if output_attentions:
+            logger.warning_once(
+                "OlmoeModel is using OlmoeSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
+                'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+            )
+            return super().forward(
+                hidden_states=hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+                cache_position=cache_position,
+                position_embeddings=position_embeddings,
+            )
+        bsz, q_len, _ = hidden_states.size()
+        query_states = self.q_norm(self.q_proj(hidden_states))
+        key_states = self.k_norm(self.k_proj(hidden_states))
+        value_states = self.v_proj(hidden_states)
+        if self.config.clip_qkv is not None:
+            query_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+            key_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+            value_states.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+        query_states = query_states.view(
+            bsz, q_len, self.num_heads, self.head_dim
+        ).transpose(1, 2)
+        key_states = key_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
+        value_states = value_states.view(
+            bsz, q_len, self.num_key_value_heads, self.head_dim
+        ).transpose(1, 2)
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(
+            query_states, key_states, cos, sin
+        )
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            key_states, value_states = past_key_value.update(
+                key_states, value_states, self.layer_idx, cache_kwargs
+            )
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        causal_mask = attention_mask
+        if attention_mask is not None:
+            causal_mask = causal_mask[:, :, :, : key_states.shape[-2]]
+        if query_states.device.type == "cuda" and causal_mask is not None:
+            query_states = query_states.contiguous()
+            key_states = key_states.contiguous()
+            value_states = value_states.contiguous()
+        is_causal = True if causal_mask is None and q_len > 1 else False
+        attn_output = torch.nn.functional.scaled_dot_product_attention(
+            query_states,
+            key_states,
+            value_states,
+            attn_mask=causal_mask,
+            dropout_p=self.attention_dropout if self.training else 0.0,
+            is_causal=is_causal,
+        )
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.view(bsz, q_len, self.hidden_size)
+        attn_output = self.o_proj(attn_output)
+        return attn_output, None, past_key_value
 
-def rotate_half(x):
-    """Rotates half the hidden dims of the input."""
-    x1 = x[..., : x.shape[-1] // 2]
-    x2 = x[..., x.shape[-1] // 2 :]
-    return torch.cat((-x2, x1), dim=-1)
 
-# Define the attention classes dictionary
 OLMOE_ATTENTION_CLASSES = {
     "eager": OlmoeAttention,
+    "flash_attention_2": OlmoeFlashAttention2,
+    "sdpa": OlmoeSdpaAttention,
 }
 
-class OlmoeRMSNorm(nn.Module):
-    """RMSNorm implementation matching the original OLMoE implementation"""
-    def __init__(self, hidden_size, eps=1e-5):
-        super().__init__()
-        self.weight = nn.Parameter(torch.ones(hidden_size))
-        self.variance_epsilon = eps
 
-    def forward(self, hidden_states):
-        input_dtype = hidden_states.dtype
-        hidden_states = hidden_states.to(torch.float32)
-        variance = hidden_states.pow(2).mean(-1, keepdim=True)
-        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
-        return self.weight * hidden_states.to(input_dtype)
-
-class OlmoeMLP(nn.Module):
-    """Feed-forward network implementation"""
+class OlmoeSparseMoeBlock(nn.Module):
     def __init__(self, config):
-        super().__init__()
-        self.config = config
-        self.hidden_size = config.hidden_size
-        self.intermediate_size = config.intermediate_size
-        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
-        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
-        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
-        self.act_fn = ACT2FN[config.hidden_act]
-
-    def forward(self, x):
-        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
-
-class MyOLMoERouting(nn.Module):
-    """Custom routing mechanism for MyOLMoE with different routing strategies."""
-    
-    def __init__(self, config: OlmoeConfig):
         super().__init__()
         self.num_experts = config.num_experts
         self.top_k = config.num_experts_per_tok
-        self.hidden_size = config.hidden_size
-        self.routing_type = getattr(config, "routing_type", "sparse")
-        self.router_temperature = getattr(config, "router_temperature", 1.0)
-        
-        # Shared components
+        self.norm_topk_prob = config.norm_topk_prob
         self.gate = nn.Linear(config.hidden_size, self.num_experts, bias=False)
-        
-        # For non-deterministic routing
-        self.gumbel_noise = getattr(config, "gumbel_noise", 0.1)
-        
-    def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
-        batch_size, sequence_length, hidden_dim = hidden_states.shape
-        hidden_states = hidden_states.view(-1, hidden_dim)
-        router_logits = self.gate(hidden_states)
-
-        # Always use softmax, even for "dense" routing
-        routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
-        routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
-        
-        if self.norm_topk_prob:
-            routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
-            
-        routing_weights = routing_weights.to(hidden_states.dtype)
-        
-        if self.routing_type == "dense":
-            # Dense routing - use all experts equally
-            routing_weights = torch.ones_like(router_logits) / self.num_experts
-            selected_experts = torch.topk(routing_weights, self.top_k, dim=-1).indices
-            
-        elif self.routing_type == "non_deterministic":
-            # Non-deterministic routing with temperature and Gumbel noise
-            if self.training:
-                # Add Gumbel noise during training
-                noise = torch.rand_like(router_logits) * self.gumbel_noise
-                router_logits = router_logits + noise
-            
-            # Apply temperature scaling
-            routing_weights = F.softmax(router_logits / self.router_temperature, dim=-1)
-            selected_experts = torch.multinomial(routing_weights, self.top_k)
-            
-        else:  # Default sparse routing
-            # Standard sparse top-k routing
-            routing_weights = F.softmax(router_logits, dim=-1)
-            routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
-        
-        return routing_weights, selected_experts, router_logits
-
-class OlmoeRotaryEmbedding(nn.Module):
-    def __init__(self, config: OlmoeConfig, device=None):
-        super().__init__()
-        # BC: "rope_type" was originally "type"
-        if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
-            self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
-        else:
-            self.rope_type = "default"
-        self.max_seq_len_cached = config.max_position_embeddings
-        self.original_max_seq_len = config.max_position_embeddings
-
-        self.config = config
-        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
-
-        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
-        self.register_buffer("inv_freq", inv_freq, persistent=False)
-        self.original_inv_freq = self.inv_freq
-
-    @torch.no_grad()
-    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
-    def forward(self, x, position_ids):
-        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
-        position_ids_expanded = position_ids[:, None, :].float()
-
-        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
-        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
-            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
-            emb = torch.cat((freqs, freqs), dim=-1)
-            cos = emb.cos() * self.attention_scaling
-            sin = emb.sin() * self.attention_scaling
-
-        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+        self.experts = nn.ModuleList(
+            [OlmoeMLP(config) for _ in range(self.num_experts)]
+        )
 
-class MyOLMoESparseMoeBlock(nn.Module):
-    """Modified sparse MoE block with custom routing."""
-    
-    def __init__(self, config: OlmoeConfig):
-        super().__init__()
-        self.num_experts = config.num_experts
-        self.top_k = config.num_experts_per_tok
-        self.hidden_size = config.hidden_size
-        self.intermediate_size = config.intermediate_size
-        self.norm_topk_prob = config.norm_topk_prob
-        
-        # Custom routing mechanism
-        self.router = MyOLMoERouting(config)
-        
-        # Expert networks
-        self.experts = nn.ModuleList([OlmoeMLP(config) for _ in range(self.num_experts)])
-        
-    def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
-        print(f"DEBUG: MoE forward start - hidden_states shape: {hidden_states.shape}")
-        batch_size, seq_len, hidden_dim = hidden_states.shape
-        print("absolute precision")
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        batch_size, sequence_length, hidden_dim = hidden_states.shape
         hidden_states = hidden_states.view(-1, hidden_dim)
-        
-        # Get routing weights and selected experts
-        print(f"DEBUG: 123: {self.router(hidden_states).shape}")
-        routing_weights, selected_experts, router_logits = self.router(hidden_states)
         router_logits = self.gate(hidden_states)
-
         routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
-        routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
+        routing_weights, selected_experts = torch.topk(
+            routing_weights, self.top_k, dim=-1
+        )
         if self.norm_topk_prob:
             routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
-        # we cast back to the input dtype
         routing_weights = routing_weights.to(hidden_states.dtype)
-        print(f"DEBUG: MoE forward mid - routing_weights shape: {routing_weights.shape}, selected_experts shape: {selected_experts.shape}")
-
         final_hidden_states = torch.zeros(
-            (batch_size * seq_len, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device
+            (batch_size * sequence_length, hidden_dim),
+            dtype=hidden_states.dtype,
+            device=hidden_states.device,
         )
-        
-        # One-hot expert mask
-        expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts)
-        
-        # Dispatch to experts
+        expert_mask = torch.nn.functional.one_hot(
+            selected_experts, num_classes=self.num_experts
+        ).permute(2, 1, 0)
         for expert_idx in range(self.num_experts):
             expert_layer = self.experts[expert_idx]
-            idx, top_x = torch.where(expert_mask[:, :, expert_idx])
-            
-            if idx.shape[0] == 0:
-                continue
-                
-            current_state = hidden_states[None, top_x].reshape(-1, self.hidden_size)
-            current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None]
-            
-            final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))
-        
-        final_hidden_states = final_hidden_states.reshape(batch_size, seq_len, self.hidden_size)
-        
-        # Return 3 values: (hidden_states, router_logits, aux_loss)
-        # For now, we'll return None for aux_loss - you can compute it here if needed
-        aux_loss = None
-        print(f"DEBUG: MoE forward returning - final_hidden_states shape: {final_hidden_states.shape}, router_logits shape: {router_logits.shape}, aux_loss: {aux_loss}")
-        return final_hidden_states, router_logits, aux_loss
+            idx, top_x = torch.where(expert_mask[expert_idx])
+            current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
+            current_hidden_states = (
+                expert_layer(current_state) * routing_weights[top_x, idx, None]
+            )
+            final_hidden_states.index_add_(
+                0, top_x, current_hidden_states.to(hidden_states.dtype)
+            )
+        final_hidden_states = final_hidden_states.reshape(
+            batch_size, sequence_length, hidden_dim
+        )
+        return final_hidden_states, router_logits
 
-class MyOLMoEDecoderLayer(nn.Module):
-    """Modified decoder layer with custom MoE routing."""
-    
+
+class OlmoeDecoderLayer(nn.Module):
     def __init__(self, config: OlmoeConfig, layer_idx: int):
         super().__init__()
         self.hidden_size = config.hidden_size
-        
-        # Self-attention
-        self.self_attn = OLMOE_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx)
-        
-        # Custom MoE layer
-        self.mlp = MyOLMoESparseMoeBlock(config)
-        
-        # Layer norms
+        self.self_attn = OLMOE_ATTENTION_CLASSES[config._attn_implementation](
+            config=config, layer_idx=layer_idx
+        )
+        self.mlp = OlmoeSparseMoeBlock(config)
         self.input_layernorm = OlmoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.post_attention_layernorm = OlmoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        
+        self.post_attention_layernorm = OlmoeRMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps
+        )
+
     def forward(
         self,
         hidden_states: torch.Tensor,
@@ -392,12 +518,12 @@ class MyOLMoEDecoderLayer(nn.Module):
         use_cache: Optional[bool] = False,
         cache_position: Optional[torch.LongTensor] = None,
         position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Cache], Optional[torch.Tensor]]:
+        **kwargs,
+    ) -> Tuple[
+        torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
+    ]:
         residual = hidden_states
-        
-        # Self-attention
         hidden_states = self.input_layernorm(hidden_states)
-        print(f"DEBUG: Before MoE call - hidden_states shape: {hidden_states.shape}")
         hidden_states, self_attn_weights, present_key_value = self.self_attn(
             hidden_states=hidden_states,
             attention_mask=attention_mask,
@@ -407,32 +533,36 @@ class MyOLMoEDecoderLayer(nn.Module):
             use_cache=use_cache,
             cache_position=cache_position,
             position_embeddings=position_embeddings,
+            **kwargs,
         )
-        print(f"DEBUG: After MoE call - hidden_states shape: {hidden_states.shape}")
         hidden_states = residual + hidden_states
-        
-        # MoE layer
         residual = hidden_states
         hidden_states = self.post_attention_layernorm(hidden_states)
-        hidden_states, router_logits, aux_loss = self.mlp(hidden_states)
+        hidden_states, router_logits = self.mlp(hidden_states)
         hidden_states = residual + hidden_states
-        print(f"DEBUG: End Decoder call - hidden_states shape: {hidden_states.shape}")
-        
-        # Always return 4 values in consistent order
-        return (
-            hidden_states,
-            self_attn_weights if output_attentions else None,
-            present_key_value if use_cache else None,
-            router_logits if output_router_logits else None
-        )
+        outputs = (hidden_states,)
+        if output_attentions:
+            outputs += (self_attn_weights,)
+        if use_cache:
+            outputs += (present_key_value,)
+        if output_router_logits:
+            outputs += (router_logits,)
+        return outputs
 
-class MyOLMoEPreTrainedModel(PreTrainedModel):
+
+@auto_docstring
+class OlmoePreTrainedModel(PreTrainedModel):
     config_class = OlmoeConfig
     base_model_prefix = "model"
     supports_gradient_checkpointing = True
-    _no_split_modules = ["MyOLMoEDecoderLayer"]
+    _no_split_modules = ["OlmoeDecoderLayer"]
     _skip_keys_device_placement = ["past_key_values"]
-    
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_cache_class = True
+    _supports_quantized_cache = True
+    _supports_static_cache = False
+
     def _init_weights(self, module):
         std = self.config.initializer_range
         if isinstance(module, nn.Linear):
@@ -447,31 +577,33 @@ class MyOLMoEPreTrainedModel(PreTrainedModel):
                 module.weight.data[module.padding_idx].zero_()
 
 
-class MyOLMoEModel(MyOLMoEPreTrainedModel):
-    """Modified OLMoE model with custom routing."""
-    
+@auto_docstring
+class OlmoeModel(OlmoePreTrainedModel):
     def __init__(self, config: OlmoeConfig):
         super().__init__(config)
         self.padding_idx = config.pad_token_id
         self.vocab_size = config.vocab_size
-        
-        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.embed_tokens = nn.Embedding(
+            config.vocab_size, config.hidden_size, self.padding_idx
+        )
         self.layers = nn.ModuleList(
-            [MyOLMoEDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+            [
+                OlmoeDecoderLayer(config, layer_idx)
+                for layer_idx in range(config.num_hidden_layers)
+            ]
         )
         self.norm = OlmoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.rotary_emb = OlmoeRotaryEmbedding(config)
+        self.rotary_emb = OlmoeRotaryEmbedding(config=config)
         self.gradient_checkpointing = False
-        
-        # Initialize weights
         self.post_init()
-        
+
     def get_input_embeddings(self):
         return self.embed_tokens
-    
+
     def set_input_embeddings(self, value):
         self.embed_tokens = value
-        
+
+    @auto_docstring
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -486,88 +618,119 @@ class MyOLMoEModel(MyOLMoEPreTrainedModel):
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.LongTensor] = None,
     ) -> Union[Tuple, MoeModelOutputWithPast]:
-        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
         output_router_logits = (
-            output_router_logits if output_router_logits is not None else self.config.output_router_logits
+            output_router_logits
+            if output_router_logits is not None
+            else self.config.output_router_logits
         )
         output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
         )
         use_cache = use_cache if use_cache is not None else self.config.use_cache
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError(
+                "You must specify exactly one of input_ids or inputs_embeds"
+            )
+        if self.gradient_checkpointing and self.training and use_cache:
+            logger.warning_once(
+                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
+            )
+            use_cache = False
         if inputs_embeds is None:
             inputs_embeds = self.embed_tokens(input_ids)
-            
-        if past_key_values is None:
-            past_key_values = DynamicCache()
-            
+        return_legacy_cache = False
+        if use_cache and not isinstance(past_key_values, Cache):
+            return_legacy_cache = True
+            if past_key_values is None:
+                past_key_values = DynamicCache()
+            else:
+                past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+                logger.warning_once(
+                    "We detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and "
+                    "will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class "
+                    "(https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)"
+                )
         if cache_position is None:
-            past_seen_tokens = past_key_values.get_seq_length()
+            past_seen_tokens = (
+                past_key_values.get_seq_length() if past_key_values is not None else 0
+            )
             cache_position = torch.arange(
-                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+                past_seen_tokens,
+                past_seen_tokens + inputs_embeds.shape[1],
+                device=inputs_embeds.device,
             )
-            
         if position_ids is None:
             position_ids = cache_position.unsqueeze(0)
-            
         causal_mask = self._update_causal_mask(
-            attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
+            attention_mask,
+            inputs_embeds,
+            cache_position,
+            past_key_values,
+            output_attentions,
         )
-        
         hidden_states = inputs_embeds
         position_embeddings = self.rotary_emb(hidden_states, position_ids)
-        
         all_hidden_states = () if output_hidden_states else None
         all_self_attns = () if output_attentions else None
         all_router_logits = () if output_router_logits else None
         next_decoder_cache = None
-        
-        # In MyOLMoEModel.forward(), replace the layer processing loop with:
-        
-        for decoder_layer in self.layers:
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
             if output_hidden_states:
                 all_hidden_states += (hidden_states,)
-                
-            layer_outputs = decoder_layer(
-                hidden_states,
-                attention_mask=causal_mask,
-                position_ids=position_ids,
-                past_key_value=past_key_values,
-                output_attentions=output_attentions,
-                output_router_logits=output_router_logits,
-                use_cache=use_cache,
-                cache_position=cache_position,
-                position_embeddings=position_embeddings,
-            )
-            
-            # Unpack the consistent 4-value return
-            hidden_states, self_attn_weights, present_key_value, router_logits = layer_outputs
-            
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    decoder_layer.__call__,
+                    hidden_states,
+                    causal_mask,
+                    position_ids,
+                    past_key_values,
+                    output_attentions,
+                    output_router_logits,
+                    use_cache,
+                    cache_position,
+                    position_embeddings,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=causal_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_values,
+                    output_attentions=output_attentions,
+                    output_router_logits=output_router_logits,
+                    use_cache=use_cache,
+                    cache_position=cache_position,
+                    position_embeddings=position_embeddings,
+                )
+            hidden_states = layer_outputs[0]
             if use_cache:
-                next_decoder_cache = present_key_value
-                
-            if output_attentions and self_attn_weights is not None:
-                all_self_attns += (self_attn_weights,)
-                
-            if output_router_logits and router_logits is not None:
-                all_router_logits += (router_logits,)
-                
+                next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
             if output_router_logits and layer_outputs[-1] is not None:
                 all_router_logits += (layer_outputs[-1],)
-                
         hidden_states = self.norm(hidden_states)
-        
         if output_hidden_states:
             all_hidden_states += (hidden_states,)
-            
         next_cache = next_decoder_cache if use_cache else None
-        
+        if return_legacy_cache:
+            next_cache = next_cache.to_legacy_cache()
         if not return_dict:
             return tuple(
-                v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_router_logits] if v is not None
+                v
+                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
+                if v is not None
             )
-            
         return MoeModelOutputWithPast(
             last_hidden_state=hidden_states,
             past_key_values=next_cache,
@@ -575,44 +738,139 @@ class MyOLMoEModel(MyOLMoEPreTrainedModel):
             attentions=all_self_attns,
             router_logits=all_router_logits,
         )
-        
-    def _update_causal_mask(self, attention_mask, input_tensor, cache_position, past_key_values, output_attentions):
-        # Same as original implementation
-        if attention_mask is not None and 0.0 in attention_mask:
-            return attention_mask
-        return None
 
-
-class MyOLMoEForCausalLM(MyOLMoEPreTrainedModel):
-    """MyOLMoE model for causal language modeling with custom routing."""
-    
+    def _update_causal_mask(
+        self,
+        attention_mask: torch.Tensor,
+        input_tensor: torch.Tensor,
+        cache_position: torch.Tensor,
+        past_key_values: Cache,
+        output_attentions: bool,
+    ):
+        if self.config._attn_implementation == "flash_attention_2":
+            if attention_mask is not None and 0.0 in attention_mask:
+                return attention_mask
+            return None
+        past_seen_tokens = (
+            past_key_values.get_seq_length() if past_key_values is not None else 0
+        )
+        using_static_cache = isinstance(past_key_values, StaticCache)
+        if (
+            self.config._attn_implementation == "sdpa"
+            and not using_static_cache
+            and not output_attentions
+        ):
+            if AttentionMaskConverter._ignore_causal_mask_sdpa(
+                attention_mask,
+                inputs_embeds=input_tensor,
+                past_key_values_length=past_seen_tokens,
+                is_training=self.training,
+            ):
+                return None
+        dtype, device = input_tensor.dtype, input_tensor.device
+        sequence_length = input_tensor.shape[1]
+        if using_static_cache:
+            target_length = past_key_values.get_max_cache_shape()
+        else:
+            target_length = (
+                attention_mask.shape[-1]
+                if isinstance(attention_mask, torch.Tensor)
+                else past_seen_tokens + sequence_length + 1
+            )
+        causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
+            attention_mask,
+            sequence_length=sequence_length,
+            target_length=target_length,
+            dtype=dtype,
+            device=device,
+            cache_position=cache_position,
+            batch_size=input_tensor.shape[0],
+        )
+        if (
+            self.config._attn_implementation == "sdpa"
+            and attention_mask is not None
+            and attention_mask.device.type in ["cuda", "xpu", "npu"]
+            and not output_attentions
+        ):
+            min_dtype = torch.finfo(dtype).min
+            causal_mask = AttentionMaskConverter._unmask_unattended(
+                causal_mask, min_dtype
+            )
+        return causal_mask
+
+    @staticmethod
+    def _prepare_4d_causal_attention_mask_with_cache_position(
+        attention_mask: torch.Tensor,
+        sequence_length: int,
+        target_length: int,
+        dtype: torch.dtype,
+        device: torch.device,
+        cache_position: torch.Tensor,
+        batch_size: int,
+        **kwargs,
+    ):
+        if attention_mask is not None and attention_mask.dim() == 4:
+            causal_mask = attention_mask
+        else:
+            min_dtype = torch.finfo(dtype).min
+            causal_mask = torch.full(
+                (sequence_length, target_length),
+                fill_value=min_dtype,
+                dtype=dtype,
+                device=device,
+            )
+            if sequence_length != 1:
+                causal_mask = torch.triu(causal_mask, diagonal=1)
+            causal_mask *= torch.arange(
+                target_length, device=device
+            ) > cache_position.reshape(-1, 1)
+            causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
+            if attention_mask is not None:
+                causal_mask = causal_mask.clone()
+                mask_length = attention_mask.shape[-1]
+                padding_mask = (
+                    causal_mask[:, :, :, :mask_length]
+                    + attention_mask[:, None, None, :]
+                )
+                padding_mask = padding_mask == 0
+                causal_mask[:, :, :, :mask_length] = causal_mask[
+                    :, :, :, :mask_length
+                ].masked_fill(padding_mask, min_dtype)
+        return causal_mask
+
+
+class OlmoeForCausalLM(OlmoePreTrainedModel, GenerationMixin):
     _tied_weights_keys = ["lm_head.weight"]
-    
-    def __init__(self, config: OlmoeConfig):
+
+    def __init__(self, config):
         super().__init__(config)
-        self.model = MyOLMoEModel(config)
+        self.model = OlmoeModel(config)
         self.vocab_size = config.vocab_size
         self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
-        
         self.router_aux_loss_coef = config.router_aux_loss_coef
         self.num_experts = config.num_experts
         self.num_experts_per_tok = config.num_experts_per_tok
-        
-        # Initialize weights
         self.post_init()
-        
+
     def get_input_embeddings(self):
         return self.model.embed_tokens
-    
+
     def set_input_embeddings(self, value):
         self.model.embed_tokens = value
-        
+
     def get_output_embeddings(self):
         return self.lm_head
-    
+
     def set_output_embeddings(self, new_embeddings):
         self.lm_head = new_embeddings
-        
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    @auto_docstring
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -627,17 +885,27 @@ class MyOLMoEForCausalLM(MyOLMoEPreTrainedModel):
         output_router_logits: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.LongTensor] = None,
+        logits_to_keep: Union[int, torch.Tensor] = 0,
+        **loss_kwargs,
     ) -> Union[Tuple, MoeCausalLMOutputWithPast]:
-        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
         output_router_logits = (
-            output_router_logits if output_router_logits is not None else self.config.output_router_logits
+            output_router_logits
+            if output_router_logits is not None
+            else self.config.output_router_logits
         )
         output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
         )
-        use_cache = use_cache if use_cache is not None else self.config.use_cache
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        
         outputs = self.model(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -651,14 +919,16 @@ class MyOLMoEForCausalLM(MyOLMoEPreTrainedModel):
             return_dict=return_dict,
             cache_position=cache_position,
         )
-        
         hidden_states = outputs[0]
-        logits = self.lm_head(hidden_states)
-        
+        slice_indices = (
+            slice(-logits_to_keep, None)
+            if isinstance(logits_to_keep, int)
+            else logits_to_keep
+        )
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
         loss = None
         if labels is not None:
-            loss = F.cross_entropy(logits.view(-1, self.vocab_size), labels.view(-1))
-            
+            loss = self.loss_function(logits, labels, self.vocab_size, **loss_kwargs)
         aux_loss = None
         if output_router_logits:
             aux_loss = load_balancing_loss_func(
@@ -669,13 +939,11 @@ class MyOLMoEForCausalLM(MyOLMoEPreTrainedModel):
             )
             if labels is not None:
                 loss += self.router_aux_loss_coef * aux_loss.to(loss.device)
-                
         if not return_dict:
             output = (logits,) + outputs[1:]
             if output_router_logits:
                 output = (aux_loss,) + output
             return (loss,) + output if loss is not None else output
-            
         return MoeCausalLMOutputWithPast(
             loss=loss,
             aux_loss=aux_loss,
@@ -685,4 +953,3 @@ class MyOLMoEForCausalLM(MyOLMoEPreTrainedModel):
             attentions=outputs.attentions,
             router_logits=outputs.router_logits,
         )
-        

myolmoe/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|padding|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

myolmoe/tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": null,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": null
+}

olmoe_wrapper.py

@@ -1,358 +0,0 @@
-"""
-LM Evaluation Harness Wrapper for Modified MyOLMoE
-"""
-import torch
-from typing import List, Optional, Union, Dict, Any
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from lm_eval.api.model import LM
-from lm_eval.api.registry import register_model
-import numpy as np
-
-
-@register_model("myolmoe")
-class MyOLMoELM(LM):
-    """LM Evaluation Harness wrapper for MYOLMoE model."""
-    
-    def __init__(
-        self,
-        pretrained: str = None,
-        device: str = "cuda",
-        batch_size: int = 1,
-        max_length: int = 2048,
-        trust_remote_code: bool = False,
-        dtype: str = "float16",
-        parallelize: bool = False,
-        device_map: Optional[str] = None,
-        **kwargs
-    ):
-        super().__init__()
-        
-        # Initialize device and batch size
-        if device == "cuda" and not torch.cuda.is_available():
-            device = "cpu"
-        self._device = torch.device(device)
-        self._batch_size = batch_size
-        self._max_length = max_length
-        
-        # Set dtype
-        if dtype == "float16":
-            self._dtype = torch.float16
-        elif dtype == "bfloat16":
-            self._dtype = torch.bfloat16
-        else:
-            self._dtype = torch.float32
-        
-        # Load tokenizer and model
-        if pretrained:
-            self.tokenizer = AutoTokenizer.from_pretrained(
-                pretrained, 
-                trust_remote_code=trust_remote_code,
-                padding_side="left"
-            )
-            
-            # Ensure pad token is set
-            if self.tokenizer.pad_token is None:
-                if self.tokenizer.eos_token is not None:
-                    self.tokenizer.pad_token = self.tokenizer.eos_token
-                else:
-                    self.tokenizer.add_special_tokens({'pad_token': '<pad>'})
-            
-            self.model = AutoModelForCausalLM.from_pretrained(
-                pretrained,
-                torch_dtype=self._dtype,
-                device_map=device_map if parallelize else None,
-                trust_remote_code=trust_remote_code,
-                **kwargs
-            )
-            
-            if not parallelize:
-                self.model = self.model.to(self._device)
-            
-            self.model.eval()
-        else:
-            raise ValueError("pretrained model path must be specified")
-    
-    @property
-    def eot_token_id(self):
-        """End of text token ID."""
-        return self.tokenizer.eos_token_id
-    
-    @property
-    def max_length(self):
-        """Maximum sequence length."""
-        return self._max_length
-    
-    @property
-    def max_gen_toks(self):
-        """Maximum number of tokens to generate."""
-        return 256
-    
-    @property
-    def batch_size(self):
-        """Batch size for evaluation."""
-        return self._batch_size
-    
-    @property
-    def device(self):
-        """Device used for evaluation."""
-        return self._device
-    
-    def tok_encode(self, string: str, add_special_tokens=True) -> List[int]:
-        """Encode a string to token IDs."""
-        return self.tokenizer.encode(string, add_special_tokens=add_special_tokens)
-    
-    def tok_decode(self, tokens: List[int]) -> str:
-        """Decode token IDs to string."""
-        return self.tokenizer.decode(tokens, skip_special_tokens=True)
-    
-    def loglikelihood(self, requests: List[tuple]) -> List[tuple]:
-        """
-        Compute log-likelihood for each request.
-        Each request is a tuple of (context, continuation).
-        """
-        results = []
-        
-        # Process requests in batches
-        for i in range(0, len(requests), self.batch_size):
-            batch = requests[i:i + self.batch_size]
-            batch_results = self._loglikelihood_batch(batch)
-            results.extend(batch_results)
-        
-        return results
-    
-    def _loglikelihood_batch(self, batch: List[tuple]) -> List[tuple]:
-        """Process a batch of loglikelihood requests."""
-        contexts, continuations = zip(*batch)
-        
-        # Encode full sequences (context + continuation)
-        full_sequences = [ctx + cont for ctx, cont in zip(contexts, continuations)]
-        full_encodings = [self.tok_encode(seq) for seq in full_sequences]
-        
-        # Encode contexts only
-        context_encodings = [self.tok_encode(ctx) for ctx in contexts]
-        
-        # Pad sequences to the same length
-        max_len = min(max(len(seq) for seq in full_encodings), self.max_length)
-        
-        input_ids = []
-        attention_masks = []
-        continuation_masks = []
-        
-        for full_seq, ctx_seq in zip(full_encodings, context_encodings):
-            # Truncate if necessary (keep the end)
-            if len(full_seq) > max_len:
-                full_seq = full_seq[-max_len:]
-                ctx_len = max(0, len(ctx_seq) - (len(full_encodings[0]) - max_len))
-            else:
-                ctx_len = len(ctx_seq)
-            
-            # Create padding
-            pad_length = max_len - len(full_seq)
-            padded_seq = [self.tokenizer.pad_token_id] * pad_length + full_seq
-            attention_mask = [0] * pad_length + [1] * len(full_seq)
-            
-            # Create mask for continuation tokens only
-            continuation_mask = [0] * max_len
-            continuation_start = pad_length + ctx_len
-            for j in range(continuation_start, max_len):
-                continuation_mask[j] = 1
-            
-            input_ids.append(padded_seq)
-            attention_masks.append(attention_mask)
-            continuation_masks.append(continuation_mask)
-        
-        # Convert to tensors
-        input_ids = torch.tensor(input_ids, device=self.device)
-        attention_masks = torch.tensor(attention_masks, device=self.device)
-        continuation_masks = torch.tensor(continuation_masks, device=self.device)
-        
-        # Forward pass
-        with torch.no_grad():
-            outputs = self.model(input_ids=input_ids, attention_mask=attention_masks)
-            logits = outputs.logits
-        
-        # Compute log-likelihoods
-        results = []
-        for i in range(len(batch)):
-            # Get logits for positions where we predict continuation tokens
-            # Shift logits and tokens for next-token prediction
-            shifted_logits = logits[i, :-1]  # Remove last position
-            shifted_tokens = input_ids[i, 1:]  # Remove first position
-            shifted_mask = continuation_masks[i][1:]  # Remove first position
-            
-            # Only consider continuation tokens
-            valid_positions = shifted_mask.bool()
-            if valid_positions.sum() == 0:
-                results.append((float('-inf'), False))
-                continue
-            
-            # Get log probabilities
-            log_probs = torch.log_softmax(shifted_logits, dim=-1)
-            token_log_probs = log_probs.gather(1, shifted_tokens.unsqueeze(1)).squeeze(1)
-            
-            # Sum only over continuation tokens
-            valid_log_probs = token_log_probs[valid_positions]
-            total_log_prob = valid_log_probs.sum().item()
-            
-            # For simplicity, assume greedy is True
-            is_greedy = True
-            
-            results.append((total_log_prob, is_greedy))
-        
-        return results
-    
-    def generate_until(self, requests: List[tuple]) -> List[str]:
-        """
-        Generate text until stopping criteria are met.
-        Each request is a tuple of (context, generation_kwargs).
-        """
-        results = []
-        
-        # Process requests in batches
-        for i in range(0, len(requests), self.batch_size):
-            batch = requests[i:i + self.batch_size]
-            batch_results = self._generate_until_batch(batch)
-            results.extend(batch_results)
-        
-        return results
-    
-    def _generate_until_batch(self, batch: List[tuple]) -> List[str]:
-        """Process a batch of generation requests."""
-        contexts = []
-        gen_kwargs_list = []
-        
-        for context, gen_kwargs in batch:
-            contexts.append(context)
-            gen_kwargs_list.append(gen_kwargs)
-        
-        # Encode contexts
-        context_encodings = [self.tok_encode(ctx) for ctx in contexts]
-        
-        # Pad contexts
-        max_ctx_len = min(max(len(seq) for seq in context_encodings), 
-                         self.max_length - self.max_gen_toks)
-        
-        input_ids = []
-        attention_masks = []
-        
-        for ctx_seq in context_encodings:
-            # Truncate if necessary (keep the end)
-            if len(ctx_seq) > max_ctx_len:
-                ctx_seq = ctx_seq[-max_ctx_len:]
-            
-            # Pad sequence
-            pad_length = max_ctx_len - len(ctx_seq)
-            padded_seq = [self.tokenizer.pad_token_id] * pad_length + ctx_seq
-            attention_mask = [0] * pad_length + [1] * len(ctx_seq)
-            
-            input_ids.append(padded_seq)
-            attention_masks.append(attention_mask)
-        
-        # Convert to tensors
-        input_ids = torch.tensor(input_ids, device=self.device)
-        attention_masks = torch.tensor(attention_masks, device=self.device)
-        
-        # Generate
-        with torch.no_grad():
-            # Use first gen_kwargs for simplicity (can be extended)
-            gen_kwargs = gen_kwargs_list[0] if gen_kwargs_list else {}
-            
-            # Set default generation parameters
-            generation_kwargs = {
-                'max_new_tokens': gen_kwargs.get('max_gen_toks', self.max_gen_toks),
-                'do_sample': gen_kwargs.get('do_sample', False),
-                'temperature': gen_kwargs.get('temperature', 1.0),
-                'top_p': gen_kwargs.get('top_p', 1.0),
-                'pad_token_id': self.tokenizer.pad_token_id,
-                'eos_token_id': self.tokenizer.eos_token_id,
-                'attention_mask': attention_masks,
-                'use_cache': True,
-            }
-            
-            generated = self.model.generate(
-                input_ids=input_ids,
-                **generation_kwargs
-            )
-        
-        # Decode generated text
-        results = []
-        for i, gen_seq in enumerate(generated):
-            # Get original context length (without padding)
-            original_ctx_len = len(context_encodings[i])
-            
-            # Extract only the newly generated tokens
-            if len(gen_seq) > len(input_ids[i]):
-                new_tokens = gen_seq[len(input_ids[i]):].tolist()
-            else:
-                new_tokens = []
-            
-            # Decode
-            if new_tokens:
-                generated_text = self.tok_decode(new_tokens)
-            else:
-                generated_text = ""
-            
-            # Apply stopping criteria if specified
-            if 'until' in gen_kwargs_list[i]:
-                stop_strings = gen_kwargs_list[i]['until']
-                if isinstance(stop_strings, str):
-                    stop_strings = [stop_strings]
-                
-                for stop_str in stop_strings:
-                    if stop_str in generated_text:
-                        generated_text = generated_text[:generated_text.index(stop_str)]
-                        break
-            
-            results.append(generated_text)
-        
-        return results
-    
-    def loglikelihood_rolling(self, requests: List[tuple]) -> List[float]:
-        """
-        Compute rolling log-likelihood for each request.
-        Each request is a tuple containing the text to evaluate.
-        """
-        results = []
-        
-        for request in requests:
-            text = request[0] if isinstance(request, tuple) else request
-            tokens = self.tok_encode(text)
-            
-            if len(tokens) <= 1:
-                results.append(0.0)
-                continue
-            
-            # Compute log-likelihood using sliding window approach
-            total_log_prob = 0.0
-            total_tokens = 0
-            
-            # Use sliding window for long sequences
-            window_size = min(self.max_length, len(tokens))
-            
-            for i in range(1, len(tokens)):
-                # Define the window
-                start_idx = max(0, i - window_size + 1)
-                end_idx = i + 1
-                
-                window_tokens = tokens[start_idx:end_idx]
-                input_ids = torch.tensor([window_tokens], device=self.device)
-                
-                with torch.no_grad():
-                    outputs = self.model(input_ids=input_ids)
-                    logits = outputs.logits
-                
-                # Get log probability for the target token
-                target_pos = len(window_tokens) - 1
-                target_token = window_tokens[target_pos]
-                
-                if target_pos > 0:  # Ensure we have a position to predict from
-                    token_logits = logits[0, target_pos - 1]
-                    log_prob = torch.log_softmax(token_logits, dim=-1)[target_token].item()
-                    total_log_prob += log_prob
-                    total_tokens += 1
-            
-            # Return mean log-likelihood per token
-            avg_log_prob = total_log_prob / total_tokens if total_tokens > 0 else 0.0
-            results.append(avg_log_prob)
-        
-        return results

scripts/downloadmodel.py

@@ -0,0 +1,7 @@
+from transformers import AutoConfig
+import os
+os.environ["HF_HUB_READ_TIMEOUT"] = "60"
+
+config = AutoConfig.from_pretrained("allenai/OLMoE-1B-7B-0924", timeout=60)
+
+print(config)

scripts/downloadweights.py

@@ -1,20 +0,0 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-model = AutoModelForCausalLM.from_pretrained(
-    "allenai/OLMoE-7B",            # Exact name from Hugging Face
-    trust_remote_code=True,        # Required if they use custom modeling_olmoe.py
-    use_safetensors=True           # Ensures .safetensors file is used
-)
-
-tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-7B")
-print(model.config)
-print(model.__class__)
-
-from transformers.utils.hub import cached_file
-
-# Example: get the path to the config file or model weights index
-config_path = cached_file("allenai/OLMoE-7B", "config.json", trust_remote_code=True)
-print(config_path)
-import os
-model_path = os.path.dirname(config_path)
-print(model_path)