| from transformers.configuration_utils import PretrainedConfig |
| from typing import Tuple, List, Optional |
|
|
| class NegativeConfig(PretrainedConfig): |
| model_type = "negative" |
| keys_to_ignore_at_inference = ["past_key_values"] |
|
|
| def __init__( |
| self, |
| vocab_size: int = 2564, |
| hidden_size: int = 128, |
| num_hidden_layers: int = 21, |
| num_attention_heads: int = 4, |
| num_key_value_heads: int = 2, |
| intermediate_size: int = 345, |
| swiglu_interval: int = 3, |
| num_lanes: int = 4, |
| use_engram: bool = True, |
| engram_entries: int = 2400, |
| engram_ngram_orders: Tuple[int, ...] = (2, 3), |
| use_xsa: bool = False, |
| use_per_head_gating: bool = False, |
| max_position_embeddings: int = 2048, |
| rope_theta: float = 2500.0, |
| rms_norm_eps: float = 1e-5, |
| tie_word_embeddings: bool = True, |
| use_cache: bool = False, |
| initializer_range: float = 0.02, |
| **kwargs, |
| ): |
| self.vocab_size = vocab_size |
| self.hidden_size = hidden_size |
| self.num_hidden_layers = num_hidden_layers |
| self.num_attention_heads = num_attention_heads |
| self.num_key_value_heads = num_key_value_heads |
| self.intermediate_size = intermediate_size |
| self.swiglu_interval = swiglu_interval |
| self.num_lanes = num_lanes |
| self.use_engram = use_engram |
| self.engram_entries = engram_entries |
| self.engram_ngram_orders = list(engram_ngram_orders) |
| self.use_xsa = use_xsa |
| self.use_per_head_gating = use_per_head_gating |
| self.max_position_embeddings = max_position_embeddings |
| self.rope_theta = rope_theta |
| self.rms_norm_eps = rms_norm_eps |
| self.initializer_range = initializer_range |
| self.head_dim = hidden_size // num_attention_heads |
| self.auto_map = { |
| "AutoConfig": "configuration_negative.NegativeConfig", |
| "AutoModel": "modeling_negative.NegativeModel", |
| "AutoModelForCausalLM": "modeling_negative.NegativeModelForCausalLM", |
| } |
|
|
| super().__init__( |
| tie_word_embeddings=tie_word_embeddings, |
| use_cache=use_cache, |
| **kwargs, |
| ) |
|
|
