| from transformers import PretrainedConfig | |
| class YConfig31(PretrainedConfig): | |
| model_type = "ynet31" | |
| def __init__(self, **kwargs): | |
| self.dropout = kwargs.pop("dropout", 0.0) | |
| self.bos_token_id = kwargs.pop("bos_token_id", 151644) | |
| self.eos_token_id = kwargs.pop("eos_token_id", 151645) | |
| self.pad_token_id = kwargs.pop("pad_token_id", 151643) | |
| self.hidden_act = kwargs.pop("hidden_act", "silu") | |
| self.hidden_size = kwargs.pop("hidden_size", 768) | |
| self.num_hidden_layers = kwargs.pop("num_hidden_layers", 8) | |
| self.max_position_embeddings = kwargs.pop("max_position_embeddings", 8192) | |
| self.vocab_size = kwargs.pop("vocab_size", 6400) | |
| self.rms_norm_eps = kwargs.pop("rms_norm_eps", 1e-6) | |
| self.rope_theta = kwargs.pop("rope_theta", 5e4) | |
| self.rope_scaling = kwargs.pop("rope_scaling", None) | |
| self.dtype = kwargs.pop("dtype", "float32") | |
| self.self_distill = kwargs.pop("self_distill", True) | |
| self.intermediate_size = kwargs.pop("intermediate_size", 1536) | |
| self.num_heads = kwargs.pop("num_heads", 12) | |
| self.mla_kv_lora_rank = kwargs.pop("mla_kv_lora_rank", 64) | |
| self.mla_qk_nope_head_dim = kwargs.pop("mla_qk_nope_head_dim", 64) | |
| self.mla_qk_rope_head_dim = kwargs.pop("mla_qk_rope_head_dim", 32) | |
| self.mla_attn_impl = kwargs.pop("mla_attn_impl", "absorb") | |
| self.qkv_lora = kwargs.pop("qkv_lora", False) | |
| self.gradient_checkpointing = kwargs.pop("gradient_checkpointing", 0) | |
| self.use_sengram = kwargs.pop("use_sengram", True) | |
| self.sengram_bucket_size = kwargs.pop("sengram_bucket_size", 4096) | |
| self.sengram_topk = kwargs.pop("sengram_topk", 2) | |
| self.engram_bucket_size = kwargs.pop("engram_bucket_size", self.sengram_bucket_size) | |
| self.engram_topk = kwargs.pop("engram_topk", self.sengram_topk) | |
| super().__init__( | |
| bos_token_id=self.bos_token_id, | |
| eos_token_id=self.eos_token_id, | |
| pad_token_id=self.pad_token_id, | |
| **kwargs, | |
| ) | |