File size: 2,082 Bytes
e5e01a9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | from transformers import PretrainedConfig
class YConfig31(PretrainedConfig):
model_type = "ynet31"
def __init__(self, **kwargs):
self.dropout = kwargs.pop("dropout", 0.0)
self.bos_token_id = kwargs.pop("bos_token_id", 151644)
self.eos_token_id = kwargs.pop("eos_token_id", 151645)
self.pad_token_id = kwargs.pop("pad_token_id", 151643)
self.hidden_act = kwargs.pop("hidden_act", "silu")
self.hidden_size = kwargs.pop("hidden_size", 768)
self.num_hidden_layers = kwargs.pop("num_hidden_layers", 8)
self.max_position_embeddings = kwargs.pop("max_position_embeddings", 8192)
self.vocab_size = kwargs.pop("vocab_size", 6400)
self.rms_norm_eps = kwargs.pop("rms_norm_eps", 1e-6)
self.rope_theta = kwargs.pop("rope_theta", 5e4)
self.rope_scaling = kwargs.pop("rope_scaling", None)
self.dtype = kwargs.pop("dtype", "float32")
self.self_distill = kwargs.pop("self_distill", True)
self.intermediate_size = kwargs.pop("intermediate_size", 1536)
self.num_heads = kwargs.pop("num_heads", 12)
self.mla_kv_lora_rank = kwargs.pop("mla_kv_lora_rank", 64)
self.mla_qk_nope_head_dim = kwargs.pop("mla_qk_nope_head_dim", 64)
self.mla_qk_rope_head_dim = kwargs.pop("mla_qk_rope_head_dim", 32)
self.mla_attn_impl = kwargs.pop("mla_attn_impl", "absorb")
self.qkv_lora = kwargs.pop("qkv_lora", False)
self.gradient_checkpointing = kwargs.pop("gradient_checkpointing", 0)
self.use_sengram = kwargs.pop("use_sengram", True)
self.sengram_bucket_size = kwargs.pop("sengram_bucket_size", 4096)
self.sengram_topk = kwargs.pop("sengram_topk", 2)
self.engram_bucket_size = kwargs.pop("engram_bucket_size", self.sengram_bucket_size)
self.engram_topk = kwargs.pop("engram_topk", self.sengram_topk)
super().__init__(
bos_token_id=self.bos_token_id,
eos_token_id=self.eos_token_id,
pad_token_id=self.pad_token_id,
**kwargs,
)
|