File size: 2,311 Bytes
44928d5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | """CodVa-1 configuration for HuggingFace Transformers compatibility."""
from transformers import PretrainedConfig
class CodVa1Config(PretrainedConfig):
model_type = "codva1"
def __init__(
self,
vocab_size: int = 32064,
d_model: int = 1536,
n_layers: int = 28,
n_heads: int = 24,
n_kv_heads: int = 6,
max_len: int = 2048,
rope_theta: float = 5_000_000.0,
ffn_hidden: int = 4096,
use_moe: bool = True,
moe_experts: int = 16,
moe_top_k: int = 2,
moe_shared: int = 2,
moe_hidden: int = 1024,
moe_every: int = 2,
use_qk_norm: bool = True,
use_structural_bias: bool = True,
n_struct_rel: int = 4,
rms_norm_eps: float = 1e-6,
tie_word_embeddings: bool = True,
fim_pre_id: int = -1,
fim_suf_id: int = -1,
fim_mid_id: int = -1,
**kwargs,
):
self.vocab_size = vocab_size
self.d_model = d_model
self.n_layers = n_layers
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.max_len = max_len
self.rope_theta = rope_theta
self.ffn_hidden = ffn_hidden
self.use_moe = use_moe
self.moe_experts = moe_experts
self.moe_top_k = moe_top_k
self.moe_shared = moe_shared
self.moe_hidden = moe_hidden
self.moe_every = moe_every
self.use_qk_norm = use_qk_norm
self.use_structural_bias = use_structural_bias
self.n_struct_rel = n_struct_rel
self.rms_norm_eps = rms_norm_eps
# FIM special token ids (padded into vocab during training)
self.fim_pre_id = fim_pre_id
self.fim_suf_id = fim_suf_id
self.fim_mid_id = fim_mid_id
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
# Newer `transformers` internals (cache utils, generation config, etc.)
# look for these standard attribute names regardless of custom naming,
# even when the model declares no cache support. Alias them through.
@property
def num_hidden_layers(self):
return self.n_layers
@property
def num_attention_heads(self):
return self.n_heads
@property
def hidden_size(self):
return self.d_model |