File size: 1,408 Bytes
92a7d9d c54f415 92a7d9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """
Quark model configuration — compatibile con AutoConfig / AutoModel HuggingFace.
"""
from transformers import PretrainedConfig
class QuarkConfig(PretrainedConfig):
model_type = "quark"
# Spezza la ricorsione in to_diff_dict(): senza questo, transformers tenta
# di costruire self.__class__() per il diff, che richiama __init__ -> __repr__
# -> to_diff_dict() -> self.__class__() all'infinito.
has_no_defaults_at_init = True
def __init__(
self,
vocab_size = 65_536,
d_model = 512,
n_heads = 8,
n_kv_heads = 2,
n_layers = 14,
d_ff = 1344,
head_dim = 64,
max_seq_len = 2048,
rope_theta = 10_000.0,
rms_eps = 1e-5,
qkv_bias = True,
dropout = 0.0,
tie_word_embeddings = True,
**kwargs,
):
self.vocab_size = vocab_size
self.d_model = d_model
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.n_layers = n_layers
self.d_ff = d_ff
self.head_dim = head_dim
self.max_seq_len = max_seq_len
self.rope_theta = rope_theta
self.rms_eps = rms_eps
self.qkv_bias = qkv_bias
self.dropout = dropout
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|