| """ | |
| Quark model configuration — compatibile con AutoConfig / AutoModel HuggingFace. | |
| """ | |
| from transformers import PretrainedConfig | |
| class QuarkConfig(PretrainedConfig): | |
| model_type = "quark" | |
| # Spezza la ricorsione in to_diff_dict(): senza questo, transformers tenta | |
| # di costruire self.__class__() per il diff, che richiama __init__ -> __repr__ | |
| # -> to_diff_dict() -> self.__class__() all'infinito. | |
| has_no_defaults_at_init = True | |
| def __init__( | |
| self, | |
| vocab_size = 65_536, | |
| d_model = 512, | |
| n_heads = 8, | |
| n_kv_heads = 2, | |
| n_layers = 14, | |
| d_ff = 1344, | |
| head_dim = 64, | |
| max_seq_len = 2048, | |
| rope_theta = 10_000.0, | |
| rms_eps = 1e-5, | |
| qkv_bias = True, | |
| dropout = 0.0, | |
| tie_word_embeddings = True, | |
| **kwargs, | |
| ): | |
| self.vocab_size = vocab_size | |
| self.d_model = d_model | |
| self.n_heads = n_heads | |
| self.n_kv_heads = n_kv_heads | |
| self.n_layers = n_layers | |
| self.d_ff = d_ff | |
| self.head_dim = head_dim | |
| self.max_seq_len = max_seq_len | |
| self.rope_theta = rope_theta | |
| self.rms_eps = rms_eps | |
| self.qkv_bias = qkv_bias | |
| self.dropout = dropout | |
| super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) | |