| from __future__ import annotations |
|
|
| from dataclasses import dataclass |
| from pathlib import Path |
|
|
| import yaml |
|
|
|
|
| @dataclass |
| class BoccaccioConfig: |
| vocab_size: int = 32000 |
| hidden_size: int = 1536 |
| num_layers: int = 26 |
| num_attention_heads: int = 12 |
| num_kv_heads: int = 4 |
| intermediate_size: int = 4096 |
| max_position_embeddings: int = 2048 |
| activation_function: str = "swiglu" |
| positional_embedding: str = "rope" |
| rope_theta: float = 10000.0 |
| norm_type: str = "rmsnorm" |
| norm_eps: float = 1e-5 |
| attention_dropout: float = 0.0 |
| hidden_dropout: float = 0.0 |
| embedding_dropout: float = 0.0 |
| initializer_range: float = 0.02 |
| tie_word_embeddings: bool = True |
|
|
| def __post_init__(self) -> None: |
| self.validate() |
|
|
| @property |
| def head_dim(self) -> int: |
| return self.hidden_size // self.num_attention_heads |
|
|
| def validate(self) -> None: |
| if self.hidden_size % self.num_attention_heads != 0: |
| raise ValueError( |
| f"hidden_size ({self.hidden_size}) must be divisible by " |
| f"num_attention_heads ({self.num_attention_heads})" |
| ) |
| if self.num_attention_heads % self.num_kv_heads != 0: |
| raise ValueError( |
| f"num_attention_heads ({self.num_attention_heads}) must be divisible by " |
| f"num_kv_heads ({self.num_kv_heads})" |
| ) |
|
|
| @classmethod |
| def from_yaml(cls, path: str, variant: str = "model") -> BoccaccioConfig: |
| raw = Path(path).read_text(encoding="utf-8") |
| data = yaml.safe_load(raw) |
| section = data[variant] |
| fields = {f.name for f in cls.__dataclass_fields__.values()} |
| kwargs = {k: v for k, v in section.items() if k in fields} |
| return cls(**kwargs) |
|
|
| @classmethod |
| def nano(cls) -> BoccaccioConfig: |
| return cls( |
| hidden_size=256, |
| num_layers=4, |
| num_attention_heads=8, |
| num_kv_heads=2, |
| intermediate_size=704, |
| max_position_embeddings=512, |
| ) |
|
|
