ereniko commited on
Commit
e976a47
·
verified ·
1 Parent(s): 0c4aba4

Delete config.py

Browse files
Files changed (1) hide show
  1. config.py +0 -29
config.py DELETED
@@ -1,29 +0,0 @@
1
- from dataclasses import dataclass
2
-
3
-
4
- @dataclass
5
- class IvmeConfig:
6
- """Ivme-Conversate-v2 (Dense) architecture config.
7
-
8
- Every field here corresponds to a decision in Section 4 of the design doc.
9
- Values are chosen to match v1 wherever the doc calls for it, so any quality
10
- difference between v1 and v2 is attributable to data/training, not size.
11
- """
12
-
13
- vocab_size: int = 16_000 # Section 4.9: 16k tokens, English-only
14
- hidden_dim: int = 384 # Section 4.2: matches v1
15
- n_layers: int = 10 # Section 4.3: matches v1
16
- n_heads: int = 6 # Section 4.4: full attention, no GQA
17
- context_len: int = 1024 # Section 4.10: matches v1
18
- ffn_mult: float = 4.0 # SwiGLU hidden expansion (adjusted below for param parity)
19
- rope_theta: float = 10_000.0 # standard RoPE base frequency
20
- norm_eps: float = 1e-5 # RMSNorm epsilon
21
- tie_embeddings: bool = True # Section 4.8
22
- dropout: float = 0.0 # no dropout at this data:param ratio (heavily overtrained regime)
23
-
24
- def __post_init__(self):
25
- assert self.hidden_dim % self.n_heads == 0, "hidden_dim must be divisible by n_heads"
26
-
27
- @property
28
- def head_dim(self) -> int:
29
- return self.hidden_dim // self.n_heads