model_config.txt

Model: TinyLM-10M
vocab_size: 2000
dim: 384
n_layers: 4
n_heads: 4
n_kv_heads: 4
ffn_dim: 1536
max_seq_len: 256
norm_eps: 1e-6
rope_theta: 10000.0
architecture: Decoder-only Transformer (GQA + RoPE + RMSNorm + SwiGLU)
tokenizer: tinystories_10m_tokenizer.json
dataset: TinyStories (roneneldan/TinyStories)
training_steps: 50000
batch_size: 64
learning_rate: 3e-4
optimizer: AdamW (betas=0.9,0.95, weight_decay=0.01)

---

Model: TinyLM-7M
vocab_size: 1500
dim: 224
n_layers: 8
n_heads: 4
n_kv_heads: 4
ffn_dim: 896
max_seq_len: 256
norm_eps: 1e-6
rope_theta: 10000.0
architecture: Decoder-only Transformer (GQA + RoPE + RMSNorm + SwiGLU)
tokenizer: tinystories_7m_tokenizer.json
dataset: TinyStories (roneneldan/TinyStories)
training_steps: 50000
batch_size: 64
learning_rate: 3e-4
optimizer: AdamW (betas=0.9,0.95, weight_decay=0.01)

---

Model: TinyLM-5M
vocab_size: 1000
dim: 384
n_layers: 2
n_heads: 4
n_kv_heads: 4
ffn_dim: 1536
max_seq_len: 256
norm_eps: 1e-6
rope_theta: 10000.0
architecture: Decoder-only Transformer (GQA + RoPE + RMSNorm + SwiGLU)
tokenizer: tinystories_5m_tokenizer.json
dataset: TinyStories (roneneldan/TinyStories)
training_steps: 50000
batch_size: 64
learning_rate: 3e-4
optimizer: AdamW (betas=0.9,0.95, weight_decay=0.01)

---

Model: TinyLM-2.5M
vocab_size: 512
dim: 272
n_layers: 2
n_heads: 4
n_kv_heads: 4
ffn_dim: 1088
max_seq_len: 256
norm_eps: 1e-6
rope_theta: 10000.0
architecture: Decoder-only Transformer (GQA + RoPE + RMSNorm + SwiGLU)
tokenizer: tinystories_2_5m_tokenizer.json
dataset: TinyStories (roneneldan/TinyStories)
training_steps: 50000
batch_size: 64
learning_rate: 3e-4
optimizer: AdamW (betas=0.9,0.95, weight_decay=0.01)