Upload config.yaml with huggingface_hub

config.yaml

@@ -0,0 +1,47 @@
+inherit: configs/base.yaml
+experiment_name: chinchilla_best
+
+model:
+  dim: 2048
+  n_layers: 24
+
+attention:
+  type: gated_gqa
+  num_heads: 16
+  num_kv_heads: 4
+  head_dim: 128
+  qk_norm: true
+
+ffn:
+  type: relu2
+  intermediate_mult: 2.667
+
+residual:
+  type: mhc
+  n_streams: 4
+
+optim:
+  type: muon
+  lr: 3.0e-4              # Adam LR for 1D params/embeddings
+  muon_lr: 0.007
+  normuon: true
+  normuon_beta2: 0.95
+  scheduler: trapezoidal
+  cooldown_fraction: 0.45
+  warmup_steps: 500
+
+training:
+  tokens: 20_000_000_000
+  batch_size: 8              # per-GPU; 131GB peak on H200 at seq_len=4096
+  seq_len: 4096
+  grad_accum_steps: 4        # effective batch = 8*8*4 = 256 seqs = 1.05M tok/step
+  liger: true
+  compile: true
+  checkpoint_every_tokens: 2_000_000_000
+  hf_repo: GoedelMachines/chinchilla-1b-best
+
+data:
+  shard_dir: data/fineweb_edu
+
+logging:
+  wandb_enabled: true