Upload gpt2_reference/config.yaml with huggingface_hub

gpt2_reference/config.yaml

@@ -1,37 +1,48 @@
-# GPT-2 124M Reference — reproduce original GPT-2 for baseline comparison
-# Standard MHA + GELU FFN + LayerNorm
-
 d_model: 768
 n_layers: 12
 vocab_size: 50257
-
 attention:
   type: mha
   n_heads: 12
+  n_kv_heads: null
+  kv_compression_dim: 256
+  q_compression_dim: 384
+  rope_dim: 64
   dropout: 0.0
-
 ffn:
   type: gelu
+  hidden_mult: 2.6666666666666665
+  n_experts: 8
+  top_k: 2
+  shared_experts: 1
+  load_balance_weight: 0.01
   dropout: 0.0
-
 position:
-  type: rope  # Upgrade from sinusoidal for fair comparison
+  type: rope
   max_seq_len: 1024
   rope_base: 10000.0
-
+  rope_dim: null
 norm:
   type: layernorm
-  eps: 1.0e-5
-
+  eps: 1.0e-05
 output:
   type: tied
-
 training:
-  batch_size: 64
+  batch_size: 8
   seq_len: 1024
-  lr: 6.0e-4
-  min_lr: 6.0e-5
+  lr: 0.0006
+  min_lr: 6.0e-05
   warmup_steps: 1000
   max_steps: 50000
   weight_decay: 0.1
   grad_clip: 1.0
+  beta1: 0.9
+  beta2: 0.95
+  dataset: HuggingFaceFW/fineweb-edu
+  tokenizer: gpt2
+  log_interval: 10
+  eval_interval: 500
+  save_interval: 2500
+  eval_steps: 100
+embed_dropout: 0.0
+residual_dropout: 0.0