init:
  init_type: mup
  std_embed: 0.02
  std_o: 0.01
  std_qkv: 0.02
loss:
  label_smoothing: 0.0
  type: next_token_ce
model:
  attn_scale: 0.35
  attn_type: quadratic
  d_model: 16
  n_ctx: 16
  n_head: 2
  n_layers: 2
  norm_places:
  - pre_unembed
  norm_type: none
  rope_base: 10000
  use_bias_qk: false
  use_rmsnorm_qk: false
  vocab_size: 16
name: big_experiment
seed: 42
train:
  batch_size: 128
  betas:
  - 0.9
  - 0.95
  checkpoint_every: 60
  checkpoint_linear_count: 20
  checkpoint_log_count: 0
  checkpoint_schedule: interval
  debug: true
  dtype: bfloat16
  eval_every: 100
  grad_clip: 1.0
  lr: 0.0003
  lr_decay_frac: 1.0
  max_steps: 6000
  muon_lr: 0.03
  use_muon: true
  warmup_steps: 500
  weight_decay: 0.01