long_warmup/config.json

{
  "data": "data/pile_10B_seed1234.bin",
  "out_dir": "checkpoints/long_warmup",
  "model_size": "31M",
  "readout_lr_mult": 1.0,
  "peak_lr": 0.001,
  "weight_decay": 0.1,
  "wu_weight_decay": -1.0,
  "warmup_steps": 2860,
  "global_batch": 1024,
  "micro_batch": 8,
  "max_tokens": 10000000000.0,
  "max_steps": 0,
  "max_hours": 11.0,
  "val_blocks": 256,
  "seed": 0,
  "data_seed": 1234,
  "precision": "fp16",
  "log_every": 20,
  "latest_every": 100,
  "n_params": 30494720,
  "tokens_per_step": 2097152,
  "wu_weight_decay_effective": 0.1
}