{
  "dataset_id": "openbmb/Ultra-FineWeb-L1",
  "train_config": "CC-MAIN-2025-51",
  "val_config": "CC-MAIN-2025-47",
  "text_field": "content",
  "token_budget": 600000000,
  "val_docs": 512,
  "tokenizer_fit_docs": 4000,
  "vocab_size": 16384,
  "block_size": 1024,
  "n_layer": 8,
  "n_head": 8,
  "n_embd": 512,
  "dropout": 0.1,
  "batch_size": 256,
  "learning_rate": 0.0006,
  "min_lr_ratio": 0.1,
  "warmup_steps": 200,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "seed": 1337,
  "checkpoint_every_steps": 200,
  "keep_checkpoints": 3,
  "log_every_steps": 20,
  "amp_bf16": true,
  "hub_checkpoint_repo": "ProCreations/booper-pretrain",
  "hub_private": false
}