| { | |
| "dataset_id": "openbmb/Ultra-FineWeb-L1", | |
| "train_config": "CC-MAIN-2025-51", | |
| "val_config": "CC-MAIN-2025-47", | |
| "text_field": "content", | |
| "token_budget": 600000000, | |
| "val_docs": 512, | |
| "tokenizer_fit_docs": 4000, | |
| "vocab_size": 16384, | |
| "block_size": 1024, | |
| "n_layer": 8, | |
| "n_head": 8, | |
| "n_embd": 512, | |
| "dropout": 0.1, | |
| "batch_size": 256, | |
| "learning_rate": 0.0006, | |
| "min_lr_ratio": 0.1, | |
| "warmup_steps": 200, | |
| "weight_decay": 0.1, | |
| "grad_clip": 1.0, | |
| "seed": 1337, | |
| "checkpoint_every_steps": 200, | |
| "keep_checkpoints": 3, | |
| "log_every_steps": 20, | |
| "amp_bf16": true, | |
| "hub_checkpoint_repo": "ProCreations/booper-pretrain", | |
| "hub_private": false | |
| } | |