| { |
| "mode": "pretrain", |
| "dataset": "HuggingFaceFW/fineweb", |
| "dataset_config": "sample-10BT", |
| "split": "train", |
| "text_field": "text", |
| "model_size": "small", |
| "block_size": 512, |
| "batch_size": 4, |
| "grad_accum": 8, |
| "max_steps": 20000, |
| "lr": 0.0003, |
| "min_lr": 3e-05, |
| "warmup_steps": 100, |
| "weight_decay": 0.1, |
| "grad_clip": 1.0, |
| "dropout": 0.1, |
| "eval_interval": 1000, |
| "eval_batches": 20, |
| "save_interval": 1000, |
| "log_interval": 50, |
| "pcs_a": 0.8309193524478643, |
| "pcs_b": 0.0, |
| "out_dir": "runs/latest", |
| "hub_repo_id": "Daisuke675/scugnizz", |
| "push": true, |
| "push_every_save": true, |
| "seed": 42, |
| "dtype": "auto", |
| "no_resume": false, |
| "no_sample": true, |
| "prompt": "User: What is artificial intelligence?\nAssistant:", |
| "max_new_tokens": 120 |
| } |
