aramkris's picture
tinystories-colab: 5 files, 205 MB
f78fc59 verified
Raw History Blame Contribute Delete
671 Bytes
{
"model_type": "masked-diffusion",
"model": {
"vocab_size": 50304,
"seq_len": 256,
"d_model": 512,
"n_layers": 8,
"n_heads": 8,
"d_ff": 2048,
"dropout": 0.0,
"diffusion_steps": 64,
"mask_id": 50257,
"tokenizer": "gpt2",
"arch": "v1"
},
"train": {
"steps": 20000,
"batch_size": 32,
"grad_accum": 1,
"lr": 0.0006,
"min_lr": 6e-05,
"warmup_steps": 1000,
"weight_decay": 0.1,
"grad_clip": 1.0,
"eval_interval": 1000,
"eval_batches": 40,
"sample_interval": 5000,
"seed": 42,
"compile": true,
"snapshot_every": 0
},
"datasets": [
"roneneldan/TinyStories"
]
}