File size: 671 Bytes
f78fc59 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | {
"model_type": "masked-diffusion",
"model": {
"vocab_size": 50304,
"seq_len": 256,
"d_model": 512,
"n_layers": 8,
"n_heads": 8,
"d_ff": 2048,
"dropout": 0.0,
"diffusion_steps": 64,
"mask_id": 50257,
"tokenizer": "gpt2",
"arch": "v1"
},
"train": {
"steps": 20000,
"batch_size": 32,
"grad_accum": 1,
"lr": 0.0006,
"min_lr": 6e-05,
"warmup_steps": 1000,
"weight_decay": 0.1,
"grad_clip": 1.0,
"eval_interval": 1000,
"eval_batches": 40,
"sample_interval": 5000,
"seed": 42,
"compile": true,
"snapshot_every": 0
},
"datasets": [
"roneneldan/TinyStories"
]
} |