{ "model_type": "masked-diffusion", "model": { "vocab_size": 50304, "seq_len": 256, "d_model": 512, "n_layers": 8, "n_heads": 8, "d_ff": 2048, "dropout": 0.0, "diffusion_steps": 64, "mask_id": 50257, "tokenizer": "gpt2", "arch": "v1" }, "train": { "steps": 20000, "batch_size": 32, "grad_accum": 1, "lr": 0.0006, "min_lr": 6e-05, "warmup_steps": 1000, "weight_decay": 0.1, "grad_clip": 1.0, "eval_interval": 1000, "eval_batches": 40, "sample_interval": 5000, "seed": 42, "compile": true, "snapshot_every": 0 }, "datasets": [ "roneneldan/TinyStories" ] }