File size: 671 Bytes
f78fc59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
{
  "model_type": "masked-diffusion",
  "model": {
    "vocab_size": 50304,
    "seq_len": 256,
    "d_model": 512,
    "n_layers": 8,
    "n_heads": 8,
    "d_ff": 2048,
    "dropout": 0.0,
    "diffusion_steps": 64,
    "mask_id": 50257,
    "tokenizer": "gpt2",
    "arch": "v1"
  },
  "train": {
    "steps": 20000,
    "batch_size": 32,
    "grad_accum": 1,
    "lr": 0.0006,
    "min_lr": 6e-05,
    "warmup_steps": 1000,
    "weight_decay": 0.1,
    "grad_clip": 1.0,
    "eval_interval": 1000,
    "eval_batches": 40,
    "sample_interval": 5000,
    "seed": 42,
    "compile": true,
    "snapshot_every": 0
  },
  "datasets": [
    "roneneldan/TinyStories"
  ]
}