| { | |
| "model": { | |
| "d_model": 1024, | |
| "n_layers": 24, | |
| "n_heads": 24, | |
| "n_kv_heads": 4, | |
| "head_dim": 64, | |
| "vocab": 32768, | |
| "seq_len": 2048, | |
| "d_ff": [ | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464, | |
| 2464 | |
| ], | |
| "memory_layers": [], | |
| "memory": null, | |
| "norm_eps": 1e-05, | |
| "rope_theta": 10000.0, | |
| "init_std": 0.02, | |
| "layers": [ | |
| 0, | |
| 1, | |
| 2, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 7, | |
| 8, | |
| 9, | |
| 10, | |
| 11, | |
| 12, | |
| 13, | |
| 14, | |
| 15, | |
| 16, | |
| 17, | |
| 18, | |
| 19, | |
| 20, | |
| 21, | |
| 22, | |
| 23 | |
| ], | |
| "memory_overrides": {}, | |
| "draft": null, | |
| "ssm": null | |
| }, | |
| "train": { | |
| "seed": 1234, | |
| "data_seed": 1234, | |
| "val_tokens": 20000000, | |
| "global_batch": 256, | |
| "micro_batch": 2, | |
| "tokens_total": 2000000000, | |
| "lr": 0.0006, | |
| "min_lr_ratio": 0.1, | |
| "warmup_frac": 0.02, | |
| "warmup_min_steps": 100, | |
| "betas": [ | |
| 0.9, | |
| 0.95 | |
| ], | |
| "eps": 1e-08, | |
| "weight_decay": 0.1, | |
| "grad_clip": 1.0, | |
| "table_lr_mult": 4.0, | |
| "precision": "fp16", | |
| "act_ckpt": true, | |
| "attn_backend": "sdpa", | |
| "zero1": true, | |
| "ckpt_every": 200, | |
| "keep_last": 3, | |
| "keep_every": 1000, | |
| "eval_every": 200, | |
| "eval_max_batches": null, | |
| "diag_batches": 4, | |
| "log_every": 10, | |
| "scaler_min": 1.0, | |
| "scaler_patience": 100, | |
| "nan_patience": 20, | |
| "save_steps": [ | |
| 1000, | |
| 2000, | |
| 3815 | |
| ], | |
| "strip_old_optimizer": true, | |
| "seq_len": 2048 | |
| }, | |
| "name": "w3_dense270", | |
| "params": { | |
| "expected_non_embedding": null, | |
| "reference": false, | |
| "match_reference": false | |
| }, | |
| "wandb": { | |
| "project": "morbius-wave3" | |
| } | |
| } |