| { | |
| "architecture": "LoopedTransformer (loopscape.linsys.model)", | |
| "n_embd": 256, | |
| "n_head": 8, | |
| "n_layer_shared": 1, | |
| "mlp_ratio": 1.0, | |
| "vocab_size": 32, | |
| "block_size": 256, | |
| "positional_encoding": "rope", | |
| "causal": false, | |
| "update_rule": "x <- LN(x + step_size * (block(x + e) - (x + e)))", | |
| "step_size": 1.0, | |
| "init_std": 0.1, | |
| "init_slots_only": true, | |
| "default_loops": 16, | |
| "max_grad_loops": 8, | |
| "task": { | |
| "field": "GF(3)", | |
| "n_unknowns": 8, | |
| "n_rows": 8, | |
| "full_rank": true, | |
| "row_weight_kmax": 8, | |
| "test_frac": 0.05 | |
| }, | |
| "training": { | |
| "steps": 300000, | |
| "batch_size": 1024, | |
| "lr": 0.0002, | |
| "schedule": "wsd, 2000 warmup, 5% 1-sqrt cooldown", | |
| "optimizer": "AdamW betas (0.9, 0.99), weight_decay 1e-3 on matrices", | |
| "loops": "lognormal-Poisson mean 16 sigma 0.3", | |
| "precision": "bf16 autocast", | |
| "seed": 2, | |
| "source_run": "pk8_b1024 step 298000" | |
| } | |
| } |