Hugging Face's logo

GilpinLab
/
linsys-gf3

looped-transformer
linear-systems
dynamical-systems
Model card Files
xet
 Community
linsys-gf3
File size: 922 Bytes
54db2f6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
{
  "architecture": "LoopedTransformer (loopscape.linsys.model)",
  "n_embd": 256,
  "n_head": 8,
  "n_layer_shared": 1,
  "mlp_ratio": 1.0,
  "vocab_size": 32,
  "block_size": 256,
  "positional_encoding": "rope",
  "causal": false,
  "update_rule": "x <- LN(x + step_size * (block(x + e) - (x + e)))",
  "step_size": 1.0,
  "init_std": 0.1,
  "init_slots_only": true,
  "default_loops": 16,
  "max_grad_loops": 8,
  "task": {
    "field": "GF(3)",
    "n_unknowns": 8,
    "n_rows": 8,
    "full_rank": true,
    "row_weight_kmax": 8,
    "test_frac": 0.05
  },
  "training": {
    "steps": 300000,
    "batch_size": 1024,
    "lr": 0.0002,
    "schedule": "wsd, 2000 warmup, 5% 1-sqrt cooldown",
    "optimizer": "AdamW betas (0.9, 0.99), weight_decay 1e-3 on matrices",
    "loops": "lognormal-Poisson mean 16 sigma 0.3",
    "precision": "bf16 autocast",
    "seed": 2,
    "source_run": "pk8_b1024 step 298000"
  }
}