File size: 811 Bytes
1c016ea
9b4717b
a10c7e3
9b4717b
 
 
a10c7e3
9b4717b
 
 
1c016ea
9b4717b
1c016ea
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9b4717b
1c016ea
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
{
  "step": 7000,
  "max_steps": 28500,
  "percent_complete": 24.561,
  "tokens_seen": 17301504000,
  "compute_tokens_seen": 2883584000,
  "compute_tokens_total": 14942208000,
  "percent_tokens": 19.298,
  "val_loss": 6.348879925906658,
  "val_ppl": 571.8518337425736,
  "params_M": 149.976,
  "pushed_at": "2026-11-15T09:39:11Z",
  "architecture": "Oleander-150M",
  "features": [
    "GDN2 (gated delta net 2) x21: 9h x 64, conv4",
    "gated GQA x7: 9 Q / 3 KV heads, head_dim 64",
    "QK-Norm (offset form)",
    "partial RoPE (rotary_dim 32, theta 100000)",
    "value residual (v_mix)",
    "SwiGLU",
    "RMSNorm",
    "AttnRes (block attention residuals)",
    "tied embeddings",
    "TST (token superposition training, bag 6)"
  ],
  "optimizer": "muon_adamw",
  "phase": "tst",
  "kind": "round 14"
}