File size: 1,115 Bytes
76cdc5c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
{
  "architecture": "HNet",
  "config": {
    "arch_layout": [
      "m4",
      [
        "T12"
      ],
      "m4"
    ],
    "d_model": [
      512,
      768
    ],
    "d_intermediate": [
      0,
      2048
    ],
    "vocab_size": 256,
    "ssm_cfg": {
      "chunk_size": 256,
      "d_conv": 4,
      "d_state": 128,
      "expand": 2
    },
    "attn_cfg": {
      "num_heads": [
        8,
        12
      ],
      "rotary_emb_dim": [
        32,
        32
      ],
      "window_size": [
        1023,
        -1
      ]
    },
    "tie_embeddings": false
  },
  "iter": 72799,
  "n_params": 99765120,
  "tied_weights": {},
  "training_args": {
    "config": "configs/hnet_small.json",
    "block_size": 8192,
    "micro_bs": 2,
    "grad_accum": 16,
    "max_iters": 72800,
    "lr": 0.0006,
    "lr_mult": "2.0,1.0",
    "min_lr_frac": 0.1,
    "warmup_iters": 2000,
    "weight_decay": 0.1,
    "grad_clip": 1.0,
    "target_n": 4.6,
    "ratio_weight": 0.03,
    "eval_every": 500,
    "eval_windows": 64,
    "ckpt_every": 1000,
    "snap_every": 8000,
    "log_every": 20,
    "seed": 1337
  }
}