File size: 674 Bytes
2309cb1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | {
"name": "Têtu Code 2",
"step": 910,
"tokens": null,
"config": {
"vocab_size": 32768,
"d_model": 1152,
"n_layers": 28,
"n_heads": 18,
"n_kv_heads": 6,
"mlp_mult": 4,
"max_seq_len": 65536,
"window": 2048,
"global_every": 4,
"rope_local": 10000.0,
"rope_global": 500000.0,
"log_base": 4096,
"engram": [
[
1,
[
2,
3
]
],
[
9,
[
3,
4
]
],
[
17,
[
4,
6
]
]
],
"engram_heads": 2,
"engram_slots": 393209,
"engram_dim": 128,
"n_value_embeds": 3,
"struct_sizes": [
16,
16,
4,
2
],
"softcap": 30.0,
"doubt_weight": 0.1,
"eos_id": 1
}
} |