Tetu-Code-2 / config.json
tetu-AI's picture
Initial release
2309cb1 verified
Raw History Blame Contribute Delete
674 Bytes
{
"name": "Têtu Code 2",
"step": 910,
"tokens": null,
"config": {
"vocab_size": 32768,
"d_model": 1152,
"n_layers": 28,
"n_heads": 18,
"n_kv_heads": 6,
"mlp_mult": 4,
"max_seq_len": 65536,
"window": 2048,
"global_every": 4,
"rope_local": 10000.0,
"rope_global": 500000.0,
"log_base": 4096,
"engram": [
[
1,
[
2,
3
]
],
[
9,
[
3,
4
]
],
[
17,
[
4,
6
]
]
],
"engram_heads": 2,
"engram_slots": 393209,
"engram_dim": 128,
"n_value_embeds": 3,
"struct_sizes": [
16,
16,
4,
2
],
"softcap": 30.0,
"doubt_weight": 0.1,
"eos_id": 1
}
}