{ "name": "TĂȘtu Code 2", "step": 910, "tokens": null, "config": { "vocab_size": 32768, "d_model": 1152, "n_layers": 28, "n_heads": 18, "n_kv_heads": 6, "mlp_mult": 4, "max_seq_len": 65536, "window": 2048, "global_every": 4, "rope_local": 10000.0, "rope_global": 500000.0, "log_base": 4096, "engram": [ [ 1, [ 2, 3 ] ], [ 9, [ 3, 4 ] ], [ 17, [ 4, 6 ] ] ], "engram_heads": 2, "engram_slots": 393209, "engram_dim": 128, "n_value_embeds": 3, "struct_sizes": [ 16, 16, 4, 2 ], "softcap": 30.0, "doubt_weight": 0.1, "eos_id": 1 } }