File size: 620 Bytes
d0e32f5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
{
"architectures": [
"TinyLlama"
],
"model_type": "brandon-tiny",
"dim": 256,
"n_layers": 24,
"n_heads": 8,
"n_kv_heads": 2,
"vocab_size": 8192,
"hidden_dim": 720,
"max_seq_len": 512,
"dropout": 0.05,
"weight_tying": true,
"norm_eps": 1e-05,
"rope_theta": 10000.0,
"block_sharing": true,
"n_predict": 1,
"dense_former": true,
"value_residual": true,
"n_registers": 4,
"n_loops": 1,
"ternary": false,
"activation": "swiglu",
"normalization": "rmsnorm",
"position_encoding": "rope",
"chat_format": "chatml",
"total_parameters": 10706776
} |