PhiloLM / config.json
gustavecortal's picture
Unix line endings
1b603cd verified
Raw History Blame Contribute Delete
984 Bytes
{
"model": {
"blocks": 16,
"width": 576,
"query_heads": 9,
"kv_heads": 9,
"global_kv_heads": 3,
"head_dim": 64,
"ffn_intermediate": 1512,
"rmsnorm_epsilon": 1e-06,
"rope_base": 10000,
"rope_dimensions": 32,
"context_tokens": 4096,
"tie_embeddings": true,
"linear_bias": false,
"initialization_std": 0.02,
"residual_projection_initialization_scale": "1/sqrt(2*blocks)",
"qk_norm": true,
"attention_gate": true,
"local_window": 256,
"global_every": 4,
"ngram": true,
"ngram_orders": [
2
],
"ngram_rows": 245760,
"ngram_dim": 128,
"ngram_heads": null,
"ngram_layers": null,
"key_offset": true,
"document_mask": true
},
"vocab_size": 32768,
"eos_id": 0,
"architectures": [
"PhiloLMForCausalLM"
],
"model_type": "philolm",
"auto_map": {
"AutoConfig": "configuration_philolm.PhiloLMConfig",
"AutoModelForCausalLM": "modeling_philolm.PhiloLMForCausalLM"
},
"bos_token_id": 0,
"eos_token_id": 0,
"pad_token_id": 0
}