| { |
| "architecture_options": { |
| "attention_backend": "flash_2", |
| "attention_bias": false, |
| "attn_qk_norm": true, |
| "d_model": 1536, |
| "enable_swa": true, |
| "ffn_bias": false, |
| "ffn_hidden": 6144, |
| "gdn_allow_neg_eigval": true, |
| "gdn_conv_bias": false, |
| "gdn_conv_size": 4, |
| "gdn_expand_v": 2.0, |
| "head_dim": 128, |
| "layer_norm_bias": false, |
| "layer_norm_eps": 1e-05, |
| "lm_head_bias": false, |
| "n_heads": 12, |
| "n_kv_heads": 4, |
| "n_layers": 26, |
| "rope_full_precision": true, |
| "rope_theta": 500000, |
| "swa_full_attention_first_layer": false, |
| "swa_full_attention_last_layer": false, |
| "swa_window": 4096, |
| "tie_word_embeddings": true |
| }, |
| "architecture_preset": "main", |
| "architectures": [ |
| "NovaForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "configuration_nova.NovaConfig", |
| "AutoModel": "modeling_nova.NovaModel", |
| "AutoModelForCausalLM": "modeling_nova.NovaForCausalLM" |
| }, |
| "bos_token_id": 1, |
| "compute_dtype": "bfloat16", |
| "dtype": "float32", |
| "eos_token_id": 2, |
| "fast_decode": true, |
| "generation_context_policy": "remaining", |
| "hidden_size": 1536, |
| "inference_backend": "torch", |
| "max_position_embeddings": 8192, |
| "model_type": "nova_gdn2", |
| "num_hidden_layers": 26, |
| "pad_token_id": 0, |
| "tokenizer_vocab_size": 65536, |
| "transformers_version": "4.57.6", |
| "use_cache": false, |
| "vocab_size": 65536 |
| } |
|
|
