| {"bos_token_id": 128000, "eos_token_id": 128001, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "is_qwen2_config": true, "max_position_embeddings": 131072, "num_attention_heads": 32, "num_hidden_layers": 16, "num_key_value_heads": 8, "pad_token_id": null, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": {"factor": 32.0, "high_freq_factor": 4.0, "low_freq_factor": 1.0, "original_max_position_embeddings": 8192, "rope_type": "llama3"}, "rope_theta": 500000.0, "rope_interleaved": false, "rope_seq_len_interpolation_factor": null, "tie_word_embeddings": true, "use_cache": true, "vocab_size": 128256, "_attn_implementation": "flash_attention_2", "flex_attention_mask": null, "attention_bias": false, "sliding_window_size": null, "z_loss_enabled": false, "z_loss_coefficient": 0.0001, "no_rope_layer": null, "_fused_rotary_emb": true, "_fused_rms_norm": true, "_use_qkv_packed": true, "_use_doc_masking": false, "log_attn_probs": false, "ring_attn_heads_k_stride": null, "moe_config": null} |