| { |
| "vocab_size": 16000, |
| "d_model": 576, |
| "n_layers": 14, |
| "d_state": 16, |
| "d_conv": 4, |
| "expand": 2, |
| "dt_rank": 72, |
| "dt_min": 0.001, |
| "dt_max": 0.1, |
| "dt_init": "random", |
| "dt_scale": 1.0, |
| "scan_chunk_size": 32, |
| "scan_impl": "segmented", |
| "scan_backend": "torch", |
| "ssm_scan_in_fp32": true, |
| "scan_segment_size": 2048, |
| "ssm_state_schedule": [ |
| 16, |
| 32, |
| 64 |
| ], |
| "dropout": 0.1, |
| "bias": false, |
| "activation": "silu", |
| "norm_type": "rmsnorm", |
| "norm_eps": 1e-06, |
| "norm_bias": false, |
| "max_seq_len": 2048, |
| "pad_token_id": 100258, |
| "model_kind": "causal_lm", |
| "input_modality": "text", |
| "output_modality": "text", |
| "tie_word_embeddings": true, |
| "initializer_range": 0.02, |
| "residual_scale_init": true, |
| "residual_in_fp32": true, |
| "gradient_checkpointing": false, |
| "model_alias": "SIGER", |
| "use_moe": true, |
| "moe_num_experts": 8, |
| "moe_top_k": 2, |
| "moe_expert_hidden_mult": 3, |
| "moe_layers_every": 2, |
| "moe_aux_loss_weight": 0.01, |
| "moe_router_jitter": 0.01, |
| "moe_z_loss_weight": 0.001, |
| "moe_hierarchy_enabled": true, |
| "moe_hierarchy_branching": 2, |
| "moe_hierarchy_aux_weight": 0.25, |
| "use_hybrid_attention": true, |
| "attn_layers_every": 7, |
| "attn_num_heads": 8, |
| "attn_window_size": 2048, |
| "use_rope": true, |
| "rope_theta": 1000000.0, |
| "attn_global_every": 6, |
| "attn_allow_global": false, |
| "attn_query_chunk_size": 256, |
| "attn_runtime_cache_limit": 16, |
| "rope_scaling_factor": 8.0, |
| "long_context_mode": true, |
| "stream_chunk_size": 256, |
| "memory_slots": 256, |
| "memory_top_k": 8, |
| "logits_chunk_size": 256, |
| "stream_detach_state": true, |
| "use_mtp": true, |
| "mtp_num_heads": 2, |
| "mtp_loss_weight": 0.0, |
| "model_base_name": "SIGER" |
| } |
