OOOPS. Forgot to add model config

model_config.yaml

@@ -0,0 +1,58 @@
+name: multiscreen-pure-9l256-tadam-4b
+
+block_size: 1024
+vocab_size: 4096
+padded_vocab_size: 4096
+
+n_layer: 9
+n_embd: 256
+n_head: 8
+head_size: 32
+n_query_groups: 1
+
+intermediate_size: 1104
+
+bias: false
+attn_bias: false
+
+norm_class_name: RMSNorm
+norm_eps: 1.0e-05
+parallel_residual: false
+shared_attention_norm: false
+post_attention_norm: false
+post_mlp_norm: false
+
+mlp_class_name: TileRoutedEulerGLUMLP
+
+sparse_mlp_num_groups: 4
+sparse_mlp_min_active_groups: 4
+sparse_mlp_max_active_groups: 4
+
+multiscreen_enabled: true
+multiscreen_num_heads: 8
+multiscreen_key_dim: 16
+multiscreen_value_dim: 128
+multiscreen_query_chunk_size: 128
+multiscreen_window_threshold: 256
+multiscreen_landmark_stride: 1
+multiscreen_layer_interval: 0
+multiscreen_mlp_enabled: false
+multiscreen_triton_inference: true
+
+sliding_window_size: 256
+sliding_window_indices:
+  - 1
+  - 1
+  - 1
+  - 0
+  - 1
+  - 1
+  - 1
+  - 0
+  - 1
+
+rotary_percentage: 1.0
+rope_base: 10000
+
+lm_head_bias: false
+scale_embeddings: false