Update config.json

config.json

@@ -1,7 +1,37 @@
 {
-    "d_model": 1808,
-    "n_head": 16,
-    "n_layer": 38,
-    "residual_in_fp32": true,
-    "pad_vocab_size_multiple": 8
-}
+        "n_embd": 1792,
+        "n_inner": 3584,
+        "n_head": 16,
+        "n_layer": 36,
+
+        "activation_function": "swiglu",
+        "resid_pdrop": 0.0,
+        "residual_in_fp32": True,
+        "pad_vocab_size_multiple": 8,
+        "use_flash_attn": True,
+        "special_initializer": True,
+        "max_position_embeddings": 0,
+
+        "alt_mixer_layers": [1, 6, 11, 16, 21, 27, 33],
+        "alt_2_mixer_layers": [2, 7, 12, 17, 22, 28, 34],
+        "mixer": {
+            "_target_": "based.models.mixers.base_conv.BaseConvWithSiLU4",
+            "expand_proj": 4,
+            "l_max": 2048,
+            "kernel_sizes": [3],
+        },
+        "alt_mixer": {
+            "_target_": "based.models.mixers.linear_attn.LinearAttention",
+            "feature_dim": 16,
+            "l_max": 2048,
+            "num_heads": 16,
+            "num_key_value_heads": 16,
+            "train_view": "linear",
+        },
+        "alt_mixer_2": {
+            "_target_": "based.models.mixers.slide_fa2.SlidingsMHA",
+            "causal": True,
+            "num_heads": 16,
+            "window_size": 128,
+        }
+    }