Update config.json

config.json

@@ -1,48 +1,48 @@
-config_check = {
-        "n_embd": 1792,
-        "n_inner": 3584,
-        "n_head": 16,
-        "n_layer": 36,
+{
+    "n_embd": 1792,
+    "n_inner": 3584,
+    "n_head": 16,
+    "n_layer": 36,
 
-        "mlp_fc1_bias": False,
-        "mlp_fc2_bias": False,
-        "out_proj_bias": False,
-        "qkv_proj_bias": False,
-        "reorder_and_upcast_attn": False,
-        "scale_attn_by_inverse_layer_idx": False,
+    "mlp_fc1_bias": false,
+    "mlp_fc2_bias": false,
+    "out_proj_bias": false,
+    "qkv_proj_bias": false,
+    "reorder_and_upcast_attn": false,
+    "scale_attn_by_inverse_layer_idx": false,
 
-        "activation_function": "swiglu",
-        "resid_pdrop": 0.0,
-        "rms_norm": True,
-        "residual_in_fp32": True,
-        "pad_vocab_size_multiple": 8,
-        "use_flash_attn": True,
-        "special_initializer": True,
-        "rotary_emb_fraction": 1,
-        "max_position_embeddings": 0,
+    "activation_function": "swiglu",
+    "resid_pdrop": 0.0,
+    "rms_norm": true,
+    "residual_in_fp32": true,
+    "pad_vocab_size_multiple": 8,
+    "use_flash_attn": true,
+    "special_initializer": true,
+    "rotary_emb_fraction": 1,
+    "max_position_embeddings": 0,
 
-        "alt_mixer_layers": [1, 6, 11, 16, 21, 27, 33],
-        "alt_mixer_2_layers": [2, 7, 12, 17, 22, 28, 34],
-        "mixer": {
-            "_target_": "based.models.mixers.base_conv.BaseConvWithSiLU4",
-            "expand_proj": 4,
-            "l_max": 2048,
-            "kernel_sizes": [3],
-            "use_bias": True,
-        },
-        "alt_mixer": {
-            "_target_": "based.models.mixers.linear_attn.LinearAttention",
-            "feature_dim": 16,
-            "feature_name": "taylor_exp",
-            "l_max": 2048,
-            "num_heads": 16,
-            "num_key_value_heads": 16,
-            "train_view": "linear",
-        },
-        "alt_mixer_2": {
-            "_target_": "based.models.mixers.slide_fa2.SlidingsMHA",
-            "causal": True,
-            "num_heads": 16,
-            "window_size": 128,
-        }
-    }
+    "alt_mixer_layers": [1, 6, 11, 16, 21, 27, 33],
+    "alt_mixer_2_layers": [2, 7, 12, 17, 22, 28, 34],
+    "mixer": {
+        "_target_": "based.models.mixers.base_conv.BaseConvWithSiLU4",
+        "expand_proj": 4,
+        "l_max": 2048,
+        "kernel_sizes": [3],
+        "use_bias": true
+    },
+    "alt_mixer": {
+        "_target_": "based.models.mixers.linear_attn.LinearAttention",
+        "feature_dim": 16,
+        "feature_name": "taylor_exp",
+        "l_max": 2048,
+        "num_heads": 16,
+        "num_key_value_heads": 16,
+        "train_view": "linear"
+    },
+    "alt_mixer_2": {
+        "_target_": "based.models.mixers.slide_fa2.SlidingsMHA",
+        "causal": true,
+        "num_heads": 16,
+        "window_size": 128
+    }
+}