Update configuration_super_linear.py

configuration_super_linear.py

@@ -30,24 +30,21 @@ class SuperLinearConfig(PretrainedConfig):
         pred_len=96,
         inf_pred_len=96,
         max_horizon=96,
-        moe_n_experts=8,
+        moe_n_experts=12,
         top_k_experts=5,
         moe =1,
-        freq_experts='mean_naive_1/6_1/7_1/8_1/12_1/14_1/16_1/21_1/24_1/28_1/30_1/32_1/36_1/42_1/48_1/52_1/56_1/60_1/72_1/84_1/96_1/120_1/144_1/168_1/180_1/224_1/252_1/288_1/336_1/365_1/504_1/672_1/1008_1/1440_1/2016_1/3600',
+        freq_experts= 'mean_naive_1/4_1/6_1/7_1/8_1/12_1/14_1/16_1/21_1/24_1/28_1/30_1/32_1/36_1/42_1/48_1/52_1/56_1/60_1/72_1/84_1/90_1/96_1/120_1/144_1/168_1/180_1/224_1/252_1/288_1/336_1/365_1/504_1/672_1/1008_1/1440_1/2016_1/3600',
         auto_regressive= 1,
-        con= 0,
         d_model= 128,
         dropout= 0.0,
-        fft_len= 10000,
+        fft_len= 5000,
         freeze_experts= 1,
-        ker_len= 50,
         layer_type= "RLinear",
         linear_checkpoints_dir= "checkpoints5",
         linear_checkpoints_path= "/cs/azencot_fsas/MoE/",
         load_linear = 1,
-        manual_moe = 0,
-        misc_moe = 1,
-        mlp_gating = 1,
+        misc_moe = 10,
+        mlp_gating = 0,
         model_type= "super_linear",
         moe_temp = 1,
         noisy_gating_std = 0.1,
@@ -71,12 +68,9 @@ class SuperLinearConfig(PretrainedConfig):
         self.linear_checkpoints_path  = linear_checkpoints_path
         self.linear_checkpoints_dir   = linear_checkpoints_dir
         self.load_linear              = load_linear 
-        self.manual_moe              = manual_moe
         self.misc_moe                = misc_moe
         self.noisy_gating_std        = noisy_gating_std
         self.noisy_gating_std_decay  = noisy_gating_std_decay
-        self.ker_len                 = ker_len
-        self.con                     = con
         self.d_model                 = d_model
         self.mlp_gating              = mlp_gating
         self.moe_temp                = moe_temp