Update modeling_super_linear.py

modeling_super_linear.py

@@ -360,8 +360,8 @@ class SparseNoisyMoE(nn.Module):
         self.topk_gates = F.softmax(self.topk_values, dim=1)
         
         batch_size = x.size(0)
-        if x.shape[1] < 512:
-            x = self.fourier_interp_dim1(x)
+        '''if x.shape[1] < 512:
+            x = self.fourier_interp_dim1(x)'''
         expert_outputs = torch.stack([self.experts[i](x) for i in range(self.num_experts)], dim=1)
 
         topk_indices_expanded = topk_indices.unsqueeze(-1).expand(-1, -1, expert_outputs.size(2))
@@ -639,7 +639,6 @@ class SuperLinearForCausalLM(PreTrainedModel, GenerationMixin):
         
         # backbone expects (B, C, L)
         x_enc = inputs_embeds
-                
 
         
         if x_enc.shape[1] < 512:
@@ -647,7 +646,6 @@ class SuperLinearForCausalLM(PreTrainedModel, GenerationMixin):
             #x_enc = self.fourier_interp_dim1(x_enc)
             pass
 
-        #self.backbone.inf_pred_len = 336
 
         # backbone returns (B, pred_len, C)