Update modeling_super_linear.py

modeling_super_linear.py

@@ -590,10 +590,12 @@ class SuperLinearForCausalLM(PreTrainedModel, GenerationMixin):
         # backbone expects (B, C, L)
         x_enc = inputs_embeds
         
-        '''if x_enc.shape[1] < 512:
-            x_enc = self.fourier_interp_dim1(x_enc)'''
-        
-                        
+        if x_enc.shape[1] < 512:
+            x_enc = self.fourier_interp_dim1(x_enc)
+            mean = x_enc.mean()
+            std  = x_enc.std().clamp_min(1e-6)
+            x_enc = (x_enc - mean) / std
+   
         # backbone returns (B, pred_len, C)
         preds = self.backbone(x_enc)
         return CausalLMOutputWithCrossAttentions(loss=None,logits=preds,past_key_values=None,hidden_states=None,attentions=None,)