Update modeling_deepseek.py

modeling_deepseek.py

@@ -422,10 +422,10 @@ class MoEGate(nn.Module):
         ### compute gating score
         hidden_states = hidden_states.view(-1, h)
         logits = F.linear(
-            hidden_states.type(torch.float32), self.weight.type(torch.float32), None
+            hidden_states, self.weight, None
         )
         if self.scoring_func == "softmax":
-            scores = logits.softmax(dim=-1, dtype=torch.float32)
+            scores = logits.softmax(dim=-1)
         else:
             raise NotImplementedError(
                 f"insupportable scoring function for MoE gating: {self.scoring_func}"