fix

attn.py

@@ -35,7 +35,7 @@ def _get_alibi_slopes(self, n_heads: int, interpolation_factor: float = 0.25):
         extra_slopes = self._generate_slopes(2 * n)
         extra_slopes_trunc = extra_slopes[0::2][: n_heads - n]
         slopes = slopes_power_of_two + extra_slopes_trunc
-    slopes = torch.tensor(slopes, device=self.device)
+    slopes = torch.tensor(slopes, device=self.device, dtype=torch.float32)
     slopes = slopes * interpolation_factor  # https://arxiv.org/pdf/2310.13017
     return slopes