fix

attn.py

@@ -37,6 +37,7 @@ def _get_alibi_slopes(self, n_heads: int, interpolation_factor: float = 0.25):
         slopes = slopes_power_of_two + extra_slopes_trunc
     slopes = torch.tensor(slopes, device=self.device, dtype=torch.float32)
     slopes = slopes * interpolation_factor  # https://arxiv.org/pdf/2310.13017
+    slopes = slopes.to(torch.float32)
     return slopes