Fix device and type in RotaryPosinionalEmbeddings

encoder.py

@@ -354,9 +354,9 @@ class RotaryPositionalEmbedding(PositionalEncoding):
             return None
         positions = torch.arange(0, length, dtype=torch.float32, device=device)
         inv_freq = 1.0 / (
-            self.base ** (torch.arange(0, self.dim, 2).float() / self.dim)
+            self.base ** (torch.arange(0, self.dim, 2, device=positions.device).float() / self.dim)
         )
-        t = torch.arange(length, device=positions.device).type_as(inv_freq)
+        t = torch.arange(length, device=positions.device, dtype=inv_freq.dtype)
         freqs = torch.einsum("i,j->ij", t, inv_freq)
         emb = torch.cat((freqs, freqs), dim=-1).to(positions.device)
         return torch.cat([emb.cos()[:, None, None, :], emb.sin()[:, None, None, :]])