fix: inv_freq persistent=True + incluso nel safetensors

modeling_quark.py

@@ -26,13 +26,14 @@ class RotaryEmbedding(nn.Module):
         super().__init__()
         assert head_dim % 2 == 0
         inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
-        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.register_buffer("inv_freq", inv_freq, persistent=True)
         self._build_cache(max_seq_len)
 
     def _build_cache(self, seq_len):
-        t     = torch.arange(seq_len, device=self.inv_freq.device).float()
-        freqs = torch.outer(t, self.inv_freq)
-        emb   = torch.cat([freqs, freqs], dim=-1)
+        device = self.inv_freq.device
+        t      = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)
+        freqs  = torch.outer(t, self.inv_freq)
+        emb    = torch.cat([freqs, freqs], dim=-1)
         self.register_buffer("cos_cache", emb.cos()[None, None], persistent=False)
         self.register_buffer("sin_cache", emb.sin()[None, None], persistent=False)
         self._max = seq_len