User01110
/

tinyLM-8M-exp

Text Generation

tinyqwen3_novelty

tiny-language-model

novelty-gated-attention

trust-remote-code

Model card Files Files and versions

User01110 commited on Jun 23

Commit

3a0d1d2

·

verified ·

1 Parent(s): dbb89f9

Persist RoPE inv_freq buffers

Files changed (2) hide show

model.safetensors +2 -2
modeling_tinyqwen3_novelty.py +1 -1

model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96aa6a05a7d1de5562ce71a856395c929cb16e3d25eefce03a7964e52300faad
-size 32536112

 version https://git-lfs.github.com/spec/v1
+oid sha256:2d8c33d716ce718e80e83cd327c7be3d72114f69d08df63c7d2ee941bbe1f52d
+size 32537376

modeling_tinyqwen3_novelty.py CHANGED Viewed

@@ -76,7 +76,7 @@ class RotaryEmbedding(nn.Module):
     def __init__(self, head_dim, theta):
         super().__init__()
         inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
-        self.register_buffer("inv_freq", inv_freq, persistent=False)
     def forward(self, seq_len, device):
         pos = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)

     def __init__(self, head_dim, theta):
         super().__init__()
         inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
+        self.register_buffer("inv_freq", inv_freq)
     def forward(self, seq_len, device):
         pos = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)