hanzceo commited on
Commit
10877c1
·
verified ·
1 Parent(s): be30ecf

Upload model.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. model.py +689 -0
model.py ADDED
@@ -0,0 +1,689 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Penny (DeepSeek-V4.1-Flash port) for Hugging Face transformers, via trust_remote_code.
2
+
3
+ from transformers import AutoModelForCausalLM, AutoTokenizer
4
+ tok = AutoTokenizer.from_pretrained("LocalWisdom/Penny-Alpha")
5
+ model = AutoModelForCausalLM.from_pretrained("LocalWisdom/Penny-Alpha", trust_remote_code=True, dtype="float32")
6
+ ids = tok(tok.eos_token + "The history of the bicycle begins", add_special_tokens=False, return_tensors="pt")
7
+ out = model.generate(**ids, max_new_tokens=200, do_sample=True, temperature=0.8, top_k=50, use_cache=False)
8
+ print(tok.decode(out[0]))
9
+
10
+ Inference-only port of the training model (no KV cache: every step re-runs the whole sequence, which is
11
+ fine at this size). Weights use the official DeepSeek-V4.1 tensor names, so `model.safetensors` loads as is.
12
+ Penny is a base model (no chat template). Prompts must start with EOS (marks a fresh document); the tokenizer does not add it, so prepend tok.eos_token.
13
+ """
14
+
15
+ import math
16
+ from dataclasses import dataclass
17
+
18
+ import numpy as np
19
+ import torch
20
+ import torch.nn.functional as F
21
+ from torch import nn
22
+ from transformers import AutoTokenizer, GenerationMixin, PretrainedConfig, PreTrainedModel
23
+ from transformers.modeling_outputs import CausalLMOutput
24
+
25
+
26
+ # ---------------------------------------------------------------------------------------------
27
+ # config
28
+
29
+
30
+ class DeepseekV41Config(PretrainedConfig):
31
+ """Wraps the nested config.json: top-level HF fields + `text_config` (the architecture) + `penny`."""
32
+
33
+ model_type = "deepseek_v41"
34
+
35
+ def __init__(self, text_config=None, penny=None, **kwargs):
36
+ super().__init__(**kwargs)
37
+ # kept as `arch`, not `text_config`: transformers treats a `text_config` attribute as a sub-config object
38
+ self.arch = dict(text_config or {})
39
+ self.penny = dict(penny or {})
40
+
41
+ def to_dict(self):
42
+ d = super().to_dict()
43
+ d["text_config"] = d.pop("arch", self.arch)
44
+ return d
45
+
46
+
47
+ @dataclass
48
+ class Args:
49
+ vocab_size: int
50
+ dim: int
51
+ moe_inter_dim: int
52
+ n_layers: int
53
+ n_heads: int
54
+ n_routed_experts: int
55
+ n_shared_experts: int
56
+ n_activated_experts: int
57
+ score_func: str
58
+ norm_topk_prob: bool
59
+ route_scale: float
60
+ swiglu_limit: float
61
+ q_lora_rank: int
62
+ head_dim: int
63
+ rope_head_dim: int
64
+ norm_eps: float
65
+ o_groups: int
66
+ o_lora_rank: int
67
+ window_size: int
68
+ compress_ratios: tuple
69
+ kv_source_layers: tuple
70
+ index_source_layers: tuple
71
+ compress_rope_theta: float
72
+ original_seq_len: int
73
+ rope_theta: float
74
+ rope_factor: float
75
+ beta_fast: int
76
+ beta_slow: int
77
+ index_n_heads: int
78
+ index_head_dim: int
79
+ index_topk: int
80
+ candidate_source_layer: int
81
+ candidate_topk_blocks: int
82
+ candidate_block_size: int
83
+ hc_mult: int
84
+ hc_sinkhorn_iters: int
85
+ hc_eps: float
86
+ engram_layer_ids: tuple
87
+ engram_num_embeddings: tuple
88
+ engram_max_ngram_size: int
89
+ engram_vocab_size: int
90
+ engram_n_heads: int
91
+ engram_head_dim: int
92
+ engram_pad_id: int
93
+ engram_compressed_vocab_size: int
94
+ max_seq_len: int
95
+ attn_chunk: int = 256
96
+
97
+ @classmethod
98
+ def from_config(cls, cfg: DeepseekV41Config) -> "Args":
99
+ t = cfg.arch
100
+ rs = t.get("rope_scaling") or {}
101
+ n = t["num_hidden_layers"]
102
+ return cls(
103
+ vocab_size=t["vocab_size"], dim=t["hidden_size"], moe_inter_dim=t["moe_intermediate_size"],
104
+ n_layers=n, n_heads=t["num_attention_heads"], n_routed_experts=t["n_routed_experts"],
105
+ n_shared_experts=t["n_shared_experts"], n_activated_experts=t["num_experts_per_tok"],
106
+ score_func=t["scoring_func"], norm_topk_prob=t["norm_topk_prob"],
107
+ route_scale=t["routed_scaling_factor"], swiglu_limit=t["swiglu_limit"],
108
+ q_lora_rank=t["q_lora_rank"], head_dim=t["head_dim"], rope_head_dim=t["qk_rope_head_dim"],
109
+ norm_eps=t["rms_norm_eps"], o_groups=t["o_groups"], o_lora_rank=t["o_lora_rank"],
110
+ window_size=t["sliding_window"], compress_ratios=tuple(t["compress_ratios"][:n]),
111
+ kv_source_layers=tuple(t["kv_source_layer_ids"]), index_source_layers=tuple(t["index_source_layer_ids"]),
112
+ compress_rope_theta=t["compress_rope_theta"],
113
+ original_seq_len=rs.get("original_max_position_embeddings", 0), rope_theta=t["rope_theta"],
114
+ rope_factor=rs.get("factor", 1), beta_fast=rs.get("beta_fast", 32), beta_slow=rs.get("beta_slow", 1),
115
+ index_n_heads=t["index_n_heads"], index_head_dim=t["index_head_dim"], index_topk=t["index_topk"],
116
+ candidate_source_layer=t["candidate_source_layer_id"], candidate_topk_blocks=t["candidate_topk_blocks"],
117
+ candidate_block_size=t["candidate_block_size"], hc_mult=t["hc_mult"],
118
+ hc_sinkhorn_iters=t["hc_sinkhorn_iters"], hc_eps=t["hc_eps"],
119
+ engram_layer_ids=tuple(t["engram_layer_ids"]), engram_num_embeddings=tuple(t["engram_num_embeddings"]),
120
+ engram_max_ngram_size=t["engram_max_ngram_size"], engram_vocab_size=t["engram_vocab_size"],
121
+ engram_n_heads=t["engram_n_heads"], engram_head_dim=t["engram_head_dim"],
122
+ engram_pad_id=t["engram_pad_token_id"], engram_compressed_vocab_size=t["engram_compressed_vocab_size"],
123
+ max_seq_len=cfg.penny.get("trained_seq_len", 8192),
124
+ )
125
+
126
+
127
+ # ---------------------------------------------------------------------------------------------
128
+ # Engram n-gram hashing (hash ids match the DeepSeek reference exactly)
129
+
130
+
131
+ def _is_prime(n: int) -> bool:
132
+ if n < 2:
133
+ return False
134
+ i = 2
135
+ while i * i <= n:
136
+ if n % i == 0:
137
+ return False
138
+ i += 1
139
+ return True
140
+
141
+
142
+ def _next_prime(start: int, seen: set) -> int:
143
+ c = start + 1
144
+ while not _is_prime(c) or c in seen:
145
+ c += 1
146
+ return c
147
+
148
+
149
+ def _compressed_token_map(tokenizer):
150
+ """Collapse tokens that normalize alike (" The", "the", "THE") onto one compressed id."""
151
+ from tokenizers import Regex, normalizers
152
+
153
+ sentinel = "\ue000" # private-use char, as in the reference
154
+ normalizer = normalizers.Sequence([
155
+ normalizers.NFKC(), normalizers.NFD(), normalizers.StripAccents(), normalizers.Lowercase(),
156
+ normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), normalizers.Replace(Regex(r"^ $"), sentinel),
157
+ normalizers.Strip(), normalizers.Replace(sentinel, " "),
158
+ ])
159
+ backend = tokenizer.backend_tokenizer
160
+ key_to_new, lookup = {}, [0] * len(tokenizer)
161
+ for token_id in range(len(tokenizer)):
162
+ text = backend.decode([token_id], skip_special_tokens=False)
163
+ if "\ufffd" in text:
164
+ key = backend.id_to_token(token_id)
165
+ else:
166
+ normalized = normalizer.normalize_str(text)
167
+ key = normalized if normalized else text
168
+ new_id = key_to_new.get(key)
169
+ if new_id is None:
170
+ new_id = len(key_to_new)
171
+ key_to_new[key] = new_id
172
+ lookup[token_id] = new_id
173
+ return lookup, len(key_to_new)
174
+
175
+
176
+ class NgramHash:
177
+ """input_ids [B, L] -> hash ids [B, L, n_engram_layers, (max_ngram_size-1) * n_heads]. Built lazily
178
+ (needs the tokenizer) and not an nn.Module, so it never touches the weights."""
179
+
180
+ def __init__(self, a: Args, tokenizer):
181
+ layer_ids = tuple(a.engram_layer_ids)
182
+ self.max_n = a.engram_max_ngram_size
183
+ primes, seen = [], set()
184
+ for _ in layer_ids:
185
+ per_ngram = []
186
+ for _ in range(self.max_n - 1):
187
+ sizes, current = [], a.engram_vocab_size - 1
188
+ for _ in range(a.engram_n_heads):
189
+ current = _next_prime(current, seen)
190
+ seen.add(current)
191
+ sizes.append(current)
192
+ per_ngram.append(sizes)
193
+ primes.append(per_ngram)
194
+ token_map, vocab_size = _compressed_token_map(tokenizer)
195
+ assert vocab_size == a.engram_compressed_vocab_size, (vocab_size, a.engram_compressed_vocab_size)
196
+ self.pad_id = token_map[a.engram_pad_id]
197
+ flat = [[p for per_ngram in layer for p in per_ngram] for layer in primes]
198
+ offsets = [np.cumsum([0, *sizes[:-1]]) for sizes in flat]
199
+ max_long = np.iinfo(np.int64).max
200
+ bound = max(1, (max_long // vocab_size) // 2)
201
+ mult = []
202
+ for layer_id in layer_ids:
203
+ g = np.random.default_rng(10007 * layer_id)
204
+ mult.append(torch.tensor(g.integers(low=0, high=bound, size=(self.max_n,), dtype=np.int64) * 2 + 1))
205
+ self.primes = torch.tensor(primes)
206
+ self.offsets = torch.tensor(np.array(offsets))
207
+ self.multipliers = torch.stack(mult)
208
+ self.token_map = torch.tensor(token_map)
209
+
210
+ @torch.no_grad()
211
+ def __call__(self, input_ids: torch.Tensor) -> torch.Tensor:
212
+ dev = input_ids.device
213
+ primes, offsets, multipliers, token_map = (
214
+ self.primes.to(dev), self.offsets.to(dev), self.multipliers.to(dev), self.token_map.to(dev))
215
+ compressed = token_map[input_ids]
216
+ tokens = [compressed]
217
+ for shift in range(1, self.max_n):
218
+ shifted = torch.full_like(compressed, self.pad_id)
219
+ shifted[:, shift:] = compressed[:, :-shift]
220
+ tokens.append(shifted)
221
+ tokens = torch.stack(tokens, dim=-1)
222
+ products = tokens.unsqueeze(2) * multipliers
223
+ rolling, hashes = products[..., 0], []
224
+ for i in range(1, self.max_n):
225
+ rolling = torch.bitwise_xor(rolling, products[..., i])
226
+ hashes.append(rolling.unsqueeze(-1) % primes[:, i - 1])
227
+ return torch.cat(hashes, dim=-1) + offsets
228
+
229
+
230
+ # ---------------------------------------------------------------------------------------------
231
+ # primitives
232
+
233
+
234
+ class RMSNorm(nn.Module):
235
+ def __init__(self, dim: int, eps: float):
236
+ super().__init__()
237
+ self.eps = eps
238
+ self.weight = nn.Parameter(torch.ones(dim))
239
+
240
+ def forward(self, x):
241
+ dtype = x.dtype
242
+ x = x.float()
243
+ x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.eps)
244
+ return (self.weight.float() * x).to(dtype)
245
+
246
+
247
+ def precompute_freqs_cis(dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow, device):
248
+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
249
+ if original_seq_len > 0:
250
+ def corrected_dim(rotations):
251
+ return dim * math.log(original_seq_len / (rotations * 2 * math.pi)) / (2 * math.log(base))
252
+
253
+ low = max(math.floor(corrected_dim(beta_fast)), 0)
254
+ high = min(math.ceil(corrected_dim(beta_slow)), dim - 1)
255
+ ramp = ((torch.arange(dim // 2, dtype=torch.float32) - low) / max(high - low, 1e-3)).clamp(0, 1)
256
+ smooth = 1 - ramp
257
+ freqs = freqs / factor * (1 - smooth) + freqs * smooth
258
+ freqs = torch.outer(torch.arange(seqlen), freqs)
259
+ return torch.polar(torch.ones_like(freqs), freqs).to(device)
260
+
261
+
262
+ def rope(x, freqs_cis, rd, inverse=False):
263
+ """Rotate the last `rd` dims of x (adjacent pairs as complex numbers). x: [b,s,d] or [b,s,h,d]."""
264
+ nope, pe = x[..., :-rd], x[..., -rd:]
265
+ z = torch.view_as_complex(pe.float().unflatten(-1, (-1, 2)).contiguous())
266
+ f = freqs_cis.conj() if inverse else freqs_cis
267
+ f = f.view(1, z.size(1), *([1] * (z.ndim - 3)), z.size(-1))
268
+ pe = torch.view_as_real(z * f).flatten(-2).to(x.dtype)
269
+ return torch.cat([nope, pe], dim=-1)
270
+
271
+
272
+ def idxs_to_mask(idxs, n):
273
+ """[b,m,k] int indices (-1 = empty) -> [b,m,n] bool mask."""
274
+ mask = torch.zeros(*idxs.shape[:2], n + 1, dtype=torch.bool, device=idxs.device)
275
+ mask.scatter_(-1, torch.where(idxs < 0, n, idxs).long(), True)
276
+ return mask[..., :n]
277
+
278
+
279
+ def chunked_sink_attention(q, kv, attn_sink, window, scale, comp_kv=None, comp_idxs=None, chunk=256):
280
+ """Sliding-window attention over raw KV plus top-k compressed KV, with a learned sink logit.
281
+ q [b,L,h,d]; kv [b,L,d]; comp_kv [b,T,d]; comp_idxs [b,L,k] into comp_kv (-1 = empty)."""
282
+ L = q.size(1)
283
+ outs = []
284
+ for s0 in range(0, L, chunk):
285
+ s1 = min(s0 + chunk, L)
286
+ k0 = max(0, s0 - window + 1)
287
+ qpos = torch.arange(s0, s1, device=q.device).unsqueeze(1)
288
+ kpos = torch.arange(k0, s1, device=q.device)
289
+ wmask = (kpos <= qpos) & (kpos > qpos - window)
290
+ qc, kw = q[:, s0:s1], kv[:, k0:s1]
291
+ s = torch.einsum("bmhd,bnd->bhmn", qc, kw).float() * scale
292
+ s = s.masked_fill(~wmask, -torch.inf)
293
+ vals = kw
294
+ if comp_kv is not None:
295
+ cmask = idxs_to_mask(comp_idxs[:, s0:s1], comp_kv.size(1))
296
+ sc = torch.einsum("bmhd,bnd->bhmn", qc, comp_kv).float() * scale
297
+ s = torch.cat([s, sc.masked_fill(~cmask.unsqueeze(1), -torch.inf)], dim=-1)
298
+ vals = torch.cat([kw, comp_kv], dim=1)
299
+ sink = attn_sink.float().view(1, -1, 1, 1).expand(*s.shape[:3], 1)
300
+ lse = torch.logsumexp(torch.cat([s, sink], dim=-1), dim=-1)
301
+ p = torch.exp(s - lse.unsqueeze(-1))
302
+ outs.append(torch.einsum("bhmn,bnd->bmhd", p.to(vals.dtype), vals))
303
+ return torch.cat(outs, dim=1)
304
+
305
+
306
+ def hc_split_sinkhorn(mixes, hc_scale, hc_base, hc, iters, eps):
307
+ pre = torch.sigmoid(mixes[..., :hc] * hc_scale[0] + hc_base[:hc]) + eps
308
+ post = 2 * torch.sigmoid(mixes[..., hc : 2 * hc] * hc_scale[1] + hc_base[hc : 2 * hc])
309
+ comb = mixes[..., 2 * hc :] * hc_scale[2] + hc_base[2 * hc :]
310
+ comb = comb.unflatten(-1, (hc, hc)).softmax(-1) + eps
311
+ comb = comb / (comb.sum(-2, keepdim=True) + eps)
312
+ for _ in range(iters - 1):
313
+ comb = comb / (comb.sum(-1, keepdim=True) + eps)
314
+ comb = comb / (comb.sum(-2, keepdim=True) + eps)
315
+ return pre, post, comb
316
+
317
+
318
+ # ---------------------------------------------------------------------------------------------
319
+ # attention
320
+
321
+
322
+ class Shared:
323
+ """Per-forward hand-off between source layers and the layers that reuse their results."""
324
+
325
+ def __init__(self):
326
+ self.compress_kv = None
327
+ self.index_k = None
328
+ self.topk_idxs = None
329
+ self.candidates = None
330
+
331
+
332
+ class Compressor(nn.Module):
333
+ """Pools `ratio` consecutive tokens into one KV latent with a learned softmax gate (pre-RoPE)."""
334
+
335
+ def __init__(self, a: Args, layer_id: int):
336
+ super().__init__()
337
+ self.ratio = a.compress_ratios[layer_id]
338
+ self.norm = RMSNorm(a.head_dim, a.norm_eps)
339
+ self.wkv = nn.Linear(a.dim, a.head_dim, bias=False)
340
+ self.wgate = nn.Linear(a.dim, a.head_dim, bias=False) if self.ratio > 1 else None
341
+
342
+ def forward(self, x):
343
+ if self.ratio == 1:
344
+ return self.norm(self.wkv(x))
345
+ cutoff = x.size(1) - x.size(1) % self.ratio # trailing partial group is not visible yet
346
+ xf = x[:, :cutoff].float() # the reference pools in fp32
347
+ kv = F.linear(xf, self.wkv.weight.float()).unflatten(1, (-1, self.ratio))
348
+ score = F.linear(xf, self.wgate.weight.float()).unflatten(1, (-1, self.ratio))
349
+ kv = (kv * score.softmax(dim=2)).sum(dim=2)
350
+ return self.norm(kv.to(x.dtype))
351
+
352
+
353
+ def select_candidate_blocks(logits, compress_lens, topk_blocks, block_size):
354
+ width = logits.size(-1)
355
+ scores = F.pad(logits, (0, -width % block_size), value=-torch.inf)
356
+ scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1)
357
+ num_blocks = scores.size(-1)
358
+ last = (compress_lens - 1) // block_size
359
+ scores = scores.masked_fill(torch.arange(num_blocks, device=logits.device) == last, torch.inf)
360
+ top = scores.topk(min(topk_blocks, num_blocks), dim=-1)
361
+ keep = torch.zeros_like(scores, dtype=torch.bool).scatter_(-1, top.indices, top.values > -torch.inf)
362
+ return keep.repeat_interleave(block_size, dim=-1)[..., :width]
363
+
364
+
365
+ class Indexer(nn.Module):
366
+ """Scores compressed positions and keeps the `index_topk` best per query."""
367
+
368
+ def __init__(self, a: Args, layer_id: int):
369
+ super().__init__()
370
+ self.owns_k = layer_id in a.kv_source_layers
371
+ self.ratio = a.compress_ratios[layer_id]
372
+ self.is_candidate_source = layer_id == a.candidate_source_layer
373
+ self.uses_candidates = 0 <= a.candidate_source_layer < layer_id
374
+ self.candidate_topk_blocks = a.candidate_topk_blocks
375
+ self.candidate_block_size = a.candidate_block_size
376
+ self.n_heads = a.index_n_heads
377
+ self.index_head_dim = a.index_head_dim
378
+ self.rd = a.rope_head_dim
379
+ self.index_topk = a.index_topk
380
+ self.softmax_scale = self.index_head_dim**-0.5
381
+ self.wq_b = nn.Linear(a.q_lora_rank, self.n_heads * self.index_head_dim, bias=False)
382
+ self.weights_proj = nn.Linear(a.dim, self.n_heads, bias=False)
383
+ if self.owns_k:
384
+ self.wk = nn.Linear(a.head_dim, self.index_head_dim, bias=False)
385
+ self.k_norm = RMSNorm(self.index_head_dim, a.norm_eps)
386
+
387
+ @torch.no_grad()
388
+ def forward(self, x, qr, latent, freqs_cis, shared: Shared, chunk: int):
389
+ """Returns selected compressed indices [b,s,k] (-1 = empty)."""
390
+ seqlen, ratio = x.size(1), self.ratio
391
+ if self.owns_k:
392
+ k = self.k_norm(self.wk(latent))
393
+ shared.index_k = rope(k, freqs_cis[: seqlen - seqlen % ratio : ratio], self.rd)
394
+ q = rope(self.wq_b(qr).unflatten(-1, (self.n_heads, self.index_head_dim)), freqs_cis[:seqlen], self.rd)
395
+ index_k = shared.index_k
396
+ weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads**-0.5)
397
+ T = index_k.size(1)
398
+ topk = min(self.index_topk, T)
399
+ idx_chunks, cand_chunks = [], []
400
+ for s0 in range(0, seqlen, chunk):
401
+ s1 = min(s0 + chunk, seqlen)
402
+ score = torch.einsum("bshd,btd->bsht", q[:, s0:s1], index_k)
403
+ score = (score.relu() * weights[:, s0:s1].unsqueeze(-1)).sum(dim=2).float()
404
+ lens = (torch.arange(s0 + 1, s1 + 1, device=x.device) // ratio).unsqueeze(-1)
405
+ score = score.masked_fill(torch.arange(T, device=x.device) >= lens, -torch.inf)
406
+ if self.is_candidate_source:
407
+ cand_chunks.append(select_candidate_blocks(
408
+ score, lens, self.candidate_topk_blocks, self.candidate_block_size))
409
+ elif self.uses_candidates and shared.candidates is not None:
410
+ score = score.masked_fill(~shared.candidates[:, s0:s1], -torch.inf)
411
+ idxs = score.topk(topk, dim=-1, sorted=False).indices
412
+ idx_chunks.append(torch.where(idxs < lens, idxs, -1))
413
+ if self.is_candidate_source:
414
+ shared.candidates = torch.cat(cand_chunks, dim=1)
415
+ return torch.cat(idx_chunks, dim=1)
416
+
417
+
418
+ class Attention(nn.Module):
419
+ """Sliding window over raw KV, plus (compress_ratio > 0) top-k compressed KV further back.
420
+ Low-rank Q; grouped low-rank output projection; shared K=V latent (MQA)."""
421
+
422
+ def __init__(self, layer_id: int, a: Args):
423
+ super().__init__()
424
+ self.n_heads, self.head_dim, self.rd = a.n_heads, a.head_dim, a.rope_head_dim
425
+ self.n_groups, self.o_lora_rank, self.window = a.o_groups, a.o_lora_rank, a.window_size
426
+ self.ratio = a.compress_ratios[layer_id]
427
+ self.softmax_scale = self.head_dim**-0.5
428
+ self.attn_chunk = a.attn_chunk
429
+ self.attn_sink = nn.Parameter(torch.zeros(self.n_heads))
430
+ self.wq_a = nn.Linear(a.dim, a.q_lora_rank, bias=False)
431
+ self.q_norm = RMSNorm(a.q_lora_rank, a.norm_eps)
432
+ self.wq_b = nn.Linear(a.q_lora_rank, self.n_heads * self.head_dim, bias=False)
433
+ self.wkv = nn.Linear(a.dim, self.head_dim, bias=False)
434
+ self.kv_norm = RMSNorm(self.head_dim, a.norm_eps)
435
+ self.wo_a = nn.Linear(self.n_heads * self.head_dim // self.n_groups, self.n_groups * self.o_lora_rank, bias=False)
436
+ self.wo_b = nn.Linear(self.n_groups * self.o_lora_rank, a.dim, bias=False)
437
+ self.is_kv_source = layer_id in a.kv_source_layers
438
+ self.is_index_source = layer_id in a.index_source_layers
439
+ self.compressor = Compressor(a, layer_id) if self.is_kv_source else None
440
+ self.indexer = Indexer(a, layer_id) if self.is_index_source else None
441
+ # RoPE table: built lazily on the right device (never part of the checkpoint)
442
+ if self.ratio:
443
+ self._rope = (a.original_seq_len, a.compress_rope_theta)
444
+ else:
445
+ self._rope = (0, a.rope_theta) # pure sliding window: no YaRN, base theta
446
+ self._rope_args = (a.rope_factor, a.beta_fast, a.beta_slow, a.max_seq_len)
447
+ self._fc = None
448
+
449
+ def freqs(self, seqlen, device):
450
+ if self._fc is None or self._fc.size(0) < seqlen or self._fc.device != device:
451
+ factor, bf, bs, max_len = self._rope_args
452
+ self._fc = precompute_freqs_cis(self.rd, max(seqlen, max_len), self._rope[0], self._rope[1],
453
+ factor, bf, bs, device)
454
+ return self._fc[:seqlen]
455
+
456
+ def forward(self, x, shared: Shared):
457
+ bsz, seqlen, _ = x.shape
458
+ fc = self.freqs(seqlen, x.device)
459
+ qr = self.q_norm(self.wq_a(x))
460
+ q = rope(self.wq_b(qr).unflatten(-1, (self.n_heads, self.head_dim)), fc, self.rd)
461
+ kv = rope(self.kv_norm(self.wkv(x)), fc, self.rd)
462
+
463
+ comp_kv, cidx = None, None
464
+ if self.ratio:
465
+ latent = self.compressor(x) if self.is_kv_source else None
466
+ if self.is_index_source:
467
+ shared.topk_idxs = self.indexer(x, qr, latent, fc, shared, chunk=self.attn_chunk)
468
+ cidx = shared.topk_idxs
469
+ if latent is not None: # group j sits at the position of its first token, j * ratio
470
+ shared.compress_kv = rope(latent, fc[: seqlen - seqlen % self.ratio : self.ratio], self.rd)
471
+ comp_kv = shared.compress_kv
472
+
473
+ o = chunked_sink_attention(q, kv, self.attn_sink, self.window, self.softmax_scale, comp_kv, cidx,
474
+ chunk=self.attn_chunk)
475
+ o = rope(o, fc, self.rd, inverse=True) # remove the query rotation from the output
476
+ o = o.reshape(bsz, seqlen, self.n_groups, -1)
477
+ wo_a = self.wo_a.weight.view(self.n_groups, self.o_lora_rank, -1)
478
+ o = torch.einsum("bsgd,grd->bsgr", o, wo_a.to(o.dtype))
479
+ return self.wo_b(o.flatten(2))
480
+
481
+
482
+ # ---------------------------------------------------------------------------------------------
483
+ # MoE
484
+
485
+
486
+ class Gate(nn.Module):
487
+ """sqrtsoftplus scores; a bias picks experts, weights come from the unbiased scores."""
488
+
489
+ def __init__(self, a: Args):
490
+ super().__init__()
491
+ self.topk, self.score_func = a.n_activated_experts, a.score_func
492
+ self.norm_topk_prob, self.route_scale = a.norm_topk_prob, a.route_scale
493
+ self.weight = nn.Parameter(torch.empty(a.n_routed_experts, a.dim))
494
+ self.register_buffer("bias", torch.zeros(a.n_routed_experts))
495
+
496
+ def forward(self, x):
497
+ scores = F.linear(x.float(), self.weight.float())
498
+ if self.score_func == "softmax":
499
+ scores = scores.softmax(dim=-1)
500
+ elif self.score_func == "sigmoid":
501
+ scores = scores.sigmoid()
502
+ else:
503
+ scores = F.softplus(scores).sqrt()
504
+ indices = (scores + self.bias.float()).topk(self.topk, dim=-1)[1]
505
+ weights = scores.gather(1, indices)
506
+ if self.norm_topk_prob and self.topk > 1:
507
+ weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20)
508
+ return weights * self.route_scale, indices
509
+
510
+
511
+ class Expert(nn.Module):
512
+ def __init__(self, dim, inter_dim, swiglu_limit):
513
+ super().__init__()
514
+ self.w1 = nn.Linear(dim, inter_dim, bias=False)
515
+ self.w2 = nn.Linear(inter_dim, dim, bias=False)
516
+ self.w3 = nn.Linear(dim, inter_dim, bias=False)
517
+ self.swiglu_limit = swiglu_limit
518
+
519
+ def forward(self, x, weights=None):
520
+ dtype = x.dtype
521
+ gate, up = self.w1(x).float(), self.w3(x).float()
522
+ if self.swiglu_limit > 0:
523
+ up = up.clamp(-self.swiglu_limit, self.swiglu_limit)
524
+ gate = gate.clamp(max=self.swiglu_limit)
525
+ h = F.silu(gate) * up
526
+ if weights is not None:
527
+ h = weights * h
528
+ return self.w2(h.to(dtype))
529
+
530
+
531
+ class MoE(nn.Module):
532
+ def __init__(self, a: Args):
533
+ super().__init__()
534
+ self.dim = a.dim
535
+ self.gate = Gate(a)
536
+ self.experts = nn.ModuleList([Expert(a.dim, a.moe_inter_dim, a.swiglu_limit) for _ in range(a.n_routed_experts)])
537
+ assert a.n_shared_experts == 1
538
+ self.shared_experts = Expert(a.dim, a.moe_inter_dim, a.swiglu_limit)
539
+
540
+ def forward(self, x):
541
+ shape = x.shape
542
+ x = x.reshape(-1, self.dim)
543
+ weights, indices = self.gate(x)
544
+ y = torch.zeros_like(x, dtype=torch.float32)
545
+ counts = torch.bincount(indices.flatten(), minlength=len(self.experts)).tolist()
546
+ for i, expert in enumerate(self.experts):
547
+ if counts[i] == 0:
548
+ continue
549
+ idx, top = torch.where(indices == i)
550
+ y = y.index_add(0, idx, expert(x[idx], weights[idx, top, None]).float())
551
+ y = y + self.shared_experts(x).float()
552
+ return y.to(x.dtype).view(shape)
553
+
554
+
555
+ # ---------------------------------------------------------------------------------------------
556
+ # Engram, blocks, model
557
+
558
+
559
+ class Engram(nn.Module):
560
+ """Gated n-gram lookup written into each hc copy of the residual stream."""
561
+
562
+ def __init__(self, a: Args, layer_id: int):
563
+ super().__init__()
564
+ self.layer_hash_index = tuple(a.engram_layer_ids).index(layer_id)
565
+ self.dim, self.hc_mult, self.eps = a.dim, a.hc_mult, a.norm_eps
566
+ self.embed = nn.Embedding(a.engram_num_embeddings[self.layer_hash_index], a.engram_head_dim)
567
+ n_hash_cols = (a.engram_max_ngram_size - 1) * a.engram_n_heads
568
+ self.wkv = nn.Linear(n_hash_cols * a.engram_head_dim, a.dim * (a.hc_mult + 1), bias=False)
569
+ self.q_weight = nn.Parameter(torch.ones(a.hc_mult, a.dim))
570
+ self.k_weight = nn.Parameter(torch.ones(a.hc_mult, a.dim))
571
+
572
+ def forward(self, x, hash_ids):
573
+ kv = self.wkv(self.embed(hash_ids).flatten(-2))
574
+ key, value = kv.split([self.hc_mult * self.dim, self.dim], dim=-1)
575
+ key = key.float().unflatten(-1, (self.hc_mult, self.dim))
576
+ h = x.float()
577
+ rstd = torch.rsqrt(h.square().mean(-1) + self.eps) * torch.rsqrt(key.square().mean(-1) + self.eps)
578
+ dot = (h * (self.q_weight.float() * self.k_weight.float()) * key).sum(-1) * rstd * self.dim**-0.5
579
+ gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(1e-6).sqrt(), dot))
580
+ return (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype)
581
+
582
+
583
+ class Block(nn.Module):
584
+ """Hyper-Connections block: the residual stream is carried as hc_mult copies."""
585
+
586
+ def __init__(self, layer_id: int, a: Args):
587
+ super().__init__()
588
+ self.norm_eps = a.norm_eps
589
+ self.attn = Attention(layer_id, a)
590
+ self.ffn = MoE(a)
591
+ self.engram = Engram(a, layer_id) if layer_id in a.engram_layer_ids else None
592
+ self.attn_norm = RMSNorm(a.dim, a.norm_eps)
593
+ self.ffn_norm = RMSNorm(a.dim, a.norm_eps)
594
+ self.hc, self.hc_iters, self.hc_eps = a.hc_mult, a.hc_sinkhorn_iters, a.hc_eps
595
+ mix_hc = (2 + self.hc) * self.hc
596
+ self.hc_attn_fn = nn.Parameter(torch.empty(mix_hc, self.hc * a.dim))
597
+ self.hc_ffn_fn = nn.Parameter(torch.empty(mix_hc, self.hc * a.dim))
598
+ self.hc_attn_base = nn.Parameter(torch.empty(mix_hc))
599
+ self.hc_ffn_base = nn.Parameter(torch.empty(mix_hc))
600
+ self.hc_attn_scale = nn.Parameter(torch.ones(3))
601
+ self.hc_ffn_scale = nn.Parameter(torch.ones(3))
602
+
603
+ def hc_mixes(self, x, fn, scale, base):
604
+ x = x.flatten(2).float()
605
+ mixes = F.linear(x, fn.float()) * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.norm_eps)
606
+ return hc_split_sinkhorn(mixes, scale.float(), base.float(), self.hc, self.hc_iters, self.hc_eps)
607
+
608
+ @staticmethod
609
+ def hc_pre(x, pre_mix):
610
+ return torch.sum(pre_mix.unsqueeze(-1) * x.float(), dim=2).to(x.dtype)
611
+
612
+ @staticmethod
613
+ def hc_post(x, residual, post, comb):
614
+ y = post.unsqueeze(-1) * x.unsqueeze(-2) + torch.sum(comb.unsqueeze(-1) * residual.unsqueeze(-2), dim=2)
615
+ return y.to(residual.dtype)
616
+
617
+ def forward(self, x, pre_mix, shared: Shared):
618
+ residual = x
619
+ attn_pre, attn_post, attn_comb = self.hc_mixes(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)
620
+ h = self.attn(self.attn_norm(self.hc_pre(x, pre_mix)), shared)
621
+ x = self.hc_post(h, residual, attn_post, attn_comb)
622
+ residual = x
623
+ ffn_pre, ffn_post, ffn_comb = self.hc_mixes(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)
624
+ h = self.ffn(self.ffn_norm(self.hc_pre(x, attn_pre)))
625
+ x = self.hc_post(h, residual, ffn_post, ffn_comb)
626
+ return x, ffn_pre
627
+
628
+
629
+ class DeepseekV41PreTrainedModel(PreTrainedModel):
630
+ config_class = DeepseekV41Config
631
+ base_model_prefix = ""
632
+ supports_gradient_checkpointing = False
633
+
634
+ def _init_weights(self, module):
635
+ pass # every parameter comes from model.safetensors
636
+
637
+
638
+ class DeepseekV41ForCausalLM(DeepseekV41PreTrainedModel, GenerationMixin):
639
+ """Submodule names (embed, layers, norm, head) match the checkpoint's tensor names directly."""
640
+
641
+ def __init__(self, config: DeepseekV41Config):
642
+ super().__init__(config)
643
+ a = Args.from_config(config)
644
+ self.args = a
645
+ self.embed = nn.Embedding(a.vocab_size, a.dim)
646
+ self.layers = nn.ModuleList([Block(i, a) for i in range(a.n_layers)])
647
+ self.norm = RMSNorm(a.dim, a.norm_eps)
648
+ self.head = nn.Linear(a.dim, a.vocab_size, bias=False)
649
+ self.hc = a.hc_mult
650
+ self._engram_hash = None
651
+ self.post_init()
652
+
653
+ def get_input_embeddings(self):
654
+ return self.embed
655
+
656
+ def set_input_embeddings(self, value):
657
+ self.embed = value
658
+
659
+ def get_output_embeddings(self):
660
+ return self.head
661
+
662
+ def set_output_embeddings(self, value):
663
+ self.head = value
664
+
665
+ def _engram(self):
666
+ if self._engram_hash is None: # needs the tokenizer shipped in the same repo / folder
667
+ src = getattr(self.config, "_name_or_path", None) or self.config.name_or_path
668
+ self._engram_hash = NgramHash(self.args, AutoTokenizer.from_pretrained(src))
669
+ return self._engram_hash
670
+
671
+ def forward(self, input_ids=None, attention_mask=None, labels=None, logits_to_keep=0, **kwargs):
672
+ """No KV cache; attention_mask is ignored (use one unpadded sequence per row, i.e. batch of equal
673
+ lengths or batch size 1). logits_to_keep=N returns logits for the last N positions only."""
674
+ bsz, seqlen = input_ids.shape
675
+ hashes = self._engram()(input_ids)
676
+ h = self.embed(input_ids).unsqueeze(2).repeat(1, 1, self.hc, 1)
677
+ pre_mix = torch.zeros(bsz, seqlen, self.hc, device=h.device)
678
+ pre_mix[..., 0] = 1.0
679
+ shared = Shared()
680
+ for layer in self.layers:
681
+ if layer.engram is not None:
682
+ h = layer.engram(h, hashes[:, :, layer.engram.layer_hash_index])
683
+ h, pre_mix = layer(h, pre_mix, shared)
684
+ h = self.norm(Block.hc_pre(h, pre_mix))
685
+ logits = self.head(h[:, -logits_to_keep:] if logits_to_keep else h)
686
+ loss = None
687
+ if labels is not None:
688
+ loss = F.cross_entropy(logits[:, :-1].float().flatten(0, 1), labels[:, 1:].flatten(), ignore_index=-100)
689
+ return CausalLMOutput(loss=loss, logits=logits)