Ayaka157 commited on
Commit
a9e39bc
·
verified ·
1 Parent(s): a7bf72b

Upload folder using huggingface_hub

Browse files
README.md ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: transformers
4
+ pipeline_tag: text-generation
5
+ tags: [retnet, retention, custom-architecture]
6
+ ---
7
+
8
+ # RetNet-0.5B-v21
9
+
10
+ RetNet 0.5B(504,179,896 参数)第 v21 代权重,从零训练。
11
+
12
+ ## 这一代和上一代的区别
13
+
14
+ 159,857 步 / 10.23B 训练 token / 9.81B 唯一 token(19.46 token/参数);8 段语料自动接力(预训练→中文网页→代码→教材→代码剩余→中文编程→数学→新旧混合 SFT);可热加载的训练计划(阶段预算 / LR / 存档节奏在线改)
15
+
16
+ ## 用法
17
+
18
+ ```python
19
+ from transformers import AutoModelForCausalLM, AutoTokenizer
20
+
21
+ model = AutoModelForCausalLM.from_pretrained("Ayaka157/RetNet-0.5B-v21", trust_remote_code=True)
22
+ tok = AutoTokenizer.from_pretrained("Ayaka157/RetNet-0.5B-v21", trust_remote_code=True)
23
+
24
+ ids = tok("人工智能的未来是", return_tensors="pt").input_ids
25
+ out = model.generate(ids, max_new_tokens=64, do_sample=True, temperature=0.8)
26
+ print(tok.decode(out[0], skip_special_tokens=True))
27
+ ```
28
+
29
+ `trust_remote_code=True` 是必需的:这是自定义架构,代码随仓库发布
30
+ (`modeling_retnet.py`),不在 transformers 里。
31
+
32
+ ## 训练与评测
33
+
34
+ 见项目主页的 `docs/`。本代的关键数字:
35
+
36
+ | 指标 | 值 |
37
+ |---|---|
38
+ | 域内 held-out loss(5 个分片) | 3.5023(ep3 基线 3.6287) |
39
+ | 上一代(ep3)基线 | 3.6287 |
40
+ | 相对变化 | -3.5%(中途最佳 -21.5% @ step 17,500) |
41
+ | 训练 token | 9.81B 唯一 / 19.46 tok/param |
42
+ | 硬件 | 2× RTX PRO 6000(DDP) |
bpe_tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "RetNetForCausalLM"
4
+ ],
5
+ "checkpoint_layers": -1,
6
+ "chunk_size": 128,
7
+ "d_model": 1408,
8
+ "dropout": 0.1,
9
+ "dtype": "float32",
10
+ "embed_scale": 27.712812921102035,
11
+ "ffn_dim": 2816,
12
+ "growth_old_dim": 768,
13
+ "mode": "chunkwise",
14
+ "model_type": "retnet_hf",
15
+ "norm_eps": 1e-05,
16
+ "num_heads": 22,
17
+ "num_layers": 20,
18
+ "prompt_loss_weight": 0.1,
19
+ "tie_weights": true,
20
+ "transformers_version": "4.57.6",
21
+ "use_gla": true,
22
+ "use_growth_ln": true,
23
+ "use_swiglu": false,
24
+ "value_dim": 2816,
25
+ "vocab_size": 20000,
26
+ "auto_map": {
27
+ "AutoConfig": "modeling_retnet.RetNetHFConfig",
28
+ "AutoModelForCausalLM": "modeling_retnet.RetNetForCausalLM"
29
+ }
30
+ }
fused_ce.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """融合线性 + 交叉熵:只保存 logsumexp,不保存 [N, V]。
2
+
3
+ 标准 ``F.cross_entropy`` 会保存 ``log_softmax`` 的结果用于反向,那是
4
+ ``[N, vocab]`` 的 fp32 张量 —— batch 1/seq 2048/vocab 20000 下 164 MB,
5
+ batch 2 下 328 MB;再叠上调用点显式 ``.float()`` 的那一份,batch 2 合计约 656 MB。
6
+
7
+ 本项目当前卡在显存墙上:batch 2 关掉梯度检查点会颠簸到 11,000 ms/微步,
8
+ 而 batch 2 能把微步数从 4 降到 2、内核启动数从 10,116 降到约 5,000,
9
+ 正好对上实测 20.8% 的 GPU 空闲。**所以这里省下的不是时间,是「能不能上 batch 2」的入场券。**
10
+
11
+ 本实现:前向按块算 logits 得到逐 token 的 CE,只保存 ``lse``(``[N]``);
12
+ 反向按块重算 logits 再求 ``softmax - onehot``,从而完全不需要 ``[N, V]`` 常驻。
13
+ 代价是反向多一遍 LM head 的 GEMM(batch 1 下约 2.5 ms/微步),可忽略。
14
+
15
+ 权重码 ``w`` 的语义与 ``modeling_retnet.py`` 原有的损失完全一致:
16
+ ``loss = Σ ce_i·w_i / Σ w_i``。
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import torch
21
+ import torch.nn.functional as F
22
+
23
+ #: 一次处理多少个 token 的行;峰值显存 ≈ CHUNK × vocab × 4 B。
24
+ LOGIT_CHUNK = 2048
25
+
26
+
27
+ @torch._dynamo.disable
28
+ def _fused_forward(hidden, weight, target, w):
29
+ """前向拆出来并禁用 Dynamo。
30
+
31
+ 分块循环里的 ``h[s:e]`` 是动态切片,Dynamo 追踪不了,会在此处图断裂并
32
+ 让整段退回 eager(实测直接卡住)。用 ``torch._dynamo.disable`` 明确告诉
33
+ 它「这段别管」,图断裂变成显式的边界而不是异常。
34
+ """
35
+ N, D = hidden.shape
36
+ h = hidden.reshape(N, D)
37
+ lse = torch.empty(N, device=hidden.device, dtype=torch.float32)
38
+ tgt_logit = torch.empty(N, device=hidden.device, dtype=torch.float32)
39
+ for s in range(0, N, LOGIT_CHUNK):
40
+ e = min(s + LOGIT_CHUNK, N)
41
+ lg = (h[s:e].float() @ weight.float().t()) # [c, V]
42
+ lse[s:e] = torch.logsumexp(lg, dim=-1)
43
+ tgt_logit[s:e] = lg.gather(1, target[s:e].unsqueeze(1)).squeeze(1)
44
+ del lg
45
+ return lse, tgt_logit
46
+
47
+
48
+ @torch._dynamo.disable
49
+ def _fused_backward(h, weight, target, w, grad_out, wsum):
50
+ N, D = h.shape
51
+ grad_h = torch.zeros_like(h)
52
+ grad_w = torch.zeros_like(weight)
53
+ scale = (grad_out / wsum).to(torch.float32)
54
+ for s in range(0, N, LOGIT_CHUNK):
55
+ e = min(s + LOGIT_CHUNK, N)
56
+ hc = h[s:e].float()
57
+ lg = hc @ weight.float().t() # [c, V]
58
+ p = torch.softmax(lg, dim=-1)
59
+ p.scatter_(1, target[s:e].unsqueeze(1), -1.0) # p - onehot
60
+ p.mul_(w[s:e].unsqueeze(1) * scale) # 权重码 + 上游梯度
61
+ grad_h[s:e] = (p @ weight.float()).to(h.dtype)
62
+ grad_w += (p.t() @ hc).to(weight.dtype)
63
+ del lg, p, hc
64
+ return grad_h, grad_w
65
+
66
+
67
+ class FusedLinearCrossEntropy(torch.autograd.Function):
68
+ @staticmethod
69
+ def forward(ctx, hidden, weight, target, w):
70
+ """hidden [N,D] · weight [V,D] · target [N] · w [N](已按权重码展开)。"""
71
+ lse, tgt_logit = _fused_forward(hidden, weight, target, w)
72
+ wsum = w.sum().clamp_min(1.0)
73
+ loss = ((lse - tgt_logit) * w).sum() / wsum
74
+ ctx.save_for_backward(hidden, weight, target, w)
75
+ ctx.wsum = wsum.detach()
76
+ return loss
77
+
78
+ @staticmethod
79
+ def backward(ctx, grad_out):
80
+ h, weight, target, w = ctx.saved_tensors
81
+ grad_h, grad_w = _fused_backward(h, weight, target, w, grad_out, ctx.wsum)
82
+ return grad_h, grad_w, None, None
83
+
84
+
85
+ def fused_linear_cross_entropy(hidden, weight, target, w):
86
+ return FusedLinearCrossEntropy.apply(hidden, weight, target, w)
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a76da7163b2f9718fc1f112b273efb89347940b6385f9f5ba073a5c6a1235e80
3
+ size 2016755680
modeling_retnet.py ADDED
@@ -0,0 +1,621 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """HuggingFace-style RetNet implementation — algorithm identical to the
2
+ handwritten retnet/model.py, wrapped for the mature ecosystem (Trainer,
3
+ Accelerate, safetensors, PreTrainedModel).
4
+
5
+ Parameter names are kept byte-for-byte identical to the handwritten model so
6
+ that ``load_state_dict(old_checkpoint["model_state"], strict=True)`` works
7
+ without any renaming.
8
+
9
+ model = RetNetForCausalLM(config)
10
+ model.load_state_dict(strip_optimized_prefix(old["model_state"]))
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import os
15
+ from typing import List, Optional, Tuple
16
+
17
+ import torch
18
+ import torch.nn as nn
19
+ import torch.nn.functional as F
20
+ from transformers import PreTrainedModel, PretrainedConfig
21
+
22
+ #: Opt-in batched-chunk chunkwise retention (see ``retention_fast.py``).
23
+ #: Mathematically identical to the reference per-chunk loop but replaces the
24
+ #: Python loop's per-chunk slicing + small matmuls with 3 batched matmuls and a
25
+ #: tiny state scan. Verified equivalent by ``scripts/test_fast_chunkwise.py``.
26
+ FAST_CHUNKWISE = os.environ.get("RETNET_FAST_CHUNKWISE", "0") == "1"
27
+
28
+ # NOTE(2026-10-10): 试过给 ``MultiScaleRetention._rotations`` 加一层
29
+ # ``(length, offset, device, dtype)`` 键的 sin/cos 表缓存,理由是 profile 里
30
+ # ``triton_poi_fused__to_copy*`` 合计 42.9 ms(占一个优化步 5.9%),而这 20 层
31
+ # 每次前向算的都是同一张表。**实测没有收益**:缓存开 1080 ms / 关 1084 ms
32
+ # (同一供电状态下背靠背),逐字符还原成原始实现也仍是 1078 ms。
33
+ # 结论是 inductor 已经把这部分处理掉了,Python 侧的缓存只是在编译区域里
34
+ # 多插一层间接。**不要再加这个缓存。**
35
+
36
+
37
+ class RetNetHFConfig(PretrainedConfig):
38
+ model_type = "retnet_hf"
39
+
40
+ def __init__(self, vocab_size=8192, d_model=512, num_layers=6, num_heads=8,
41
+ value_dim=None, ffn_dim=None, dropout=0.1, norm_eps=1e-5,
42
+ tie_weights=True, embed_scale=None, prompt_loss_weight=0.1,
43
+ use_gla=False, use_swiglu=False, use_growth_ln=False,
44
+ growth_old_dim=None, checkpoint_layers=-1, **kwargs):
45
+ super().__init__(**kwargs)
46
+ self.vocab_size = vocab_size
47
+ self.d_model = d_model
48
+ self.num_layers = num_layers
49
+ self.num_heads = num_heads
50
+ self.value_dim = value_dim if value_dim is not None else 2 * d_model
51
+ self.ffn_dim = ffn_dim if ffn_dim is not None else 2 * d_model
52
+ self.dropout = dropout
53
+ self.norm_eps = norm_eps
54
+ self.tie_weights = tie_weights
55
+ self.embed_scale = embed_scale
56
+ self.prompt_loss_weight = prompt_loss_weight
57
+ self.use_gla = use_gla
58
+ self.use_swiglu = use_swiglu
59
+ self.use_growth_ln = use_growth_ln
60
+ self.growth_old_dim = growth_old_dim
61
+ #: 只对前 N 层做梯度检查点;-1 = 全部(沿用 HF 的全局开关)。
62
+ #: 分层取舍的依据:重算开销与显存占用都随被检查点的层数线性变化,
63
+ #: 而激活值总量也是。取中间值可以在同样的 8 GiB 里换到更大的物理 batch,
64
+ #: 而更大的 batch 又能摊薄内核启动数(见 docs/15 §0.2)。
65
+ self.checkpoint_layers = checkpoint_layers
66
+
67
+ @property
68
+ def key_dim(self) -> int:
69
+ return self.d_model // self.num_heads
70
+
71
+
72
+ def rotate_every_two(x: torch.Tensor) -> torch.Tensor:
73
+ x1 = x[..., 0::2]
74
+ x2 = x[..., 1::2]
75
+ x_rot = torch.stack((-x2, x1), dim=-1)
76
+ return x_rot.flatten(-2)
77
+
78
+
79
+ def theta_shift(x: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor:
80
+ return x * cos + rotate_every_two(x) * sin
81
+
82
+
83
+ class MultiScaleRetention(nn.Module):
84
+ def __init__(self, config: RetNetHFConfig):
85
+ super().__init__()
86
+ d_model, num_heads, value_dim = config.d_model, config.num_heads, config.value_dim
87
+ if d_model % num_heads != 0:
88
+ raise ValueError("d_model must be divisible by num_heads")
89
+ if value_dim % num_heads != 0:
90
+ raise ValueError("value_dim must be divisible by num_heads")
91
+ self.d_model = d_model
92
+ self.num_heads = num_heads
93
+ self.key_dim = d_model // num_heads
94
+ self.value_dim = value_dim
95
+ self.value_head_dim = value_dim // num_heads
96
+ self.scale = self.key_dim ** -0.5
97
+
98
+ gamma = 1.0 - torch.pow(2.0, -5.0 - torch.arange(num_heads, dtype=torch.float32))
99
+ if config.use_gla:
100
+ # GLA: 可学习逐头门控(初始值 = RetNet 固定公式,训练中自适应)
101
+ self.gamma = nn.Parameter(gamma.clone())
102
+ else:
103
+ self.register_buffer("gamma", gamma, persistent=True)
104
+ inv_freq = 1.0 / (10000.0 ** torch.linspace(0.0, 1.0, steps=self.key_dim // 2,
105
+ dtype=torch.float32))
106
+ self.register_buffer("inv_freq", inv_freq.repeat_interleave(2), persistent=False)
107
+
108
+ self.q_proj = nn.Linear(d_model, d_model, bias=False)
109
+ self.k_proj = nn.Linear(d_model, d_model, bias=False)
110
+ self.v_proj = nn.Linear(d_model, value_dim, bias=False)
111
+ self.g_proj = nn.Linear(d_model, value_dim, bias=False)
112
+ self.out_proj = nn.Linear(value_dim, d_model, bias=False)
113
+ self.dropout = nn.Dropout(config.dropout)
114
+ self.group_norm = nn.GroupNorm(num_heads, value_dim, eps=config.norm_eps)
115
+
116
+ def _init_weights(self):
117
+ for name, p in self.named_parameters():
118
+ if name.endswith("_proj.weight"):
119
+ if name.split(".")[-1].startswith(("q_", "k_", "v_", "g_")):
120
+ nn.init.xavier_uniform_(p, gain=2 ** -2.5)
121
+ else:
122
+ nn.init.xavier_uniform_(p, gain=1.0)
123
+ elif name == "gamma":
124
+ # GLA 的逐头门控,初值必须是 __init__ 里按 RetNet 固定公式算出的
125
+ # 1 - 2^(-5-h),训练中再自适应。**绝不能置零**:
126
+ # 之前这里写的是「1 维 Parameter 一律 zeros_」兜底 bias,可本模块
127
+ # 所有 _proj 都是 bias=False,于是唯一的 1 维 Parameter 就是 gamma,
128
+ # 门控被置零 → log(0) = -inf → 掩码对角线 exp(-inf*0) = NaN,
129
+ # 批量 chunkwise 前向整层 NaN(use_gla=True 时每一条路径都中招)。
130
+ # 参考实现用 gamma ** delta,0**0 = 1 良定义,所以两条路径在
131
+ # gamma=0 处还**不等价**。详见 tests/test_gla_gamma_init.py。
132
+ continue
133
+ elif name.endswith("bias"):
134
+ nn.init.zeros_(p)
135
+
136
+ def _rotations(self, length, offset, device, dtype):
137
+ position = torch.arange(offset, offset + length, device=device, dtype=torch.float32)
138
+ freqs = torch.outer(position, self.inv_freq.to(device=device, dtype=torch.float32))
139
+ return torch.sin(freqs).to(dtype=dtype), torch.cos(freqs).to(dtype=dtype)
140
+
141
+ def _project_and_rotate(self, x, offset):
142
+ bsz, tgt_len, _ = x.shape
143
+ q = self.q_proj(x).view(bsz, tgt_len, self.num_heads, self.key_dim).transpose(1, 2)
144
+ k = self.k_proj(x).view(bsz, tgt_len, self.num_heads, self.key_dim).transpose(1, 2)
145
+ v = self.v_proj(x).view(bsz, tgt_len, self.num_heads, self.value_head_dim).transpose(1, 2)
146
+ g = self.g_proj(x)
147
+ k = k * self.scale
148
+ sin, cos = self._rotations(tgt_len, offset, x.device, q.dtype)
149
+ q = theta_shift(q, sin, cos)
150
+ k = theta_shift(k, sin, cos)
151
+ return q, k, v, g
152
+
153
+ @staticmethod
154
+ def _head_group_norm(x, num_heads, group_norm, bsz, tgt_len):
155
+ """GroupNorm over ``[B,H,L,vhd]``,输出 ``[B,L,H*vhd]``。
156
+
157
+ NOTE(2026-10-10): 试过改写成「先在 ``[B,H,L,vhd]`` 上手工算 mean/var
158
+ 归一化、最后再 permute 一次」,理由是省掉参考实现里那次物化拷贝。
159
+ **实测反而慢 9.2%**(190 vs 174 ms/微步,真实 Trainer 路径,两轮一致):
160
+ 参考实现把 permute 放在**归一化之前**,那次拷贝随即被逐元素归一化核吸收;
161
+ 而放到**之后**,它直接成为 ``out_proj`` 这个外部 cuBLAS 调用的输入,
162
+ 再也融不进任何东西。等价性没问题(fp32 相对差 1.6e-7,bf16 5.1e-3 均
163
+ 在 1 ULP 量级,见 ``tests/test_head_group_norm.py``),但收益是负的。
164
+ **不要再动这个布局。**
165
+ """
166
+ x = x.transpose(1, 2).reshape(bsz * tgt_len, num_heads * x.shape[-1])
167
+ x = group_norm(x.unsqueeze(-1)).squeeze(-1)
168
+ return x.reshape(bsz, tgt_len, -1)
169
+
170
+ def _decay_mask(self, length, device, dtype):
171
+ idx = torch.arange(length, device=device)
172
+ delta = idx[:, None] - idx[None, :]
173
+ causal = delta >= 0
174
+ exp = delta.masked_fill(~causal, 0)
175
+ mask = self.gamma.to(device=device, dtype=torch.float32)[:, None, None] ** exp
176
+ mask = mask.masked_fill(~causal, 0.0)
177
+ mask = mask / mask.sum(dim=-1, keepdim=True).sqrt().clamp_min(1e-8)
178
+ return mask.to(dtype=dtype)
179
+
180
+ def _parallel_retention(self, q, k, v):
181
+ bsz, _, tgt_len, _ = q.shape
182
+ mask = self._decay_mask(tgt_len, q.device, q.dtype)
183
+ scores = torch.matmul(q, k.transpose(-1, -2)) * mask
184
+ scores = scores / scores.detach().abs().sum(dim=-1, keepdim=True).clamp_min(1.0)
185
+ return torch.matmul(scores, v)
186
+
187
+ def _parallel_final_state(self, k, v, state):
188
+ bsz, _, tgt_len, key_dim = k.shape
189
+ idx = torch.arange(tgt_len, device=k.device)
190
+ weights = self.gamma.to(device=k.device, dtype=torch.float32)[:, None] ** (tgt_len - 1 - idx)
191
+ new_state = torch.einsum("bhtk,bhtv,ht->bhkv", k, v, weights.to(k.dtype))
192
+ if state is not None:
193
+ gamma_pow = self.gamma.to(device=k.device, dtype=k.dtype) ** tgt_len
194
+ new_state = gamma_pow.view(1, -1, 1, 1) * state + new_state
195
+ return new_state
196
+
197
+ def _recurrent_retention(self, q, k, v, state):
198
+ bsz, _, tgt_len, key_dim = q.shape
199
+ value_head_dim = v.shape[-1]
200
+ if state is None:
201
+ state = torch.zeros(bsz, self.num_heads, key_dim, value_head_dim,
202
+ device=q.device, dtype=q.dtype)
203
+ outputs = []
204
+ gamma = self.gamma.to(device=q.device, dtype=q.dtype)
205
+ for t in range(tgt_len):
206
+ state = gamma.view(1, -1, 1, 1) * state + \
207
+ k[:, :, t, :].unsqueeze(-1) * v[:, :, t, :].unsqueeze(-2)
208
+ out_t = torch.sum(q[:, :, t, :].unsqueeze(-1) * state, dim=-2)
209
+ outputs.append(out_t)
210
+ output = torch.stack(outputs, dim=2)
211
+ return output, state
212
+
213
+ def _chunkwise_retention(self, q, k, v, state, chunk_size):
214
+ """Equation (7): inner-chunk parallel + cross-chunk recurrent state."""
215
+ if FAST_CHUNKWISE:
216
+ from .retention_fast import chunkwise_batched
217
+ return chunkwise_batched(q, k, v, self.gamma, state, chunk_size)
218
+ bsz, _, tgt_len, key_dim = q.shape
219
+ if tgt_len % chunk_size != 0:
220
+ raise ValueError(f"chunkwise mode requires seq_len ({tgt_len}) to be a "
221
+ f"multiple of chunk_size ({chunk_size})")
222
+ if state is None:
223
+ state = torch.zeros(bsz, self.num_heads, key_dim, v.shape[-1],
224
+ device=q.device, dtype=q.dtype)
225
+ gamma = self.gamma.to(device=q.device, dtype=q.dtype)
226
+ outputs = []
227
+ rel = torch.arange(chunk_size, device=q.device)
228
+ delta = rel[:, None] - rel[None, :]
229
+ causal = delta >= 0
230
+ inner_mask = torch.where(causal, gamma[:, None, None] ** delta.masked_fill(~causal, 0),
231
+ torch.zeros_like(delta, dtype=torch.float32))
232
+ cross_decay = gamma[:, None] ** (rel + 1)[None, :]
233
+ value_decay = gamma[:, None] ** (chunk_size - 1 - rel)[None, :]
234
+ chunk_decay = gamma ** chunk_size
235
+
236
+ for i in range(tgt_len // chunk_size):
237
+ sl = slice(i * chunk_size, (i + 1) * chunk_size)
238
+ qi, ki, vi = q[:, :, sl], k[:, :, sl], v[:, :, sl]
239
+ inner_scores = torch.matmul(qi, ki.transpose(-1, -2)) * inner_mask
240
+ inner = torch.matmul(inner_scores, vi)
241
+ cross = torch.matmul(qi * cross_decay[None, :, :, None], state)
242
+ outputs.append(inner + cross)
243
+ weighted_v = vi * value_decay[None, :, :, None]
244
+ state = chunk_decay.view(1, -1, 1, 1) * state + \
245
+ torch.matmul(ki.transpose(-1, -2), weighted_v)
246
+
247
+ output = torch.cat(outputs, dim=2)
248
+ return output, state
249
+
250
+ def forward(self, x, mode="parallel", state=None, offset=0, return_state=False,
251
+ chunk_size=64):
252
+ bsz, tgt_len, _ = x.shape
253
+ q, k, v, g = self._project_and_rotate(x, offset)
254
+
255
+ if mode == "parallel":
256
+ output, new_state = self._parallel_retention(q, k, v), None
257
+ if return_state:
258
+ new_state = self._parallel_final_state(k, v, state)
259
+ elif mode == "recurrent":
260
+ output, new_state = self._recurrent_retention(q, k, v, state)
261
+ elif mode == "chunkwise":
262
+ output, new_state = self._chunkwise_retention(q, k, v, state, chunk_size)
263
+ else:
264
+ raise ValueError(f"unknown mode: {mode}")
265
+
266
+ output = self._head_group_norm(output, self.num_heads, self.group_norm, bsz, tgt_len)
267
+ output = F.silu(g) * output
268
+ output = self.dropout(output)
269
+ output = self.out_proj(output)
270
+ if return_state:
271
+ return output, new_state
272
+ return output
273
+
274
+
275
+ class GrowthLayerNorm(nn.Module):
276
+ """Two-segment LayerNorm for function-preserving width growth.
277
+
278
+ The old ``old_dim`` channels are normalized exactly as before (statistics
279
+ unaffected by new channels); the new channels are scaled independently with
280
+ ``gamma_new``/``beta_new`` and stay a zero side branch while
281
+ ``gamma_new=0, beta_new=0``. This is what makes pre-LN width expansion
282
+ strictly function-preserving (BERT2BERT-style FPI for LayerNorm).
283
+
284
+ state_dict keys: ``ln_old.weight``, ``ln_old.bias``, ``gamma_new``, ``beta_new``.
285
+ """
286
+
287
+ def __init__(self, old_dim: int, new_dim: int, eps: float = 1e-5,
288
+ gamma_new: float = 1.0, beta_new: float = 0.0):
289
+ super().__init__()
290
+ self.ln_old = nn.LayerNorm(old_dim, eps=eps)
291
+ self.gamma_new = nn.Parameter(torch.full((new_dim - old_dim,), gamma_new))
292
+ self.beta_new = nn.Parameter(torch.full((new_dim - old_dim,), beta_new))
293
+
294
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
295
+ old = self.ln_old(x[..., : self.ln_old.normalized_shape[0]])
296
+ new = x[..., self.ln_old.normalized_shape[0]:] * self.gamma_new + self.beta_new
297
+ return torch.cat([old, new], dim=-1)
298
+
299
+
300
+ class RetNetBlock(nn.Module):
301
+ def _norm(self, config: RetNetHFConfig, dim: int):
302
+ if config.use_growth_ln:
303
+ return GrowthLayerNorm(config.growth_old_dim, dim, eps=config.norm_eps,
304
+ gamma_new=1.0, beta_new=0.0)
305
+ return nn.LayerNorm(dim, eps=config.norm_eps)
306
+
307
+ def __init__(self, config: RetNetHFConfig, layer_idx: int = 0):
308
+ super().__init__()
309
+ self.layer_idx = layer_idx
310
+ #: 该层是否做梯度检查点。-1 表示「跟随 HF 的全局开关」。
311
+ self.checkpoint_layers = getattr(config, "checkpoint_layers", -1)
312
+ self.msr = MultiScaleRetention(config)
313
+ self.norm1 = self._norm(config, config.d_model)
314
+ self.gradient_checkpointing = False
315
+ if config.use_swiglu:
316
+ # SwiGLU FFN:y = silu(xW_gate) * (xW_up) -> W_down
317
+ self.ffn = nn.Sequential() # placeholder; real modules below
318
+ self.gate_proj = nn.Linear(config.d_model, config.ffn_dim, bias=False)
319
+ self.up_proj = nn.Linear(config.d_model, config.ffn_dim, bias=False)
320
+ self.down_proj = nn.Linear(config.ffn_dim, config.d_model, bias=False)
321
+ self.swiglu = True
322
+ else:
323
+ self.ffn = nn.Sequential(
324
+ nn.Linear(config.d_model, config.ffn_dim, bias=False),
325
+ nn.GELU(),
326
+ nn.Linear(config.ffn_dim, config.d_model, bias=False),
327
+ )
328
+ self.swiglu = False
329
+ self.norm2 = self._norm(config, config.d_model)
330
+ self.dropout = nn.Dropout(config.dropout)
331
+
332
+ def _block_forward(self, x, mode, state, offset, return_state, chunk_size=64):
333
+ msr_out = self.msr(self.norm1(x), mode=mode, state=state, offset=offset,
334
+ return_state=return_state, chunk_size=chunk_size)
335
+ if return_state:
336
+ y, new_state = msr_out
337
+ else:
338
+ y, new_state = msr_out, None
339
+ y = x + y
340
+ if self.swiglu:
341
+ ffn_out = F.silu(self.gate_proj(self.norm2(y))) * self.up_proj(self.norm2(y))
342
+ ffn_out = self.down_proj(ffn_out)
343
+ else:
344
+ ffn_out = self.ffn(self.norm2(y))
345
+ y = y + self.dropout(ffn_out)
346
+ if return_state:
347
+ return y, new_state
348
+ return y
349
+
350
+ def forward(self, x, mode="parallel", state=None, offset=0, return_state=False,
351
+ chunk_size=64):
352
+ do_ckpt = self.gradient_checkpointing and self.training
353
+ if do_ckpt and 0 <= self.checkpoint_layers <= self.layer_idx:
354
+ do_ckpt = False # 分层策略:只检查点前 checkpoint_layers 层
355
+ if do_ckpt:
356
+ return torch.utils.checkpoint.checkpoint(
357
+ self._block_forward, x, mode, state, offset, return_state, chunk_size,
358
+ use_reentrant=False)
359
+ return self._block_forward(x, mode, state, offset, return_state, chunk_size)
360
+
361
+
362
+ class RetNetForCausalLM(PreTrainedModel):
363
+ """RetNet language model with a from-scratch token embedding.
364
+
365
+ Forward signature is Trainer-compatible:
366
+ forward(input_ids, labels=None, wcodes=None, ...) -> CausalLMOutput
367
+ When labels+wcodes are given the loss is the wcode-weighted cross entropy
368
+ (wcode: 0=pad, 1=reply, 2=prompt, 3=plain LM) exactly like train.py.
369
+ """
370
+ config_class = RetNetHFConfig
371
+ base_model_prefix = "retnet"
372
+ supports_gradient_checkpointing = True
373
+
374
+ #: 本模型**不使用** ``num_items_in_batch`` 做损失归一化(loss 已按权重码
375
+ #: ``w.sum()`` 归一),但 ``forward`` 签名末尾的 ``**_kwargs`` 会让 HF 的启发式
376
+ #: 判定「模型自己会处理归一化」,从而**跳过** ``loss = loss / grad_accum_steps``。
377
+ #: 后果是梯度被放大 accum 倍、等效学习率成倍上升 —— 静默,且只在 accum>1 时发作。
378
+ #: HF 按 **``accepts_loss_kwargs``**(不是 ``model_accepts_loss_kwargs``)查找;
379
+ #: 找不到时它会去猜 forward 签名,本项目会被误判为 True。
380
+ accepts_loss_kwargs = False
381
+ model_accepts_loss_kwargs = False
382
+
383
+ def __init__(self, config: RetNetHFConfig):
384
+ super().__init__(config)
385
+ self.embedding = nn.Embedding(config.vocab_size, config.d_model)
386
+ self.embed_dropout = nn.Dropout(config.dropout)
387
+ self.embed_scale = config.embed_scale if config.embed_scale is not None \
388
+ else config.d_model ** 0.5
389
+ self.blocks = nn.ModuleList([RetNetBlock(config, i)
390
+ for i in range(config.num_layers)])
391
+ if config.use_growth_ln:
392
+ self.final_norm = GrowthLayerNorm(config.growth_old_dim, config.d_model,
393
+ eps=config.norm_eps,
394
+ gamma_new=1.0, beta_new=0.0)
395
+ else:
396
+ self.final_norm = nn.LayerNorm(config.d_model, eps=config.norm_eps)
397
+ self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
398
+ if config.tie_weights:
399
+ self.lm_head.weight = self.embedding.weight
400
+ self._tied_weights_keys = ["lm_head.weight"]
401
+ self._init_weights_local()
402
+
403
+ def _init_weights_local(self):
404
+ nn.init.normal_(self.embedding.weight, mean=0.0, std=0.02)
405
+ for block in self.blocks:
406
+ if block.swiglu:
407
+ nn.init.xavier_uniform_(block.gate_proj.weight)
408
+ nn.init.xavier_uniform_(block.up_proj.weight)
409
+ nn.init.xavier_uniform_(block.down_proj.weight)
410
+ else:
411
+ for name, p in block.ffn.named_parameters():
412
+ if name.endswith("weight"):
413
+ nn.init.xavier_uniform_(p)
414
+ block.msr._init_weights()
415
+ if not self.config.tie_weights:
416
+ nn.init.xavier_uniform_(self.lm_head.weight)
417
+
418
+ def tie_weights(self):
419
+ """共享 lm_head 与 embedding(from_pretrained 后恢复 tie)。"""
420
+ if self.config.tie_weights:
421
+ self.lm_head.weight = self.embedding.weight
422
+
423
+ def set_checkpoint_layers(self, n: int):
424
+ """只对前 ``n`` 层做梯度检查点(n<0 = 全部)。
425
+
426
+ 必须在 HF 启用梯度检查点**之后**调用,因为它会把所有层的
427
+ ``gradient_checkpointing`` 置 True。
428
+ """
429
+ self.config.checkpoint_layers = n
430
+ for b in self.blocks:
431
+ b.checkpoint_layers = n
432
+ return n
433
+
434
+ def _forward_hidden(self, input_ids, mode="parallel", states=None, offset=0,
435
+ return_state=False, chunk_size=64, return_hidden=False):
436
+ if input_ids.ndim != 2:
437
+ raise ValueError("input_ids must be [batch, seq]")
438
+ x = self.embedding(input_ids) * self.embed_scale
439
+ x = self.embed_dropout(x)
440
+ if states is None:
441
+ states = [None] * len(self.blocks)
442
+ if len(states) != len(self.blocks):
443
+ raise ValueError("states must contain one entry per block")
444
+ new_states: List[Optional[torch.Tensor]] = []
445
+ for block, block_state in zip(self.blocks, states):
446
+ if return_state or (mode == "recurrent" and block_state is not None):
447
+ x, ns = block(x, mode=mode, state=block_state, offset=offset,
448
+ return_state=True, chunk_size=chunk_size)
449
+ new_states.append(ns)
450
+ else:
451
+ x = block(x, mode=mode, state=block_state, offset=offset,
452
+ chunk_size=chunk_size)
453
+ new_states.append(block_state)
454
+ x = self.final_norm(x)
455
+ if return_hidden:
456
+ # 融合交叉熵需要的是 LM head 的**输入**(hidden),不是输出。
457
+ # 这里提前返回可以完全不物化 [B,L,V]。
458
+ if return_state or mode == "recurrent":
459
+ return x, new_states
460
+ return x
461
+ logits = self.lm_head(x)
462
+ if return_state or mode == "recurrent":
463
+ return logits, new_states
464
+ return logits
465
+
466
+ def forward(self, input_ids, labels=None, wcodes=None, mode=None,
467
+ states=None, offset=0, return_state=False, chunk_size=None, **_kwargs):
468
+ # NOTE(2026-10-10): 试过在这里调 torch.compiler.cudagraph_mark_step_begin()
469
+ # 以启用 torch.compile(mode="reduce-overhead") 的 CUDA Graphs。**无效**——
470
+ # 错误从损失移到 _forward_hidden,仍是 "accessing tensor output of
471
+ # CUDAGraphs that has been overwritten / Gradient addition node due to
472
+ # multiple use of tensor":这是 CUDA Graphs 与梯度累积(同一张图跑 4 次、
473
+ # 各自反向)的硬冲突,不是漏调一个 API 能解决的。已回滚。
474
+ # NOTE(2026-10-10): 上面这个默认值以前写的是 64,而 HF Trainer 调用模型时
475
+ # **不会**传这个关键字,于是 ``config.chunk_size`` 和任务 JSON 里的
476
+ # ``chunk_size`` 一直是**死字段**——项目以为在跑 128,实际永远 64。
477
+ # 实测:``config.chunk_size`` 设 64/128/256 的 loss **逐位相同**,而显式
478
+ # 传 ``chunk_size=`` 会改变 loss。``bench_step.py`` 的
479
+ # ``--sweep chunk_size=...`` 因此扫的是空变量,``docs/15`` §6 的
480
+ # 「chunk 512/1024 反而更慢,最优是 128」不成立。
481
+ # 现改成 None 再回落到 config,任务配置才真正生效。
482
+ # ⚠️ 对已经在跑的任务,这会让 chunk 从 64 跳到配置值 —— 数值会跳变,
483
+ # 只能在**新开一轮**时启用,不要热切。
484
+ if chunk_size is None:
485
+ chunk_size = int(getattr(self.config, "chunk_size", 64) or 64)
486
+ if mode is None:
487
+ mode = getattr(self.config, "mode", "parallel")
488
+ _fused = (os.environ.get("RETNET_FUSED_CE") == "1"
489
+ and labels is not None and wcodes is not None)
490
+ logits = None
491
+ # `loss` 必须在所有分支之前初始化:不传 labels 时(普通推理、prefill、
492
+ # 以及 HF 的 generate 路径)上面几个分支都不会赋值,line 536 的
493
+ # CausalLMOutput(loss=loss, ...) 会直接 UnboundLocalError。
494
+ loss = None
495
+ if _fused:
496
+ # 融合路径:LM head 的输入直接进 fused CE,完全不物化 [B,L,V]
497
+ hidden, new_states = self._forward_hidden(
498
+ input_ids, mode=mode, states=states, offset=offset,
499
+ return_state=True, chunk_size=chunk_size, return_hidden=True)
500
+ B_, L_, D_ = hidden.shape
501
+ w = torch.zeros_like(wcodes, dtype=torch.float32)
502
+ w = torch.where(wcodes == 1, torch.ones_like(w), w)
503
+ w = torch.where(wcodes == 2,
504
+ torch.full_like(w, self.config.prompt_loss_weight), w)
505
+ w = torch.where(wcodes == 3, torch.ones_like(w), w)
506
+ from .fused_ce import fused_linear_cross_entropy
507
+ loss = fused_linear_cross_entropy(
508
+ hidden.reshape(B_ * L_, D_), self.lm_head.weight,
509
+ labels.reshape(-1), w.reshape(-1))
510
+ elif return_state or mode == "recurrent" or mode == "chunkwise":
511
+ logits, new_states = self._forward_hidden(input_ids, mode=mode, states=states,
512
+ offset=offset, return_state=True,
513
+ chunk_size=chunk_size)
514
+ else:
515
+ logits = self._forward_hidden(input_ids, mode=mode, states=states, offset=offset)
516
+ new_states = None
517
+
518
+ if not _fused:
519
+ # 非融合路径:沿用原来的显式 fp32 上采样。
520
+ # NOTE(2026-10-10): 试过去掉 ``logits.float()``(让 ``F.cross_entropy``
521
+ # 直接吃 bf16)。loss 逐位一致但**步时 0% 变化**——profile 把这两个
522
+ # 融合核记成 3.9%,是归因假象。故保留原状。
523
+ if labels is not None and wcodes is not None:
524
+ logits_f = logits.float()
525
+ token_loss = F.cross_entropy(
526
+ logits_f.reshape(-1, logits_f.shape[-1]), labels.reshape(-1),
527
+ reduction="none")
528
+ w = torch.zeros_like(wcodes, dtype=torch.float32)
529
+ w = torch.where(wcodes == 1, torch.ones_like(w), w)
530
+ w = torch.where(wcodes == 2,
531
+ torch.full_like(w, self.config.prompt_loss_weight), w)
532
+ w = torch.where(wcodes == 3, torch.ones_like(w), w)
533
+ loss = (token_loss * w.reshape(-1)).sum() / w.sum().clamp_min(1.0)
534
+ elif labels is not None:
535
+ logits_f = logits.float()
536
+ loss = F.cross_entropy(logits_f.reshape(-1, logits_f.shape[-1]),
537
+ labels.reshape(-1))
538
+
539
+ from transformers.modeling_outputs import CausalLMOutput
540
+ return CausalLMOutput(loss=loss, logits=logits)
541
+
542
+ @torch.inference_mode()
543
+ def generate(self, input_ids: torch.Tensor, max_new_tokens: int = 32,
544
+ temperature: float = 1.0, top_k: Optional[int] = None,
545
+ top_p: Optional[float] = None, repetition_penalty: float = 1.0,
546
+ eos_id: Optional[int] = None, device: Optional[torch.device] = None,
547
+ penalize_prompt: bool = False, no_repeat_ngram_size: int = 0,
548
+ on_token=None) -> torch.Tensor:
549
+ """Autoregressive generation (same algorithm/sampling as handwritten).
550
+
551
+ ``on_token(token_id)`` is called for every accepted token, in order, as
552
+ soon as it is sampled. Returning a truthy value stops generation
553
+ immediately (the token is still part of the returned sequence). Used by
554
+ the interactive play script to stream output and to cut a reply at the
555
+ first ``用户:`` / ``助手:`` turn boundary.
556
+ """
557
+ device = device or next(self.parameters()).device
558
+ ids = input_ids.to(device)
559
+ states = [None] * len(self.blocks)
560
+ if ids.shape[1] > 1:
561
+ logits, states = self._forward_hidden(ids, mode="parallel", return_state=True)
562
+ next_logits = logits[:, -1]
563
+ else:
564
+ next_logits, states = self._forward_hidden(ids, mode="recurrent", states=states,
565
+ offset=0, return_state=True)
566
+ next_logits = next_logits[:, -1]
567
+ generated = list(ids[0].tolist())
568
+ prompt_len = ids.shape[1]
569
+ offset = ids.shape[1]
570
+
571
+ for _ in range(max_new_tokens):
572
+ penalty_source = generated if penalize_prompt else generated[prompt_len:]
573
+ banned = []
574
+ if no_repeat_ngram_size and len(generated) >= no_repeat_ngram_size:
575
+ n = no_repeat_ngram_size
576
+ suffix = tuple(generated[-(n - 1):]) if n > 1 else ()
577
+ for i in range(len(generated) - n + 1):
578
+ if tuple(generated[i:i + n - 1]) == suffix:
579
+ banned.append(generated[i + n - 1])
580
+ if temperature > 0:
581
+ logits = next_logits / temperature
582
+ if repetition_penalty and repetition_penalty != 1.0:
583
+ for token_id in set(penalty_source):
584
+ score = logits[0, token_id]
585
+ logits[0, token_id] = score / repetition_penalty if score >= 0 \
586
+ else score * repetition_penalty
587
+ for token_id in banned:
588
+ logits[0, token_id] = -torch.inf
589
+ if top_k is not None:
590
+ k = min(top_k, logits.shape[-1])
591
+ top_values, _ = torch.topk(logits, k)
592
+ logits[logits < top_values[:, -1:]] = -torch.inf
593
+ probs = F.softmax(logits, dim=-1)
594
+ if top_p is not None and top_p < 1.0:
595
+ sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
596
+ cumsum = sorted_probs.cumsum(dim=-1)
597
+ remove = cumsum > top_p
598
+ remove[..., 1:] = remove[..., :-1].clone()
599
+ remove[..., 0] = False
600
+ filtered = torch.zeros_like(probs)
601
+ filtered.scatter_(1, sorted_indices[~remove].unsqueeze(0),
602
+ sorted_probs[~remove].unsqueeze(0))
603
+ probs = filtered / filtered.sum(dim=-1, keepdim=True)
604
+ next_id = torch.multinomial(probs, num_samples=1).item()
605
+ else:
606
+ logits = next_logits.clone()
607
+ for token_id in banned:
608
+ logits[0, token_id] = -torch.inf
609
+ next_id = logits.argmax(dim=-1).item()
610
+ if eos_id is not None and next_id == eos_id:
611
+ break
612
+ generated.append(next_id)
613
+ if on_token is not None and on_token(next_id):
614
+ break
615
+ next_token = torch.tensor([[next_id]], device=device, dtype=torch.long)
616
+ next_logits, states = self._forward_hidden(next_token, mode="recurrent",
617
+ states=states, offset=offset,
618
+ return_state=True)
619
+ next_logits = next_logits[:, -1]
620
+ offset += 1
621
+ return torch.tensor([generated], device=device, dtype=torch.long)
retention_fast.py ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Batched-chunk chunkwise retention — mathematically identical to the reference
2
+ per-chunk loop in ``retnet_hf/modeling_retnet.py``, but with ~30x fewer kernel
3
+ launches.
4
+
5
+ Reference (per-chunk loop, 4 matmuls per chunk):
6
+
7
+ for i in chunks:
8
+ inner = (q_i @ k_i.T * inner_mask) @ v_i
9
+ cross = (q_i * cross_decay) @ state
10
+ out_i = inner + cross
11
+ state = chunk_decay * state + k_i.T @ (v_i * value_decay)
12
+
13
+ Batched form (3 matmuls total, independent of the chunk count):
14
+
15
+ 1. U_i = k_i.T @ (v_i * value_decay) one batched matmul
16
+ 2. S_prev_i = scan(U, chunk_decay, state) tiny sequential loop on
17
+ [B,H,kd,vd] tensors only
18
+ 3. inner = (q_r @ k_r.T * inner_mask) @ v_r one batched matmul
19
+ cross = (q_r * cross_decay) @ S_prev one batched matmul
20
+
21
+ The scan in step 2 is deliberately *not* replaced by a cumsum: with
22
+ ``chunk_decay = gamma^C`` and C=256, ``gamma^-j`` overflows fp32 for the fast
23
+ heads, while the log-depth pairwise scan costs more than it saves at 8 chunks.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import torch
28
+
29
+
30
+ _INDEX_CACHE: dict = {}
31
+
32
+
33
+ def _index_tables(chunk_size: int, device):
34
+ """``causal`` / ``expo`` / ``rel`` 只依赖 chunk_size,缓存掉。
35
+
36
+ 原实现每次调用都重算 ``arange``/``sub``/``ge``/``clamp``,是 4 次无用的小
37
+ 内核发射;而本算子的成本**几乎全是发射延迟**(见下)。
38
+ """
39
+ key = (int(chunk_size), str(device))
40
+ tables = _INDEX_CACHE.get(key)
41
+ if tables is None:
42
+ rel = torch.arange(chunk_size, device=device)
43
+ delta = rel[:, None] - rel[None, :]
44
+ causal = delta >= 0
45
+ expo = delta.clamp_min(0).to(torch.float32)
46
+ tables = (causal, expo, rel.to(torch.float32))
47
+ _INDEX_CACHE[key] = tables
48
+ return tables
49
+
50
+
51
+ def _decay_tables(gamma: torch.Tensor, chunk_size: int, dtype: torch.dtype,
52
+ device: torch.device):
53
+ """gamma: [H] (fp32)。返回 inner_mask [H,C,C], cross_decay [H,C],
54
+ value_decay [H,C], chunk_decay [H],均为 ``dtype``。
55
+
56
+ 块内掩码用**对数空间** ``exp(log γ · delta)``,这是唯一在本模型门控范围
57
+ 两端都安全的写法:
58
+
59
+ * 因式分解的 ``γ^i · γ^-j`` 一旦需要 ``γ^-255`` 就溢出 fp32。
60
+ ``retnet_272m_rp`` 的 GLA 门控最小到 0.668,``0.668**-255 = 1e44 > 3.4e38``
61
+ → Inf → 前向 NaN(``torch.where`` 只丢结果,不阻止 Inf 被算出来)。
62
+ * 朴素 ``γ ** delta`` 对 γ<1 安全(下溢到 0,那里本来就该是 0),
63
+ 但对 γ>1 的门控不成立;对数形式两端都对。
64
+
65
+ NOTE(2026-10-10): 这个算子只有 26 万个元素却要花 **410 µs** —— 成本几乎全是
66
+ 「22 个相互依赖的小内核」的发射延迟(Windows 上约 18 µs/次)。所以这里的
67
+ 优化目标一律是**减少发射次数**,而不是减少计算量:
68
+ * 索引张量进缓存(省 4 次发射);
69
+ * 三张一维表复用同一个 ``γ^j``:[H,C]:``cross = γ^j·γ``、
70
+ ``value = flip(γ^j)``、``chunk = γ^j[-1]·γ``,
71
+ 把原来的 3 次 ``exp`` 压成 1 次。
72
+ """
73
+ causal, expo, relf = _index_tables(chunk_size, device)
74
+ g = gamma.to(device=device, dtype=torch.float32)
75
+ # γ 可以合法地取到 0(门控被训练成「完全遗忘」),此时 log(0) = -inf,
76
+ # 而对角线 delta=0 会让 exp(-inf * 0) = exp(NaN) —— torch.where 只丢结果、
77
+ # 不阻止 NaN 被算出来,于是整层 NaN。夹一个正下限后
78
+ # exp(logγ·0) = 1、exp(logγ·k) 下溢到 0,正好是参考实现 gamma**delta 的
79
+ # 语义(0**0 = 1、0**k = 0),两条路径在 γ=0 处重新等价。
80
+ log_gamma = torch.log(g.clamp_min(torch.finfo(torch.float32).tiny))
81
+
82
+ inner_mask = torch.exp(log_gamma[:, None, None] * expo[None])
83
+ inner_mask = torch.where(causal, inner_mask,
84
+ torch.zeros((), device=device, dtype=torch.float32))
85
+
86
+ pow1d = torch.exp(log_gamma[:, None] * relf[None, :]) # [H,C] = γ^j
87
+ cross_decay = pow1d * g[:, None] # γ^(j+1)
88
+ value_decay = pow1d.flip(-1) # γ^(C-1-j)
89
+ chunk_decay = pow1d[:, -1] * g # γ^C
90
+
91
+ return (inner_mask.to(dtype), cross_decay.to(dtype),
92
+ value_decay.to(dtype), chunk_decay.to(dtype))
93
+
94
+
95
+ def chunkwise_batched(q, k, v, gamma, state, chunk_size):
96
+ """Drop-in replacement for ``MultiScaleRetention._chunkwise_retention``.
97
+
98
+ q, k: [B,H,L,kd] v: [B,H,L,vd] gamma: [H] fp32
99
+ state: [B,H,kd,vd] or None -> (output [B,H,L,vd], new_state)
100
+ """
101
+ B, H, L, kd = q.shape
102
+ vd = v.shape[-1]
103
+ if L % chunk_size != 0:
104
+ raise ValueError(f"seq_len {L} not a multiple of chunk_size {chunk_size}")
105
+ nc = L // chunk_size
106
+ C = chunk_size
107
+ dev = q.device
108
+
109
+ inner_mask, cross_decay, value_decay, chunk_decay = _decay_tables(
110
+ gamma.to(device=dev, dtype=torch.float32), C, q.dtype, dev)
111
+
112
+ has_state = state is not None
113
+ if state is None:
114
+ state = torch.zeros(B, H, kd, vd, device=dev, dtype=q.dtype)
115
+
116
+ # [B,H,nc,C,*] views — no copy (L is contiguous)
117
+ qr = q.view(B, H, nc, C, kd)
118
+ kr = k.view(B, H, nc, C, kd)
119
+ vr = v.view(B, H, nc, C, vd)
120
+
121
+ # (1) per-chunk state contributions, one batched matmul over all chunks
122
+ U = torch.matmul(kr.transpose(-1, -2),
123
+ vr * value_decay[None, :, None, :, None]) # [B,H,nc,kd,vd]
124
+
125
+ # (2) exclusive scan -> S_prev[i] = state before chunk i
126
+ #
127
+ # NOTE(2026-10-10): 原来的写法是 nc 次串行 ``s = decay*s + U[:,:,i]``
128
+ # 外加一次 ``torch.stack`` —— nc=16 时 33 次内核发射、实测 **591 µs**,
129
+ # 而每个参与的张量只有 0.5 MB:几乎全部是发射延迟。
130
+ #
131
+ # 关键观察是 ``decay = chunk_decay`` 在整个循环里**是常量**(它是 [H] 的
132
+ # 每头常数,不随 chunk 变),于是有闭式解
133
+ #
134
+ # S_prev[i] = d^i · state + Σ_{j<i} d^(i-1-j) · U_j
135
+ #
136
+ # 这正是对 ``T[h,i,j] = d_h^(i-1-j)``(下三角)的一次批量矩阵乘:
137
+ #
138
+ # S_all = T[None] @ U.reshape(B,H,nc,kd*vd) [B,H,nc+1,kd*vd]
139
+ #
140
+ # 多出来的一行 i=nc 恰好就是 new_state。**所有指数非负**,因此不存在
141
+ # §2.1 那种 ``γ^-j`` 溢出——d 再小也只是下溢到 0,那本就代表历史被忘光。
142
+ #
143
+ # NOTE(2026-10-10): 之前试过对数深度(Hillis-Steele)扫描把启动数从 2·nc
144
+ # 降到 ~2·log2(nc),**反而慢 2.8 倍**(3004 vs 1078 ms),内核数还从
145
+ # 10116 涨到 11472 —— 因为每轮都要 ``torch.cat`` 物化一个
146
+ # [B,H,nc,kd,vd] 张量,既贵又破坏 inductor 的融合。这里的闭式解不物化任何
147
+ # 额外的大张量(``T`` 只有 H·(nc+1)·nc 个元素),所以是另一回事。
148
+ # Collected in a list and stacked: writing into a torch.empty() buffer
149
+ # with slice assignment makes autograd emit CopySlices and leaves the
150
+ # untouched-in-backward buffer as uninitialised memory.
151
+ # 同样要夹下限:expo_t 在 (i=0, j=0) 处为 -1,clamp_min(0) 后是 0,
152
+ # 若 log_d 是 -inf 则 -inf*0 = NaN 会留在 T 的对角线上。
153
+ log_d = (torch.log(gamma.to(device=dev, dtype=torch.float32)
154
+ .clamp_min(torch.finfo(torch.float32).tiny))
155
+ * float(chunk_size))
156
+ idx = torch.arange(nc + 1, device=dev)
157
+ expo_t = idx[:, None] - torch.arange(nc, device=dev)[None, :] - 1
158
+ valid_t = expo_t >= 0
159
+ T = torch.exp(log_d[:, None, None] * expo_t.clamp_min(0).to(torch.float32)[None])
160
+ T = torch.where(valid_t[None], T, torch.zeros((), device=dev, dtype=torch.float32))
161
+ T = T.to(q.dtype) # [H,nc+1,nc]
162
+
163
+ U_flat = U.reshape(B, H, nc, kd * vd)
164
+ S_all = torch.matmul(T[None], U_flat) # [B,H,nc+1,kd*vd]
165
+ if has_state:
166
+ t_state = torch.exp(log_d[:, None] * idx.to(torch.float32)[None]).to(q.dtype)
167
+ S_all = S_all + t_state[None, :, :, None] * state.reshape(B, H, 1, kd * vd)
168
+
169
+ S_prev = S_all[:, :, :nc].reshape(B, H, nc, kd, vd)
170
+ new_state = S_all[:, :, nc].reshape(B, H, kd, vd)
171
+
172
+ # (3) inner chunk, one batched matmul
173
+ inner = torch.matmul(
174
+ torch.matmul(qr, kr.transpose(-1, -2)) * inner_mask[None, :, None, :, :], vr)
175
+
176
+ # (4) cross chunk, one batched matmul
177
+ cross = torch.matmul(qr * cross_decay[None, :, None, :, None], S_prev)
178
+
179
+ out = (inner + cross).reshape(B, H, L, vd)
180
+ return out, new_state
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<bos>",
3
+ "eos_token": "<eos>",
4
+ "pad_token": "<pad>",
5
+ "unk_token": "<unk>"
6
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<pad>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<bos>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ }
35
+ },
36
+ "bos_token": "<bos>",
37
+ "clean_up_tokenization_spaces": false,
38
+ "eos_token": "<eos>",
39
+ "extra_special_tokens": {},
40
+ "model_max_length": 2048,
41
+ "pad_token": "<pad>",
42
+ "tokenizer_class": "PreTrainedTokenizerFast",
43
+ "unk_token": "<unk>"
44
+ }