File size: 13,691 Bytes
498b9c9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
"""Dory1.0-4B forward pass in plain PyTorch, for interpretability work.

Ported from the vLLM reference implementation (vllm/model_executor/models/dory.py, PR 60149) and the
architecture section of arXiv 2611.03998. No KV cache: every call is one full-sequence forward pass, which
is the per-loop cache semantics (loop i attends to loop-i keys and values of earlier positions).

Naming. The checkpoint counts sublayers: 72 "layers", each either attention or MLP.
  input block      sublayers  0..15   run once
  recurrent block  sublayers 16..55   run n_loops times, weights shared
  output block     sublayers 56..71   run once
Residual nodes, in forward order (each is a unit-norm vector per position, except "emb"):
  ("emb",)                     raw token embedding
  ("in", i)                    after input sublayer i            i in 0..15
  ("x",)                       input block output (same tensor as ("in", 15))
  ("u", t)                     loop t input, N(s_{t-1} + x)      t in 1..n_loops
  ("rec", t, i)                after recurrent sublayer i in loop t, before the loop-end norm   i in 16..55
  ("s", t)                     loop t output, N(F(u_t))
  ("out", i)                   after output sublayer i           i in 56..71
A `tap(name, h)` callback sees every node and may return a replacement tensor.
"""
from __future__ import annotations

import json
import math
from pathlib import Path
from typing import Callable, Optional

import torch
import torch.nn.functional as F
from safetensors import safe_open
from torch import nn

EPS = 1e-6
Tap = Optional[Callable[[tuple, torch.Tensor], Optional[torch.Tensor]]]


class _LRP:
    """Backward-pass rules of Layer-wise Relevance Propagation, as used by the R-lens and the J++ lens.
    They add stop-gradients only, so the forward values are unchanged.
      ln    norm denominators are treated as constants (every L2 norm except the query and key norms)
      act   SiLU(g) is treated as g * const, const = sigmoid(g)
      half  the gradient through the gated MLP's product is split evenly between its two branches
      gate  the attention output gate is treated as a constant (Dory-specific; the posts do not cover it)
    """
    on = False
    ln = act = half = gate = True


LRP = _LRP()


class _Attn:
    """`block`: optional bool tensor [T]; key positions that no other position may attend to (a position always
    sees itself). Used by the chain roll-out to hide the anchor position until its state has been set."""
    block = None


ATTN = _Attn()


class lrp_rules:
    """Context manager: `with lrp_rules(): ...` turns the LRP backward rules on."""

    def __init__(self, **flags):
        self.flags = flags

    def __enter__(self):
        self.saved = {k: getattr(LRP, k) for k in ("on", "ln", "act", "half", "gate")}
        LRP.on = True
        for k, v in self.flags.items():
            setattr(LRP, k, v)

    def __exit__(self, *a):
        for k, v in self.saved.items():
            setattr(LRP, k, v)


def justnorm(x: torch.Tensor, dim: int = -1, lrp: bool = True) -> torch.Tensor:
    """nGPT normalisation: x * rsqrt(sum(x^2) + eps), accumulated in fp32.
    Under the LRP rules the denominator is detached, unless `lrp=False` (query and key norms)."""
    xf = x.float()
    scale = torch.rsqrt(xf.pow(2).sum(dim=dim, keepdim=True) + EPS)
    if LRP.on and LRP.ln and lrp:
        scale = scale.detach()
    return (xf * scale).to(x.dtype)


def slerp_norm(a: torch.Tensor, b: torch.Tensor, lr: torch.Tensor) -> torch.Tensor:
    ya, yb = justnorm(a).float(), justnorm(b).float()
    return justnorm(ya + lr.float() * (yb - ya)).to(a.dtype)


def rope(x: torch.Tensor, positions: torch.Tensor, theta: float) -> torch.Tensor:
    """Rotary embedding, half-split ("neox") layout, on all of the last dim. x: [B, T, H, D]."""
    d = x.shape[-1]
    inv_freq = 1.0 / (theta ** (torch.arange(0, d, 2, device=x.device, dtype=torch.float32) / d))
    ang = positions.float()[:, None] * inv_freq[None, :]            # [T, D/2]
    cos, sin = ang.cos()[None, :, None, :], ang.sin()[None, :, None, :]
    xf = x.float()
    x1, x2 = xf[..., : d // 2], xf[..., d // 2 :]
    return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(x.dtype)


class DoryAttention(nn.Module):
    def __init__(self, cfg: dict, layer_type: str):
        super().__init__()
        d, hd = cfg["hidden_size"], cfg["head_dim"]
        self.nq, self.nkv, self.hd = cfg["num_attention_heads"], cfg["num_key_value_heads"], hd
        std = cfg["initializer_range"]
        self.q_proj = nn.Linear(d, self.nq * hd, bias=False)
        self.k_proj = nn.Linear(d, self.nkv * hd, bias=False)
        self.v_proj = nn.Linear(d, self.nkv * hd, bias=False)
        self.g_proj = nn.Linear(d, self.nq * hd, bias=False)
        self.o_proj = nn.Linear(self.nq * hd, d, bias=False)
        self.sqk = nn.Parameter(torch.empty(self.nq * hd))
        self.s_gate = nn.Parameter(torch.empty(self.nq * hd))
        self.sqk_factor = hd**0.5 / std
        g0 = cfg.get("attention_output_gate_logit_scale_init") or d**0.5
        self.s_gate_factor = g0**0.5 / std
        self.theta = cfg["rope_parameters"][layer_type]["rope_theta"]
        self.window = cfg["sliding_window"] if layer_type == "sliding_attention" else None

    def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
        B, T, _ = h.shape
        q = self.q_proj(h).view(B, T, self.nq, self.hd)
        k = self.k_proj(h).view(B, T, self.nkv, self.hd)
        v = self.v_proj(h).view(B, T, self.nkv, self.hd)
        gate = self.g_proj(h)
        q, k = rope(q, positions, self.theta), rope(k, positions, self.theta)
        sqk = (self.sqk.float() * self.sqk_factor).view(1, 1, self.nq, self.hd)
        q = (sqk.square() * justnorm(q, lrp=False).float()).to(h.dtype)   # qk-norm after RoPE
        k = justnorm(k, lrp=False)
        rep = self.nq // self.nkv
        q = q.transpose(1, 2)                                          # [B, nq, T, hd]
        k = k.transpose(1, 2).repeat_interleave(rep, dim=1)
        v = v.transpose(1, 2).repeat_interleave(rep, dim=1)
        i = torch.arange(T, device=h.device)
        mask = i[None, :] <= i[:, None]                                # causal: key j <= query i
        if self.window is not None and T > self.window:
            mask = mask & (i[:, None] - i[None, :] < self.window)
        if ATTN.block is not None:
            mask = mask & (~ATTN.block.to(h.device)[None, :] | torch.eye(T, dtype=torch.bool, device=h.device))
        out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, scale=1.0 / math.sqrt(self.hd))
        out = out.transpose(1, 2).reshape(B, T, self.nq * self.hd)
        sg = (self.s_gate.float() * self.s_gate_factor).square()
        og = torch.sigmoid(gate.float() * sg)                          # output gate
        if LRP.on and LRP.gate:
            og = og.detach()
        out = (out.float() * og).to(h.dtype)
        return self.o_proj(justnorm(out))


class DoryMLP(nn.Module):
    def __init__(self, cfg: dict):
        super().__init__()
        d, m = cfg["hidden_size"], cfg["intermediate_size"]
        self.gate_proj = nn.Linear(d, m, bias=False)
        self.up_proj = nn.Linear(d, m, bias=False)
        self.down_proj = nn.Linear(m, d, bias=False)
        self.suv_gate = nn.Parameter(torch.empty(m))
        self.suv_up = nn.Parameter(torch.empty(m))
        self.suv_factor = d**0.5

    def forward(self, h: torch.Tensor) -> torch.Tensor:
        g = self.gate_proj(h).float() * (self.suv_gate.float() * self.suv_factor)
        u = self.up_proj(h).float() * (self.suv_up.float() * self.suv_factor)
        if LRP.on and (LRP.act or LRP.half):
            a = g * torch.sigmoid(g).detach() if LRP.act else F.silu(g)       # identity rule
            prod = 0.5 * (a * u.detach() + a.detach() * u) if LRP.half else a * u   # half rule (same forward value)
        else:
            prod = F.silu(g) * u
        return self.down_proj(justnorm(prod.to(h.dtype)))


class DoryLayer(nn.Module):
    """One sublayer (attention or MLP) and the nGPT residual blend."""

    def __init__(self, cfg: dict, idx: int):
        super().__init__()
        lt = cfg["layer_types"][idx]
        self.is_attention = lt != "mlp"
        self.alpha_scale = cfg["ngpt_alpha_init"] / cfg["initializer_range"]
        self.mixer = DoryAttention(cfg, lt) if self.is_attention else DoryMLP(cfg)
        self.alpha = nn.Parameter(torch.empty(cfg["hidden_size"]))   # attn_alpha or mlp_alpha in the file

    def rate(self) -> torch.Tensor:
        return (self.alpha.float() * self.alpha_scale).abs()

    def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
        out = self.mixer(h, positions) if self.is_attention else self.mixer(h)
        return slerp_norm(h, out, self.rate())


class Dory(nn.Module):
    def __init__(self, cfg: dict):
        super().__init__()
        self.cfg = cfg
        self.n_in, self.n_rec, self.n_out = cfg["n_input_layers"], cfg["n_recurrent_layers"], cfg["n_output_layers"]
        assert self.n_in + self.n_rec + self.n_out == cfg["num_hidden_layers"] == len(cfg["layer_types"])
        self.embeddings = nn.Embedding(cfg["vocab_size"], cfg["hidden_size"])
        self.layers = nn.ModuleList(DoryLayer(cfg, i) for i in range(cfg["num_hidden_layers"]))
        self.lm_head = nn.Linear(cfg["hidden_size"], cfg["vocab_size"], bias=False)
        self.logit_scale = nn.Parameter(torch.empty(cfg["vocab_size"]))

    # ---- loading -------------------------------------------------------------------------------
    @classmethod
    def from_snapshot(cls, path: str | Path, device="cuda", dtype=torch.float32) -> "Dory":
        path = Path(path)
        cfg = json.loads((path / "config.json").read_text())
        model = cls(cfg)
        own = dict(model.named_parameters())
        used = set()
        with safe_open(str(path / "model.safetensors"), framework="pt") as f:
            keys = list(f.keys())
            for k in keys:
                name = (k.replace("backbone.", "", 1)
                         .replace(".attn_alpha", ".alpha").replace(".mlp_alpha", ".alpha")
                         .replace("lm_head.logit_scale", "logit_scale"))
                assert name in own, f"checkpoint tensor {k} has no home (mapped to {name})"
                t = f.get_tensor(k)
                assert tuple(t.shape) == tuple(own[name].shape), (k, t.shape, own[name].shape)
                own[name].data = t.to(dtype)
                used.add(name)
        missing = set(own) - used
        assert not missing, f"model parameters with no checkpoint tensor: {sorted(missing)[:5]}"
        model.n_checkpoint_tensors = len(keys)
        return model.to(device).eval().requires_grad_(False)

    # ---- forward -------------------------------------------------------------------------------
    def unembed(self, h: torch.Tensor) -> torch.Tensor:
        return self.lm_head(h).float() * (self.logit_scale.float() / self.cfg["initializer_range"])

    def run_output_block(self, h: torch.Tensor, positions: torch.Tensor, tap: Tap = None) -> torch.Tensor:
        for i in range(self.n_in + self.n_rec, self.n_in + self.n_rec + self.n_out):
            h = _tap(tap, ("out", i), self.layers[i](h, positions))
        return h

    def forward(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None,
                inputs_embeds: Optional[torch.Tensor] = None) -> torch.Tensor:
        """Returns the final residual state [B, T, d]; call `unembed` for logits."""
        positions = torch.arange(input_ids.shape[1], device=input_ids.device)
        h = self.embeddings(input_ids) if inputs_embeds is None else inputs_embeds
        h = _tap(tap, ("emb",), h)
        for i in range(self.n_in):
            h = _tap(tap, ("in", i), self.layers[i](h, positions))
        x = _tap(tap, ("x",), h)
        state = x
        for t in range(1, n_loops + 1):
            h = _tap(tap, ("u", t), justnorm(state + x))
            for i in range(self.n_in, self.n_in + self.n_rec):
                h = _tap(tap, ("rec", t, i), self.layers[i](h, positions))
            state = _tap(tap, ("s", t), justnorm(h))
        return self.run_output_block(state, positions, tap)

    def logits(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None) -> torch.Tensor:
        return self.unembed(self.forward(input_ids, n_loops=n_loops, tap=tap))


def _tap(tap: Tap, name: tuple, h: torch.Tensor) -> torch.Tensor:
    if tap is None:
        return h
    r = tap(name, h)
    return h if r is None else r


def snapshot_path() -> Path:
    """The local snapshot folder in the shared Hugging Face cache (needs HF_HOME set)."""
    from huggingface_hub import snapshot_download
    return Path(snapshot_download("nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16", local_files_only=True,
                                  allow_patterns=["*.json", "*.jinja", "*.safetensors"]))


def load_tokenizer(path: str | Path):
    """The tokenizer, built from the snapshot's files directly. AutoTokenizer cannot be used: transformers
    5.17 has no `dory` config class and refuses the checkpoint's config."""
    from transformers import PreTrainedTokenizerFast
    path = Path(path)
    tc = json.loads((path / "tokenizer_config.json").read_text())
    special = {k: (v["content"] if isinstance(v, dict) else v)
               for k, v in tc.items() if k in ("bos_token", "eos_token", "unk_token", "pad_token") and v}
    tok = PreTrainedTokenizerFast(tokenizer_file=str(path / "tokenizer.json"), **special)
    tok.chat_template = (path / "chat_template.jinja").read_text()
    return tok