"""Dory1.0-4B forward pass in plain PyTorch, for interpretability work. Ported from the vLLM reference implementation (vllm/model_executor/models/dory.py, PR 60149) and the architecture section of arXiv 2611.03998. No KV cache: every call is one full-sequence forward pass, which is the per-loop cache semantics (loop i attends to loop-i keys and values of earlier positions). Naming. The checkpoint counts sublayers: 72 "layers", each either attention or MLP. input block sublayers 0..15 run once recurrent block sublayers 16..55 run n_loops times, weights shared output block sublayers 56..71 run once Residual nodes, in forward order (each is a unit-norm vector per position, except "emb"): ("emb",) raw token embedding ("in", i) after input sublayer i i in 0..15 ("x",) input block output (same tensor as ("in", 15)) ("u", t) loop t input, N(s_{t-1} + x) t in 1..n_loops ("rec", t, i) after recurrent sublayer i in loop t, before the loop-end norm i in 16..55 ("s", t) loop t output, N(F(u_t)) ("out", i) after output sublayer i i in 56..71 A `tap(name, h)` callback sees every node and may return a replacement tensor. """ from __future__ import annotations import json import math from pathlib import Path from typing import Callable, Optional import torch import torch.nn.functional as F from safetensors import safe_open from torch import nn EPS = 1e-6 Tap = Optional[Callable[[tuple, torch.Tensor], Optional[torch.Tensor]]] class _LRP: """Backward-pass rules of Layer-wise Relevance Propagation, as used by the R-lens and the J++ lens. They add stop-gradients only, so the forward values are unchanged. ln norm denominators are treated as constants (every L2 norm except the query and key norms) act SiLU(g) is treated as g * const, const = sigmoid(g) half the gradient through the gated MLP's product is split evenly between its two branches gate the attention output gate is treated as a constant (Dory-specific; the posts do not cover it) """ on = False ln = act = half = gate = True LRP = _LRP() class _Attn: """`block`: optional bool tensor [T]; key positions that no other position may attend to (a position always sees itself). Used by the chain roll-out to hide the anchor position until its state has been set.""" block = None ATTN = _Attn() class lrp_rules: """Context manager: `with lrp_rules(): ...` turns the LRP backward rules on.""" def __init__(self, **flags): self.flags = flags def __enter__(self): self.saved = {k: getattr(LRP, k) for k in ("on", "ln", "act", "half", "gate")} LRP.on = True for k, v in self.flags.items(): setattr(LRP, k, v) def __exit__(self, *a): for k, v in self.saved.items(): setattr(LRP, k, v) def justnorm(x: torch.Tensor, dim: int = -1, lrp: bool = True) -> torch.Tensor: """nGPT normalisation: x * rsqrt(sum(x^2) + eps), accumulated in fp32. Under the LRP rules the denominator is detached, unless `lrp=False` (query and key norms).""" xf = x.float() scale = torch.rsqrt(xf.pow(2).sum(dim=dim, keepdim=True) + EPS) if LRP.on and LRP.ln and lrp: scale = scale.detach() return (xf * scale).to(x.dtype) def slerp_norm(a: torch.Tensor, b: torch.Tensor, lr: torch.Tensor) -> torch.Tensor: ya, yb = justnorm(a).float(), justnorm(b).float() return justnorm(ya + lr.float() * (yb - ya)).to(a.dtype) def rope(x: torch.Tensor, positions: torch.Tensor, theta: float) -> torch.Tensor: """Rotary embedding, half-split ("neox") layout, on all of the last dim. x: [B, T, H, D].""" d = x.shape[-1] inv_freq = 1.0 / (theta ** (torch.arange(0, d, 2, device=x.device, dtype=torch.float32) / d)) ang = positions.float()[:, None] * inv_freq[None, :] # [T, D/2] cos, sin = ang.cos()[None, :, None, :], ang.sin()[None, :, None, :] xf = x.float() x1, x2 = xf[..., : d // 2], xf[..., d // 2 :] return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(x.dtype) class DoryAttention(nn.Module): def __init__(self, cfg: dict, layer_type: str): super().__init__() d, hd = cfg["hidden_size"], cfg["head_dim"] self.nq, self.nkv, self.hd = cfg["num_attention_heads"], cfg["num_key_value_heads"], hd std = cfg["initializer_range"] self.q_proj = nn.Linear(d, self.nq * hd, bias=False) self.k_proj = nn.Linear(d, self.nkv * hd, bias=False) self.v_proj = nn.Linear(d, self.nkv * hd, bias=False) self.g_proj = nn.Linear(d, self.nq * hd, bias=False) self.o_proj = nn.Linear(self.nq * hd, d, bias=False) self.sqk = nn.Parameter(torch.empty(self.nq * hd)) self.s_gate = nn.Parameter(torch.empty(self.nq * hd)) self.sqk_factor = hd**0.5 / std g0 = cfg.get("attention_output_gate_logit_scale_init") or d**0.5 self.s_gate_factor = g0**0.5 / std self.theta = cfg["rope_parameters"][layer_type]["rope_theta"] self.window = cfg["sliding_window"] if layer_type == "sliding_attention" else None def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: B, T, _ = h.shape q = self.q_proj(h).view(B, T, self.nq, self.hd) k = self.k_proj(h).view(B, T, self.nkv, self.hd) v = self.v_proj(h).view(B, T, self.nkv, self.hd) gate = self.g_proj(h) q, k = rope(q, positions, self.theta), rope(k, positions, self.theta) sqk = (self.sqk.float() * self.sqk_factor).view(1, 1, self.nq, self.hd) q = (sqk.square() * justnorm(q, lrp=False).float()).to(h.dtype) # qk-norm after RoPE k = justnorm(k, lrp=False) rep = self.nq // self.nkv q = q.transpose(1, 2) # [B, nq, T, hd] k = k.transpose(1, 2).repeat_interleave(rep, dim=1) v = v.transpose(1, 2).repeat_interleave(rep, dim=1) i = torch.arange(T, device=h.device) mask = i[None, :] <= i[:, None] # causal: key j <= query i if self.window is not None and T > self.window: mask = mask & (i[:, None] - i[None, :] < self.window) if ATTN.block is not None: mask = mask & (~ATTN.block.to(h.device)[None, :] | torch.eye(T, dtype=torch.bool, device=h.device)) out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, scale=1.0 / math.sqrt(self.hd)) out = out.transpose(1, 2).reshape(B, T, self.nq * self.hd) sg = (self.s_gate.float() * self.s_gate_factor).square() og = torch.sigmoid(gate.float() * sg) # output gate if LRP.on and LRP.gate: og = og.detach() out = (out.float() * og).to(h.dtype) return self.o_proj(justnorm(out)) class DoryMLP(nn.Module): def __init__(self, cfg: dict): super().__init__() d, m = cfg["hidden_size"], cfg["intermediate_size"] self.gate_proj = nn.Linear(d, m, bias=False) self.up_proj = nn.Linear(d, m, bias=False) self.down_proj = nn.Linear(m, d, bias=False) self.suv_gate = nn.Parameter(torch.empty(m)) self.suv_up = nn.Parameter(torch.empty(m)) self.suv_factor = d**0.5 def forward(self, h: torch.Tensor) -> torch.Tensor: g = self.gate_proj(h).float() * (self.suv_gate.float() * self.suv_factor) u = self.up_proj(h).float() * (self.suv_up.float() * self.suv_factor) if LRP.on and (LRP.act or LRP.half): a = g * torch.sigmoid(g).detach() if LRP.act else F.silu(g) # identity rule prod = 0.5 * (a * u.detach() + a.detach() * u) if LRP.half else a * u # half rule (same forward value) else: prod = F.silu(g) * u return self.down_proj(justnorm(prod.to(h.dtype))) class DoryLayer(nn.Module): """One sublayer (attention or MLP) and the nGPT residual blend.""" def __init__(self, cfg: dict, idx: int): super().__init__() lt = cfg["layer_types"][idx] self.is_attention = lt != "mlp" self.alpha_scale = cfg["ngpt_alpha_init"] / cfg["initializer_range"] self.mixer = DoryAttention(cfg, lt) if self.is_attention else DoryMLP(cfg) self.alpha = nn.Parameter(torch.empty(cfg["hidden_size"])) # attn_alpha or mlp_alpha in the file def rate(self) -> torch.Tensor: return (self.alpha.float() * self.alpha_scale).abs() def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: out = self.mixer(h, positions) if self.is_attention else self.mixer(h) return slerp_norm(h, out, self.rate()) class Dory(nn.Module): def __init__(self, cfg: dict): super().__init__() self.cfg = cfg self.n_in, self.n_rec, self.n_out = cfg["n_input_layers"], cfg["n_recurrent_layers"], cfg["n_output_layers"] assert self.n_in + self.n_rec + self.n_out == cfg["num_hidden_layers"] == len(cfg["layer_types"]) self.embeddings = nn.Embedding(cfg["vocab_size"], cfg["hidden_size"]) self.layers = nn.ModuleList(DoryLayer(cfg, i) for i in range(cfg["num_hidden_layers"])) self.lm_head = nn.Linear(cfg["hidden_size"], cfg["vocab_size"], bias=False) self.logit_scale = nn.Parameter(torch.empty(cfg["vocab_size"])) # ---- loading ------------------------------------------------------------------------------- @classmethod def from_snapshot(cls, path: str | Path, device="cuda", dtype=torch.float32) -> "Dory": path = Path(path) cfg = json.loads((path / "config.json").read_text()) model = cls(cfg) own = dict(model.named_parameters()) used = set() with safe_open(str(path / "model.safetensors"), framework="pt") as f: keys = list(f.keys()) for k in keys: name = (k.replace("backbone.", "", 1) .replace(".attn_alpha", ".alpha").replace(".mlp_alpha", ".alpha") .replace("lm_head.logit_scale", "logit_scale")) assert name in own, f"checkpoint tensor {k} has no home (mapped to {name})" t = f.get_tensor(k) assert tuple(t.shape) == tuple(own[name].shape), (k, t.shape, own[name].shape) own[name].data = t.to(dtype) used.add(name) missing = set(own) - used assert not missing, f"model parameters with no checkpoint tensor: {sorted(missing)[:5]}" model.n_checkpoint_tensors = len(keys) return model.to(device).eval().requires_grad_(False) # ---- forward ------------------------------------------------------------------------------- def unembed(self, h: torch.Tensor) -> torch.Tensor: return self.lm_head(h).float() * (self.logit_scale.float() / self.cfg["initializer_range"]) def run_output_block(self, h: torch.Tensor, positions: torch.Tensor, tap: Tap = None) -> torch.Tensor: for i in range(self.n_in + self.n_rec, self.n_in + self.n_rec + self.n_out): h = _tap(tap, ("out", i), self.layers[i](h, positions)) return h def forward(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None, inputs_embeds: Optional[torch.Tensor] = None) -> torch.Tensor: """Returns the final residual state [B, T, d]; call `unembed` for logits.""" positions = torch.arange(input_ids.shape[1], device=input_ids.device) h = self.embeddings(input_ids) if inputs_embeds is None else inputs_embeds h = _tap(tap, ("emb",), h) for i in range(self.n_in): h = _tap(tap, ("in", i), self.layers[i](h, positions)) x = _tap(tap, ("x",), h) state = x for t in range(1, n_loops + 1): h = _tap(tap, ("u", t), justnorm(state + x)) for i in range(self.n_in, self.n_in + self.n_rec): h = _tap(tap, ("rec", t, i), self.layers[i](h, positions)) state = _tap(tap, ("s", t), justnorm(h)) return self.run_output_block(state, positions, tap) def logits(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None) -> torch.Tensor: return self.unembed(self.forward(input_ids, n_loops=n_loops, tap=tap)) def _tap(tap: Tap, name: tuple, h: torch.Tensor) -> torch.Tensor: if tap is None: return h r = tap(name, h) return h if r is None else r def snapshot_path() -> Path: """The local snapshot folder in the shared Hugging Face cache (needs HF_HOME set).""" from huggingface_hub import snapshot_download return Path(snapshot_download("nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16", local_files_only=True, allow_patterns=["*.json", "*.jinja", "*.safetensors"])) def load_tokenizer(path: str | Path): """The tokenizer, built from the snapshot's files directly. AutoTokenizer cannot be used: transformers 5.17 has no `dory` config class and refuses the checkpoint's config.""" from transformers import PreTrainedTokenizerFast path = Path(path) tc = json.loads((path / "tokenizer_config.json").read_text()) special = {k: (v["content"] if isinstance(v, dict) else v) for k, v in tc.items() if k in ("bos_token", "eos_token", "unk_token", "pad_token") and v} tok = PreTrainedTokenizerFast(tokenizer_file=str(path / "tokenizer.json"), **special) tok.chat_template = (path / "chat_template.jinja").read_text() return tok