senku21x's picture
Dory forward pass, J-lens / R-lens / J++ code, fit records
498b9c9 verified
Raw History Blame Contribute Delete
13.7 kB
"""Dory1.0-4B forward pass in plain PyTorch, for interpretability work.
Ported from the vLLM reference implementation (vllm/model_executor/models/dory.py, PR 60149) and the
architecture section of arXiv 2611.03998. No KV cache: every call is one full-sequence forward pass, which
is the per-loop cache semantics (loop i attends to loop-i keys and values of earlier positions).
Naming. The checkpoint counts sublayers: 72 "layers", each either attention or MLP.
input block sublayers 0..15 run once
recurrent block sublayers 16..55 run n_loops times, weights shared
output block sublayers 56..71 run once
Residual nodes, in forward order (each is a unit-norm vector per position, except "emb"):
("emb",) raw token embedding
("in", i) after input sublayer i i in 0..15
("x",) input block output (same tensor as ("in", 15))
("u", t) loop t input, N(s_{t-1} + x) t in 1..n_loops
("rec", t, i) after recurrent sublayer i in loop t, before the loop-end norm i in 16..55
("s", t) loop t output, N(F(u_t))
("out", i) after output sublayer i i in 56..71
A `tap(name, h)` callback sees every node and may return a replacement tensor.
"""
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import Callable, Optional
import torch
import torch.nn.functional as F
from safetensors import safe_open
from torch import nn
EPS = 1e-6
Tap = Optional[Callable[[tuple, torch.Tensor], Optional[torch.Tensor]]]
class _LRP:
"""Backward-pass rules of Layer-wise Relevance Propagation, as used by the R-lens and the J++ lens.
They add stop-gradients only, so the forward values are unchanged.
ln norm denominators are treated as constants (every L2 norm except the query and key norms)
act SiLU(g) is treated as g * const, const = sigmoid(g)
half the gradient through the gated MLP's product is split evenly between its two branches
gate the attention output gate is treated as a constant (Dory-specific; the posts do not cover it)
"""
on = False
ln = act = half = gate = True
LRP = _LRP()
class _Attn:
"""`block`: optional bool tensor [T]; key positions that no other position may attend to (a position always
sees itself). Used by the chain roll-out to hide the anchor position until its state has been set."""
block = None
ATTN = _Attn()
class lrp_rules:
"""Context manager: `with lrp_rules(): ...` turns the LRP backward rules on."""
def __init__(self, **flags):
self.flags = flags
def __enter__(self):
self.saved = {k: getattr(LRP, k) for k in ("on", "ln", "act", "half", "gate")}
LRP.on = True
for k, v in self.flags.items():
setattr(LRP, k, v)
def __exit__(self, *a):
for k, v in self.saved.items():
setattr(LRP, k, v)
def justnorm(x: torch.Tensor, dim: int = -1, lrp: bool = True) -> torch.Tensor:
"""nGPT normalisation: x * rsqrt(sum(x^2) + eps), accumulated in fp32.
Under the LRP rules the denominator is detached, unless `lrp=False` (query and key norms)."""
xf = x.float()
scale = torch.rsqrt(xf.pow(2).sum(dim=dim, keepdim=True) + EPS)
if LRP.on and LRP.ln and lrp:
scale = scale.detach()
return (xf * scale).to(x.dtype)
def slerp_norm(a: torch.Tensor, b: torch.Tensor, lr: torch.Tensor) -> torch.Tensor:
ya, yb = justnorm(a).float(), justnorm(b).float()
return justnorm(ya + lr.float() * (yb - ya)).to(a.dtype)
def rope(x: torch.Tensor, positions: torch.Tensor, theta: float) -> torch.Tensor:
"""Rotary embedding, half-split ("neox") layout, on all of the last dim. x: [B, T, H, D]."""
d = x.shape[-1]
inv_freq = 1.0 / (theta ** (torch.arange(0, d, 2, device=x.device, dtype=torch.float32) / d))
ang = positions.float()[:, None] * inv_freq[None, :] # [T, D/2]
cos, sin = ang.cos()[None, :, None, :], ang.sin()[None, :, None, :]
xf = x.float()
x1, x2 = xf[..., : d // 2], xf[..., d // 2 :]
return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(x.dtype)
class DoryAttention(nn.Module):
def __init__(self, cfg: dict, layer_type: str):
super().__init__()
d, hd = cfg["hidden_size"], cfg["head_dim"]
self.nq, self.nkv, self.hd = cfg["num_attention_heads"], cfg["num_key_value_heads"], hd
std = cfg["initializer_range"]
self.q_proj = nn.Linear(d, self.nq * hd, bias=False)
self.k_proj = nn.Linear(d, self.nkv * hd, bias=False)
self.v_proj = nn.Linear(d, self.nkv * hd, bias=False)
self.g_proj = nn.Linear(d, self.nq * hd, bias=False)
self.o_proj = nn.Linear(self.nq * hd, d, bias=False)
self.sqk = nn.Parameter(torch.empty(self.nq * hd))
self.s_gate = nn.Parameter(torch.empty(self.nq * hd))
self.sqk_factor = hd**0.5 / std
g0 = cfg.get("attention_output_gate_logit_scale_init") or d**0.5
self.s_gate_factor = g0**0.5 / std
self.theta = cfg["rope_parameters"][layer_type]["rope_theta"]
self.window = cfg["sliding_window"] if layer_type == "sliding_attention" else None
def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
B, T, _ = h.shape
q = self.q_proj(h).view(B, T, self.nq, self.hd)
k = self.k_proj(h).view(B, T, self.nkv, self.hd)
v = self.v_proj(h).view(B, T, self.nkv, self.hd)
gate = self.g_proj(h)
q, k = rope(q, positions, self.theta), rope(k, positions, self.theta)
sqk = (self.sqk.float() * self.sqk_factor).view(1, 1, self.nq, self.hd)
q = (sqk.square() * justnorm(q, lrp=False).float()).to(h.dtype) # qk-norm after RoPE
k = justnorm(k, lrp=False)
rep = self.nq // self.nkv
q = q.transpose(1, 2) # [B, nq, T, hd]
k = k.transpose(1, 2).repeat_interleave(rep, dim=1)
v = v.transpose(1, 2).repeat_interleave(rep, dim=1)
i = torch.arange(T, device=h.device)
mask = i[None, :] <= i[:, None] # causal: key j <= query i
if self.window is not None and T > self.window:
mask = mask & (i[:, None] - i[None, :] < self.window)
if ATTN.block is not None:
mask = mask & (~ATTN.block.to(h.device)[None, :] | torch.eye(T, dtype=torch.bool, device=h.device))
out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, scale=1.0 / math.sqrt(self.hd))
out = out.transpose(1, 2).reshape(B, T, self.nq * self.hd)
sg = (self.s_gate.float() * self.s_gate_factor).square()
og = torch.sigmoid(gate.float() * sg) # output gate
if LRP.on and LRP.gate:
og = og.detach()
out = (out.float() * og).to(h.dtype)
return self.o_proj(justnorm(out))
class DoryMLP(nn.Module):
def __init__(self, cfg: dict):
super().__init__()
d, m = cfg["hidden_size"], cfg["intermediate_size"]
self.gate_proj = nn.Linear(d, m, bias=False)
self.up_proj = nn.Linear(d, m, bias=False)
self.down_proj = nn.Linear(m, d, bias=False)
self.suv_gate = nn.Parameter(torch.empty(m))
self.suv_up = nn.Parameter(torch.empty(m))
self.suv_factor = d**0.5
def forward(self, h: torch.Tensor) -> torch.Tensor:
g = self.gate_proj(h).float() * (self.suv_gate.float() * self.suv_factor)
u = self.up_proj(h).float() * (self.suv_up.float() * self.suv_factor)
if LRP.on and (LRP.act or LRP.half):
a = g * torch.sigmoid(g).detach() if LRP.act else F.silu(g) # identity rule
prod = 0.5 * (a * u.detach() + a.detach() * u) if LRP.half else a * u # half rule (same forward value)
else:
prod = F.silu(g) * u
return self.down_proj(justnorm(prod.to(h.dtype)))
class DoryLayer(nn.Module):
"""One sublayer (attention or MLP) and the nGPT residual blend."""
def __init__(self, cfg: dict, idx: int):
super().__init__()
lt = cfg["layer_types"][idx]
self.is_attention = lt != "mlp"
self.alpha_scale = cfg["ngpt_alpha_init"] / cfg["initializer_range"]
self.mixer = DoryAttention(cfg, lt) if self.is_attention else DoryMLP(cfg)
self.alpha = nn.Parameter(torch.empty(cfg["hidden_size"])) # attn_alpha or mlp_alpha in the file
def rate(self) -> torch.Tensor:
return (self.alpha.float() * self.alpha_scale).abs()
def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
out = self.mixer(h, positions) if self.is_attention else self.mixer(h)
return slerp_norm(h, out, self.rate())
class Dory(nn.Module):
def __init__(self, cfg: dict):
super().__init__()
self.cfg = cfg
self.n_in, self.n_rec, self.n_out = cfg["n_input_layers"], cfg["n_recurrent_layers"], cfg["n_output_layers"]
assert self.n_in + self.n_rec + self.n_out == cfg["num_hidden_layers"] == len(cfg["layer_types"])
self.embeddings = nn.Embedding(cfg["vocab_size"], cfg["hidden_size"])
self.layers = nn.ModuleList(DoryLayer(cfg, i) for i in range(cfg["num_hidden_layers"]))
self.lm_head = nn.Linear(cfg["hidden_size"], cfg["vocab_size"], bias=False)
self.logit_scale = nn.Parameter(torch.empty(cfg["vocab_size"]))
# ---- loading -------------------------------------------------------------------------------
@classmethod
def from_snapshot(cls, path: str | Path, device="cuda", dtype=torch.float32) -> "Dory":
path = Path(path)
cfg = json.loads((path / "config.json").read_text())
model = cls(cfg)
own = dict(model.named_parameters())
used = set()
with safe_open(str(path / "model.safetensors"), framework="pt") as f:
keys = list(f.keys())
for k in keys:
name = (k.replace("backbone.", "", 1)
.replace(".attn_alpha", ".alpha").replace(".mlp_alpha", ".alpha")
.replace("lm_head.logit_scale", "logit_scale"))
assert name in own, f"checkpoint tensor {k} has no home (mapped to {name})"
t = f.get_tensor(k)
assert tuple(t.shape) == tuple(own[name].shape), (k, t.shape, own[name].shape)
own[name].data = t.to(dtype)
used.add(name)
missing = set(own) - used
assert not missing, f"model parameters with no checkpoint tensor: {sorted(missing)[:5]}"
model.n_checkpoint_tensors = len(keys)
return model.to(device).eval().requires_grad_(False)
# ---- forward -------------------------------------------------------------------------------
def unembed(self, h: torch.Tensor) -> torch.Tensor:
return self.lm_head(h).float() * (self.logit_scale.float() / self.cfg["initializer_range"])
def run_output_block(self, h: torch.Tensor, positions: torch.Tensor, tap: Tap = None) -> torch.Tensor:
for i in range(self.n_in + self.n_rec, self.n_in + self.n_rec + self.n_out):
h = _tap(tap, ("out", i), self.layers[i](h, positions))
return h
def forward(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None,
inputs_embeds: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Returns the final residual state [B, T, d]; call `unembed` for logits."""
positions = torch.arange(input_ids.shape[1], device=input_ids.device)
h = self.embeddings(input_ids) if inputs_embeds is None else inputs_embeds
h = _tap(tap, ("emb",), h)
for i in range(self.n_in):
h = _tap(tap, ("in", i), self.layers[i](h, positions))
x = _tap(tap, ("x",), h)
state = x
for t in range(1, n_loops + 1):
h = _tap(tap, ("u", t), justnorm(state + x))
for i in range(self.n_in, self.n_in + self.n_rec):
h = _tap(tap, ("rec", t, i), self.layers[i](h, positions))
state = _tap(tap, ("s", t), justnorm(h))
return self.run_output_block(state, positions, tap)
def logits(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None) -> torch.Tensor:
return self.unembed(self.forward(input_ids, n_loops=n_loops, tap=tap))
def _tap(tap: Tap, name: tuple, h: torch.Tensor) -> torch.Tensor:
if tap is None:
return h
r = tap(name, h)
return h if r is None else r
def snapshot_path() -> Path:
"""The local snapshot folder in the shared Hugging Face cache (needs HF_HOME set)."""
from huggingface_hub import snapshot_download
return Path(snapshot_download("nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16", local_files_only=True,
allow_patterns=["*.json", "*.jinja", "*.safetensors"]))
def load_tokenizer(path: str | Path):
"""The tokenizer, built from the snapshot's files directly. AutoTokenizer cannot be used: transformers
5.17 has no `dory` config class and refuses the checkpoint's config."""
from transformers import PreTrainedTokenizerFast
path = Path(path)
tc = json.loads((path / "tokenizer_config.json").read_text())
special = {k: (v["content"] if isinstance(v, dict) else v)
for k, v in tc.items() if k in ("bos_token", "eos_token", "unk_token", "pad_token") and v}
tok = PreTrainedTokenizerFast(tokenizer_file=str(path / "tokenizer.json"), **special)
tok.chat_template = (path / "chat_template.jinja").read_text()
return tok