Download dory/model.py from senku21x/dory_implementation: direct link, hf CLI and curl.
- Browser
- Download file 13.7 kB
-
https://huggingface.co/senku21x/dory_implementation/resolve/main/dory/model.py
- Command line
-
hf download hf://senku21x/dory_implementation/dory/model.py
-
curl -L -o model.py https://huggingface.co/senku21x/dory_implementation/resolve/main/dory/model.py
13.7 kB
| """Dory1.0-4B forward pass in plain PyTorch, for interpretability work. | |
| Ported from the vLLM reference implementation (vllm/model_executor/models/dory.py, PR 60149) and the | |
| architecture section of arXiv 2611.03998. No KV cache: every call is one full-sequence forward pass, which | |
| is the per-loop cache semantics (loop i attends to loop-i keys and values of earlier positions). | |
| Naming. The checkpoint counts sublayers: 72 "layers", each either attention or MLP. | |
| input block sublayers 0..15 run once | |
| recurrent block sublayers 16..55 run n_loops times, weights shared | |
| output block sublayers 56..71 run once | |
| Residual nodes, in forward order (each is a unit-norm vector per position, except "emb"): | |
| ("emb",) raw token embedding | |
| ("in", i) after input sublayer i i in 0..15 | |
| ("x",) input block output (same tensor as ("in", 15)) | |
| ("u", t) loop t input, N(s_{t-1} + x) t in 1..n_loops | |
| ("rec", t, i) after recurrent sublayer i in loop t, before the loop-end norm i in 16..55 | |
| ("s", t) loop t output, N(F(u_t)) | |
| ("out", i) after output sublayer i i in 56..71 | |
| A `tap(name, h)` callback sees every node and may return a replacement tensor. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import math | |
| from pathlib import Path | |
| from typing import Callable, Optional | |
| import torch | |
| import torch.nn.functional as F | |
| from safetensors import safe_open | |
| from torch import nn | |
| EPS = 1e-6 | |
| Tap = Optional[Callable[[tuple, torch.Tensor], Optional[torch.Tensor]]] | |
| class _LRP: | |
| """Backward-pass rules of Layer-wise Relevance Propagation, as used by the R-lens and the J++ lens. | |
| They add stop-gradients only, so the forward values are unchanged. | |
| ln norm denominators are treated as constants (every L2 norm except the query and key norms) | |
| act SiLU(g) is treated as g * const, const = sigmoid(g) | |
| half the gradient through the gated MLP's product is split evenly between its two branches | |
| gate the attention output gate is treated as a constant (Dory-specific; the posts do not cover it) | |
| """ | |
| on = False | |
| ln = act = half = gate = True | |
| LRP = _LRP() | |
| class _Attn: | |
| """`block`: optional bool tensor [T]; key positions that no other position may attend to (a position always | |
| sees itself). Used by the chain roll-out to hide the anchor position until its state has been set.""" | |
| block = None | |
| ATTN = _Attn() | |
| class lrp_rules: | |
| """Context manager: `with lrp_rules(): ...` turns the LRP backward rules on.""" | |
| def __init__(self, **flags): | |
| self.flags = flags | |
| def __enter__(self): | |
| self.saved = {k: getattr(LRP, k) for k in ("on", "ln", "act", "half", "gate")} | |
| LRP.on = True | |
| for k, v in self.flags.items(): | |
| setattr(LRP, k, v) | |
| def __exit__(self, *a): | |
| for k, v in self.saved.items(): | |
| setattr(LRP, k, v) | |
| def justnorm(x: torch.Tensor, dim: int = -1, lrp: bool = True) -> torch.Tensor: | |
| """nGPT normalisation: x * rsqrt(sum(x^2) + eps), accumulated in fp32. | |
| Under the LRP rules the denominator is detached, unless `lrp=False` (query and key norms).""" | |
| xf = x.float() | |
| scale = torch.rsqrt(xf.pow(2).sum(dim=dim, keepdim=True) + EPS) | |
| if LRP.on and LRP.ln and lrp: | |
| scale = scale.detach() | |
| return (xf * scale).to(x.dtype) | |
| def slerp_norm(a: torch.Tensor, b: torch.Tensor, lr: torch.Tensor) -> torch.Tensor: | |
| ya, yb = justnorm(a).float(), justnorm(b).float() | |
| return justnorm(ya + lr.float() * (yb - ya)).to(a.dtype) | |
| def rope(x: torch.Tensor, positions: torch.Tensor, theta: float) -> torch.Tensor: | |
| """Rotary embedding, half-split ("neox") layout, on all of the last dim. x: [B, T, H, D].""" | |
| d = x.shape[-1] | |
| inv_freq = 1.0 / (theta ** (torch.arange(0, d, 2, device=x.device, dtype=torch.float32) / d)) | |
| ang = positions.float()[:, None] * inv_freq[None, :] # [T, D/2] | |
| cos, sin = ang.cos()[None, :, None, :], ang.sin()[None, :, None, :] | |
| xf = x.float() | |
| x1, x2 = xf[..., : d // 2], xf[..., d // 2 :] | |
| return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(x.dtype) | |
| class DoryAttention(nn.Module): | |
| def __init__(self, cfg: dict, layer_type: str): | |
| super().__init__() | |
| d, hd = cfg["hidden_size"], cfg["head_dim"] | |
| self.nq, self.nkv, self.hd = cfg["num_attention_heads"], cfg["num_key_value_heads"], hd | |
| std = cfg["initializer_range"] | |
| self.q_proj = nn.Linear(d, self.nq * hd, bias=False) | |
| self.k_proj = nn.Linear(d, self.nkv * hd, bias=False) | |
| self.v_proj = nn.Linear(d, self.nkv * hd, bias=False) | |
| self.g_proj = nn.Linear(d, self.nq * hd, bias=False) | |
| self.o_proj = nn.Linear(self.nq * hd, d, bias=False) | |
| self.sqk = nn.Parameter(torch.empty(self.nq * hd)) | |
| self.s_gate = nn.Parameter(torch.empty(self.nq * hd)) | |
| self.sqk_factor = hd**0.5 / std | |
| g0 = cfg.get("attention_output_gate_logit_scale_init") or d**0.5 | |
| self.s_gate_factor = g0**0.5 / std | |
| self.theta = cfg["rope_parameters"][layer_type]["rope_theta"] | |
| self.window = cfg["sliding_window"] if layer_type == "sliding_attention" else None | |
| def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: | |
| B, T, _ = h.shape | |
| q = self.q_proj(h).view(B, T, self.nq, self.hd) | |
| k = self.k_proj(h).view(B, T, self.nkv, self.hd) | |
| v = self.v_proj(h).view(B, T, self.nkv, self.hd) | |
| gate = self.g_proj(h) | |
| q, k = rope(q, positions, self.theta), rope(k, positions, self.theta) | |
| sqk = (self.sqk.float() * self.sqk_factor).view(1, 1, self.nq, self.hd) | |
| q = (sqk.square() * justnorm(q, lrp=False).float()).to(h.dtype) # qk-norm after RoPE | |
| k = justnorm(k, lrp=False) | |
| rep = self.nq // self.nkv | |
| q = q.transpose(1, 2) # [B, nq, T, hd] | |
| k = k.transpose(1, 2).repeat_interleave(rep, dim=1) | |
| v = v.transpose(1, 2).repeat_interleave(rep, dim=1) | |
| i = torch.arange(T, device=h.device) | |
| mask = i[None, :] <= i[:, None] # causal: key j <= query i | |
| if self.window is not None and T > self.window: | |
| mask = mask & (i[:, None] - i[None, :] < self.window) | |
| if ATTN.block is not None: | |
| mask = mask & (~ATTN.block.to(h.device)[None, :] | torch.eye(T, dtype=torch.bool, device=h.device)) | |
| out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, scale=1.0 / math.sqrt(self.hd)) | |
| out = out.transpose(1, 2).reshape(B, T, self.nq * self.hd) | |
| sg = (self.s_gate.float() * self.s_gate_factor).square() | |
| og = torch.sigmoid(gate.float() * sg) # output gate | |
| if LRP.on and LRP.gate: | |
| og = og.detach() | |
| out = (out.float() * og).to(h.dtype) | |
| return self.o_proj(justnorm(out)) | |
| class DoryMLP(nn.Module): | |
| def __init__(self, cfg: dict): | |
| super().__init__() | |
| d, m = cfg["hidden_size"], cfg["intermediate_size"] | |
| self.gate_proj = nn.Linear(d, m, bias=False) | |
| self.up_proj = nn.Linear(d, m, bias=False) | |
| self.down_proj = nn.Linear(m, d, bias=False) | |
| self.suv_gate = nn.Parameter(torch.empty(m)) | |
| self.suv_up = nn.Parameter(torch.empty(m)) | |
| self.suv_factor = d**0.5 | |
| def forward(self, h: torch.Tensor) -> torch.Tensor: | |
| g = self.gate_proj(h).float() * (self.suv_gate.float() * self.suv_factor) | |
| u = self.up_proj(h).float() * (self.suv_up.float() * self.suv_factor) | |
| if LRP.on and (LRP.act or LRP.half): | |
| a = g * torch.sigmoid(g).detach() if LRP.act else F.silu(g) # identity rule | |
| prod = 0.5 * (a * u.detach() + a.detach() * u) if LRP.half else a * u # half rule (same forward value) | |
| else: | |
| prod = F.silu(g) * u | |
| return self.down_proj(justnorm(prod.to(h.dtype))) | |
| class DoryLayer(nn.Module): | |
| """One sublayer (attention or MLP) and the nGPT residual blend.""" | |
| def __init__(self, cfg: dict, idx: int): | |
| super().__init__() | |
| lt = cfg["layer_types"][idx] | |
| self.is_attention = lt != "mlp" | |
| self.alpha_scale = cfg["ngpt_alpha_init"] / cfg["initializer_range"] | |
| self.mixer = DoryAttention(cfg, lt) if self.is_attention else DoryMLP(cfg) | |
| self.alpha = nn.Parameter(torch.empty(cfg["hidden_size"])) # attn_alpha or mlp_alpha in the file | |
| def rate(self) -> torch.Tensor: | |
| return (self.alpha.float() * self.alpha_scale).abs() | |
| def forward(self, h: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: | |
| out = self.mixer(h, positions) if self.is_attention else self.mixer(h) | |
| return slerp_norm(h, out, self.rate()) | |
| class Dory(nn.Module): | |
| def __init__(self, cfg: dict): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.n_in, self.n_rec, self.n_out = cfg["n_input_layers"], cfg["n_recurrent_layers"], cfg["n_output_layers"] | |
| assert self.n_in + self.n_rec + self.n_out == cfg["num_hidden_layers"] == len(cfg["layer_types"]) | |
| self.embeddings = nn.Embedding(cfg["vocab_size"], cfg["hidden_size"]) | |
| self.layers = nn.ModuleList(DoryLayer(cfg, i) for i in range(cfg["num_hidden_layers"])) | |
| self.lm_head = nn.Linear(cfg["hidden_size"], cfg["vocab_size"], bias=False) | |
| self.logit_scale = nn.Parameter(torch.empty(cfg["vocab_size"])) | |
| # ---- loading ------------------------------------------------------------------------------- | |
| def from_snapshot(cls, path: str | Path, device="cuda", dtype=torch.float32) -> "Dory": | |
| path = Path(path) | |
| cfg = json.loads((path / "config.json").read_text()) | |
| model = cls(cfg) | |
| own = dict(model.named_parameters()) | |
| used = set() | |
| with safe_open(str(path / "model.safetensors"), framework="pt") as f: | |
| keys = list(f.keys()) | |
| for k in keys: | |
| name = (k.replace("backbone.", "", 1) | |
| .replace(".attn_alpha", ".alpha").replace(".mlp_alpha", ".alpha") | |
| .replace("lm_head.logit_scale", "logit_scale")) | |
| assert name in own, f"checkpoint tensor {k} has no home (mapped to {name})" | |
| t = f.get_tensor(k) | |
| assert tuple(t.shape) == tuple(own[name].shape), (k, t.shape, own[name].shape) | |
| own[name].data = t.to(dtype) | |
| used.add(name) | |
| missing = set(own) - used | |
| assert not missing, f"model parameters with no checkpoint tensor: {sorted(missing)[:5]}" | |
| model.n_checkpoint_tensors = len(keys) | |
| return model.to(device).eval().requires_grad_(False) | |
| # ---- forward ------------------------------------------------------------------------------- | |
| def unembed(self, h: torch.Tensor) -> torch.Tensor: | |
| return self.lm_head(h).float() * (self.logit_scale.float() / self.cfg["initializer_range"]) | |
| def run_output_block(self, h: torch.Tensor, positions: torch.Tensor, tap: Tap = None) -> torch.Tensor: | |
| for i in range(self.n_in + self.n_rec, self.n_in + self.n_rec + self.n_out): | |
| h = _tap(tap, ("out", i), self.layers[i](h, positions)) | |
| return h | |
| def forward(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None, | |
| inputs_embeds: Optional[torch.Tensor] = None) -> torch.Tensor: | |
| """Returns the final residual state [B, T, d]; call `unembed` for logits.""" | |
| positions = torch.arange(input_ids.shape[1], device=input_ids.device) | |
| h = self.embeddings(input_ids) if inputs_embeds is None else inputs_embeds | |
| h = _tap(tap, ("emb",), h) | |
| for i in range(self.n_in): | |
| h = _tap(tap, ("in", i), self.layers[i](h, positions)) | |
| x = _tap(tap, ("x",), h) | |
| state = x | |
| for t in range(1, n_loops + 1): | |
| h = _tap(tap, ("u", t), justnorm(state + x)) | |
| for i in range(self.n_in, self.n_in + self.n_rec): | |
| h = _tap(tap, ("rec", t, i), self.layers[i](h, positions)) | |
| state = _tap(tap, ("s", t), justnorm(h)) | |
| return self.run_output_block(state, positions, tap) | |
| def logits(self, input_ids: torch.Tensor, n_loops: int = 2, tap: Tap = None) -> torch.Tensor: | |
| return self.unembed(self.forward(input_ids, n_loops=n_loops, tap=tap)) | |
| def _tap(tap: Tap, name: tuple, h: torch.Tensor) -> torch.Tensor: | |
| if tap is None: | |
| return h | |
| r = tap(name, h) | |
| return h if r is None else r | |
| def snapshot_path() -> Path: | |
| """The local snapshot folder in the shared Hugging Face cache (needs HF_HOME set).""" | |
| from huggingface_hub import snapshot_download | |
| return Path(snapshot_download("nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16", local_files_only=True, | |
| allow_patterns=["*.json", "*.jinja", "*.safetensors"])) | |
| def load_tokenizer(path: str | Path): | |
| """The tokenizer, built from the snapshot's files directly. AutoTokenizer cannot be used: transformers | |
| 5.17 has no `dory` config class and refuses the checkpoint's config.""" | |
| from transformers import PreTrainedTokenizerFast | |
| path = Path(path) | |
| tc = json.loads((path / "tokenizer_config.json").read_text()) | |
| special = {k: (v["content"] if isinstance(v, dict) else v) | |
| for k, v in tc.items() if k in ("bos_token", "eos_token", "unk_token", "pad_token") and v} | |
| tok = PreTrainedTokenizerFast(tokenizer_file=str(path / "tokenizer.json"), **special) | |
| tok.chat_template = (path / "chat_template.jinja").read_text() | |
| return tok | |