d1-2B-gestures-CoreAI / code /d1_torch.py
ivanfioravanti's picture
Prompt as inputs (no baked questions), split Neural Engine model for iOS, card and code
7011ac6 verified
Raw History Blame Contribute Delete
11 kB
"""d1-2B's decision pass as one fixed-shape PyTorch function (GPU layout), the questions an input:
image [1, 3, 352, 640] in [-1, 1], and the prompt (d1_common.prompt_inputs, fp16):
text [T_MAX - 220, D], head [U_MAX, D], attend [T_MAX, T_MAX], slots [S_MAX, T_MAX] -> logits [S_MAX, U_MAX]
Re-authored from mlx-vlm's lfm3 / lfm3_vl / d1_lfm3_vl (itself a port of the LiquidAI PyTorch code); weights from
LiquidAI/d1-2B safetensors. RoPE tables and the sliding mask are constants for T_MAX tokens; the image sits at
IMAGE_START, the prompt pads at the end (every layer but full attention is causal, and its mask is an input).
"""
import glob, json, math, os
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors import safe_open
from d1_common import IMAGE_START, T_MAX
CKPT = os.path.expanduser("~/.cache/huggingface/hub/models--LiquidAI--d1-2B/snapshots/*/") # hf download LiquidAI/d1-2B
def rms(x, w, eps=1e-5):
return F.rms_norm(x, (x.shape[-1],), w, eps)
def rotate(x, cos, sin):
a, b = x.chunk(2, dim=-1)
return x * cos + torch.cat([-b, a], dim=-1) * sin
def mlp(x, p):
return F.linear(F.silu(F.linear(x, p["w1"])) * F.linear(x, p["w3"]), p["w2"])
class _W(nn.Module):
"""Holds one projection weight as `.weight`: coreai-opt's w8 spec quantizes parameters named `weight` only."""
def __init__(self, t):
super().__init__()
self.weight = nn.Parameter(t, requires_grad=False)
def unwrap(module, n):
v = getattr(module, n)
return next(v.parameters()) if isinstance(v, _W) else v # its one parameter, whatever an export renamed it to
class D1Fixed(nn.Module):
def __init__(self, meta, dtype=torch.float16): # meta: any prompt (build_meta), for the image grid and window
super().__init__()
path = glob.glob(CKPT)[0]
cfg = json.load(open(path + "config.json"))
self.tc, self.vc, self.meta, self.dtype = cfg["text_config"], cfg["vision_config"], meta, dtype
f = safe_open(path + "model.safetensors", "pt")
g = lambda k: f.get_tensor(k)
self._n = 0
def P(t, q=None): # a weight constant; q: a projection weight (quantizable), by default any 2-D/4-D non-depthwise one
name = "w%d" % self._n
self._n += 1
t = t.to(dtype).contiguous()
if q if q is not None else (t.dim() == 2 or t.dim() == 4 and t.shape[1] > 1):
self.add_module(name, _W(t))
else:
self._const(name, t)
return name
_, gh, gw = meta["grid_thw"]
self.gh, self.gw = gh, gw
vt = "model.vision_tower."
# patchifier Linear(3*2*16*16 -> 1024), features (C, T, ph, pw): both temporal copies equal -> sum them
self.patch = P(g(vt + "patchifier.weight").float().reshape(1024, 3, 2, 16, 16).sum(2))
self.v_norm = P(g(vt + "embedding_norm.weight"))
self.vlayers = []
for i, kind in enumerate(self.vc["layer_types"]):
pre = vt + "layers.%d." % i
L = {"kind": kind, "operator_norm": P(g(pre + "operator_norm.weight")), "ffn_norm": P(g(pre + "ffn_norm.weight")),
"mlp": {k: P(g(pre + "feed_forward.%s.weight" % k)) for k in ("w1", "w2", "w3")}}
if kind == "conv":
L.update(in_proj=P(g(pre + "conv.in_proj.weight")), out_proj=P(g(pre + "conv.out_proj.weight")),
spatial=P(g(pre + "conv.spatial_conv.weight")))
else:
for k in ("q_proj", "k_proj", "v_proj", "out_proj", "q_layernorm", "k_layernorm"):
L[k] = P(g(pre + "self_attn.%s.weight" % k))
self.vlayers.append(L)
# vision 2-D RoPE in raster order (row, col)
hd = self.vc["hidden_size"] // self.vc["num_attention_heads"]
inv = self.vc["rope_parameters"]["rope_theta"] ** (-torch.arange(0, hd // 2, 2, dtype=torch.float64) / (hd // 2))
rows, cols = torch.meshgrid(torch.arange(gh), torch.arange(gw), indexing="ij")
coords = torch.stack([rows, cols], -1).reshape(-1, 2).double()
ang = (coords[..., None] * inv).reshape(gh * gw, -1)
ang = torch.cat([ang, ang], -1)
self._const("v_cos", ang.cos().float())
self._const("v_sin", ang.sin().float())
mp = "model.multi_modal_projector."
self.proj = {k: P(g(mp + k)) for k in ("layer_norm.weight", "layer_norm.bias", "linear_1.weight", "linear_1.bias",
"linear_2.weight", "linear_2.bias")}
lm = "model.language_model."
self.final_norm = P(g(lm + "embedding_norm.weight"))
self.layers = []
for i, kind in enumerate(self.tc["layer_types"]):
pre = lm + "layers.%d." % i
L = {"kind": kind, "mlp": {k: P(g(pre + "feed_forward.%s.weight" % k)) for k in ("w1", "w2", "w3")}}
for k in ("operator_norm", "ffn_norm", "post_operator_norm", "post_ffn_norm"):
L[k] = P(g(pre + k + ".weight"))
if kind == "conv":
L.update(in_proj=P(g(pre + "conv.in_proj.weight")), out_proj=P(g(pre + "conv.out_proj.weight")),
conv=P(g(pre + "conv.conv.weight")))
else:
for k in ("q_proj", "k_proj", "v_proj", "o_proj", "q_norm", "k_norm", "gate_proj"):
L[k] = P(g(pre + "self_attn.%s.weight" % k))
self.layers.append(L)
# text RoPE and the sliding mask, for T_MAX tokens
T = T_MAX
self.T = T
pos = torch.arange(T, dtype=torch.float64)
for kind in ("full_attention", "sliding_attention"):
rp = self.tc["rope_parameters"][kind]
d = self.tc["head_dim"]
angles = int(d * rp.get("partial_rotary_factor", 1.0)) // 2
inv = rp["rope_theta"] ** (-torch.arange(angles, dtype=torch.float64) * 2 / d)
inv = F.pad(inv, (0, d // 2 - angles))
a = pos[:, None] * inv
a = torch.cat([a, a], -1)
self._const(kind + "_cos", a.cos().to(dtype))
self._const(kind + "_sin", a.sin().to(dtype))
p = torch.arange(T)
slide = (p[:, None] >= p[None]) & (p[:, None] < p[None] + meta["sliding_window"])
self.neg = torch.finfo(dtype).min / 2
self._const("sliding_attention_mask", torch.zeros(T, T).masked_fill(~slide, self.neg).to(dtype))
def _const(self, name, t): # frozen parameter: torch.export lifts it as a weight, not as a graph input
self.register_parameter(name, nn.Parameter(t, requires_grad=False))
def w(self, name):
return unwrap(self, name)
def W(self, d):
return {k: self.w(v) for k, v in d.items()}
# ---- vision ----
def vision(self, image):
vc, gh, gw = self.vc, self.gh, self.gw
D = vc["hidden_size"]
x = F.conv2d(image.to(self.dtype), self.w(self.patch), stride=16) # [1, D, gh, gw]
x = x.flatten(2).transpose(1, 2) # [1, N, D] raster
x = rms(x, self.w(self.v_norm))
N = gh * gw
H, Hk = vc["num_attention_heads"], vc["num_key_value_heads"]
hd = D // H
for L in self.vlayers:
h = rms(x, self.w(L["operator_norm"]))
if L["kind"] == "conv":
b, c, v = F.linear(h, self.w(L["in_proj"])).chunk(3, -1)
grid = (b * v).transpose(1, 2).reshape(1, D, gh, gw)
grid = F.conv2d(grid, self.w(L["spatial"]), padding=1, groups=D)
h = F.linear(c * grid.reshape(1, D, N).transpose(1, 2), self.w(L["out_proj"]))
else:
q = rms(F.linear(h, self.w(L["q_proj"])).reshape(1, N, H, hd), self.w(L["q_layernorm"])).transpose(1, 2)
k = rms(F.linear(h, self.w(L["k_proj"])).reshape(1, N, Hk, hd), self.w(L["k_layernorm"])).transpose(1, 2)
v = F.linear(h, self.w(L["v_proj"])).reshape(1, N, Hk, hd).transpose(1, 2)
q = rotate(q.float(), self.v_cos, self.v_sin).to(self.dtype)
k = rotate(k.float(), self.v_cos, self.v_sin).to(self.dtype)
k, v = k.repeat_interleave(H // Hk, 1), v.repeat_interleave(H // Hk, 1)
o = F.scaled_dot_product_attention(q, k, v, scale=hd ** -0.5)
h = F.linear(o.transpose(1, 2).reshape(1, N, D), self.w(L["out_proj"]))
x = x + h
x = x + mlp(rms(x, self.w(L["ffn_norm"])), self.W(L["mlp"]))
# projector: LayerNorm per token, then 2x2 merge blocks in raster order of blocks
pj = self.W(self.proj)
x = F.layer_norm(x, (D,), pj["layer_norm.weight"], pj["layer_norm.bias"], 1e-6)
x = x.reshape(gh // 2, 2, gw // 2, 2, D).permute(0, 2, 1, 3, 4).reshape(gh * gw // 4, 4 * D)
x = F.gelu(F.linear(x, pj["linear_1.weight"], pj["linear_1.bias"]))
return F.linear(x, pj["linear_2.weight"], pj["linear_2.bias"]) # [220, 2048]
# ---- language ----
def forward(self, image, text, head, attend, slots):
tc, T = self.tc, self.T
D, H, Hk, hd = tc["hidden_size"], tc["num_attention_heads"], tc["num_key_value_heads"], tc["head_dim"]
dt = self.dtype
x = torch.cat([text[:IMAGE_START].to(dt), self.vision(image), text[IMAGE_START:].to(dt)], 0)[None] # [1, T, D]
masks = {"full_attention": (attend.to(dt) - 1) * -self.neg, "sliding_attention": self.w("sliding_attention_mask")}
for L in self.layers:
h = rms(x, self.w(L["operator_norm"]))
if L["kind"] == "conv":
b, c, v = F.linear(h, self.w(L["in_proj"])).chunk(3, -1)
bx = F.pad((b * v).transpose(1, 2), (6, 0))
h = F.linear(c * F.conv1d(bx, self.w(L["conv"]), groups=D).transpose(1, 2), self.w(L["out_proj"]))
else:
kind = L["kind"]
cos, sin = self.w(kind + "_cos"), self.w(kind + "_sin")
q = rms(F.linear(h, self.w(L["q_proj"])).reshape(1, T, H, hd), self.w(L["q_norm"])).transpose(1, 2)
k = rms(F.linear(h, self.w(L["k_proj"])).reshape(1, T, Hk, hd), self.w(L["k_norm"])).transpose(1, 2)
v = F.linear(h, self.w(L["v_proj"])).reshape(1, T, Hk, hd).transpose(1, 2)
q, k = rotate(q, cos, sin), rotate(k, cos, sin)
k, v = k.repeat_interleave(H // Hk, 1), v.repeat_interleave(H // Hk, 1)
o = F.scaled_dot_product_attention(q, k, v, attn_mask=masks[kind], scale=hd ** -0.5)
o = o.transpose(1, 2) * torch.sigmoid(F.linear(h, self.w(L["gate_proj"])))[..., None]
h = F.linear(o.reshape(1, T, D), self.w(L["o_proj"]))
x = x + rms(h, self.w(L["post_operator_norm"]))
x = x + rms(mlp(rms(x, self.w(L["ffn_norm"])), self.W(L["mlp"])), self.w(L["post_ffn_norm"]))
x = rms(x, self.w(self.final_norm))[0]
return ((slots.to(dt) @ x) @ head.to(dt).T).float() # [S_MAX, U_MAX]