ivanfioravanti's picture
Prompt as inputs (no baked questions), split Neural Engine model for iOS, card and code
7011ac6 verified
Raw History Blame Contribute Delete
18.6 kB
"""The same fixed d1 function re-authored for the Neural Engine (coreai-models `model-authoring` rules):
BC1S layout [1, C, 1, S] (vision NCHW [1, C, 22, 40]), 1x1 Conv2d projections, per-head attention with the
`bchq,bkhc->bkhq` einsum, softmax over the key (channel) axis, additive masks (1, key, 1, query) at -40000,
fp16 everywhere (no fp32 literals: every scalar is an fp16 constant), the projector's 2x2 merge as a stride-2 conv.
image [1, 3, 352, 640] fp32 in [-1, 1], and the prompt as d1_torch (text, head, attend, slots; fp16)
-> logits [1, U_MAX, 1, S_MAX] fp16
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from d1_common import IMAGE_START
from d1_torch import D1Fixed, _W, unwrap
NEG = -40000.0
class D1ANE(nn.Module):
def __init__(self, src: D1Fixed):
super().__init__()
assert src.dtype == torch.float16
self.tc, self.vc, self.gh, self.gw, self.T = src.tc, src.vc, src.gh, src.gw, src.T
h = torch.float16
w = src.w
conv = lambda t: t.reshape(*t.shape, 1, 1) if t.dim() == 2 else t # Linear [O, I] -> Conv2d [O, I, 1, 1]
col = lambda t: t.reshape(1, -1, 1, 1) # norm / bias weight over channels
def C(name, t): # a constant; projection / conv weights (O>1, I>1) in a `.weight` holder, quantizable
name = "p_" + name
t = t.to(h).contiguous()
if t.dim() == 4 and t.shape[0] > 1 and t.shape[1] > 1:
self.add_module(name, _W(t))
else:
self.register_parameter(name, nn.Parameter(t, requires_grad=False))
return name
self.c = {}
for k, v in {"one": 1.0, "half": 0.5, "eps": 1e-5, "eps_ln": 1e-6, "v_scale": (self.vc["hidden_size"] // self.vc["num_attention_heads"]) ** -0.5,
"t_scale": self.tc["head_dim"] ** -0.5, "ts": self.TS, "neg": -NEG, "pre": 8.0, "eps_pre": 1e-5 * 64.0, "eps_v": 1e-5 * self.VS ** 2, "inv_vs": 1 / self.VS, "inv_ts": 1 / self.TS, "inv_ps": 1 / self.PATCH, "eps_pconv": 1e-5 * self.PS["conv"] ** 2, "eps_pattn": 1e-5 * self.PS["attn"] ** 2, "eps_pmlp": 1e-5 * self.PS["mlp"] ** 2}.items():
self.c[k] = C("c_" + k, torch.tensor([v]))
self.patch = C("patch", w(src.patch))
self.v_norm = C("v_norm", col(w(src.v_norm)))
self.vl = []
for i, L in enumerate(src.vlayers):
d = {"kind": L["kind"], "operator_norm": C("v%d_on" % i, col(w(L["operator_norm"]))),
"ffn_norm": C("v%d_fn" % i, col(w(L["ffn_norm"]))),
"mlp": {k: C("v%d_%s" % (i, k), conv(w(n))) for k, n in L["mlp"].items()}}
if L["kind"] == "conv":
d.update(in_proj=C("v%d_in" % i, conv(w(L["in_proj"]))), out_proj=C("v%d_out" % i, conv(w(L["out_proj"]))),
spatial=C("v%d_sp" % i, w(L["spatial"])))
else:
for k in ("q_proj", "k_proj", "v_proj", "out_proj"):
d[k] = C("v%d_%s" % (i, k), conv(w(L[k])))
for k in ("q_layernorm", "k_layernorm"):
d[k] = C("v%d_%s" % (i, k), w(L[k]).reshape(1, 1, -1, 1))
self.vl.append(d)
N, hd = self.gh * self.gw, self.vc["hidden_size"] // self.vc["num_attention_heads"]
self.v_cos = C("v_cos", src.v_cos.T.reshape(1, 1, hd, N))
self.v_sin = C("v_sin", src.v_sin.T.reshape(1, 1, hd, N))
pj = {k: w(v) for k, v in src.proj.items()}
D = self.vc["hidden_size"]
self.ln_w, self.ln_b = C("ln_w", col(pj["layer_norm.weight"])), C("ln_b", col(pj["layer_norm.bias"]))
# merged feature index (mr*2+mc)*D + c -> conv weight [O, c, mr, mc], kernel 2 stride 2
W1 = pj["linear_1.weight"]
self.l1 = C("l1", W1.reshape(W1.shape[0], 2, 2, D).permute(0, 3, 1, 2))
self.l1b = C("l1b", pj["linear_1.bias"])
self.l2 = C("l2", conv(pj["linear_2.weight"]))
self.l2b = C("l2b", pj["linear_2.bias"])
self.final_norm = C("final_norm", col(w(src.final_norm)))
self.ll = []
for i, L in enumerate(src.layers):
d = {"kind": L["kind"], "mlp": {k: C("t%d_%s" % (i, k), conv(w(n))) for k, n in L["mlp"].items()}}
for k in ("operator_norm", "ffn_norm", "post_operator_norm", "post_ffn_norm"):
d[k] = C("t%d_%s" % (i, k), col(w(L[k])))
if L["kind"] == "conv":
d.update(in_proj=C("t%d_in" % i, conv(w(L["in_proj"]))), out_proj=C("t%d_out" % i, conv(w(L["out_proj"]))),
conv=C("t%d_conv" % i, w(L["conv"]).reshape(-1, 1, 1, w(L["conv"]).shape[-1])))
else:
for k in ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj"):
d[k] = C("t%d_%s" % (i, k), conv(w(L[k])))
for k in ("q_norm", "k_norm"):
d[k] = C("t%d_%s" % (i, k), w(L[k]).reshape(1, 1, -1, 1))
self.ll.append(d)
T, thd = self.T, self.tc["head_dim"]
for kind in ("full_attention", "sliding_attention"):
C(kind + "_cos", w(kind + "_cos").T.reshape(1, 1, thd, T))
C(kind + "_sin", w(kind + "_sin").T.reshape(1, 1, thd, T))
m = (w("sliding_attention_mask") < 0).T # [key, query]; the full attention mask is an input (attend)
C("sliding_attention_mask", torch.zeros(T, T).masked_fill(m, NEG).reshape(1, T, 1, T))
self.taps = None # debug: a list collects intermediates, forward then returns (logits, *taps)
self.rescale()
self.split_mlps()
# Scales on the 1x1 conv inputs (exact algebra): the ANE loses precision on small conv inputs (out_proj on its real
# input 43.8 dB, the same input x64 77 dB, measured with single-op exports), so (1) every norm weight is folded into the convs it feeds,
# whose input is then the unit-RMS normalised value, and (2) the rows that make the small products feeding out_proj /
# o_proj / w2 (the gate c, the values v, w3) are scaled up by S and the consuming conv down by S. S keeps the largest
# measured input (out/conv_in_probe.py) far below fp16's max.
S = {"vconv": 64.0, "vattn": 32.0, "vmlp": 32.0, "tconv": 64.0, "tattn": 32.0, "tmlp": 16.0}
# (3) the vision residual stream itself is ~0.03 and the ANE loses precision on small outputs, so the whole stream is
# carried at VS x (v_norm weight and every branch's last conv x VS, vision eps x VS^2), divided back before the
# projector LayerNorm. Largest stream value before the last layer 1.34 (x64 = 86: fp16 squares safe).
VS = 64.0
TS = 32.0 # same for the LM stream (max 28 -> 900)
PATCH = 256.0 # patch embed output x256: a mid-gray frame is 0.0039 after normalisation (blank_gray failed at 28.9 dB)
# (4) LM branch outputs only feed the (scale-invariant) post norms but are small (conv branch median token-max
# 0.0225, out/post_probe.py): their last conv is scaled up, the post norm's eps by the square.
PS = {"conv": 64.0, "attn": 16.0, "mlp": 4.0}
def rescale(self):
g = lambda n: self.g(n).data
def setp(n, t): g(n).copy_(t.to(torch.float16))
def fold(norm, convs): # norm weight [1, C, 1, 1] into conv input channels
wn = g(norm).float().reshape(1, -1, 1, 1)
for c in convs:
setp(c, g(c).float() * wn)
def rows(n, lo, hi, f):
t = g(n).float(); t[lo:hi] *= f; setp(n, t)
def scale(n, f): setp(n, g(n).float() * f)
for p, layers in (("v", self.vl), ("t", self.ll)):
for L in layers:
m = L["mlp"]
if L["kind"] == "conv":
fold(L["operator_norm"], [L["in_proj"]])
D = g(L["in_proj"]).shape[0] // 3
rows(L["in_proj"], D, 2 * D, self.S[p + "conv"]); scale(L["out_proj"], 1 / self.S[p + "conv"])
else:
fold(L["operator_norm"], [L["q_proj"], L["k_proj"], L["v_proj"]] + ([L["gate_proj"]] if p == "t" else []))
rows(L["v_proj"], 0, None, self.S[p + "attn"]); scale(L["out_proj" if p == "v" else "o_proj"], 1 / self.S[p + "attn"])
L["operator_norm"] = None
fold(L["ffn_norm"], [m["w1"], m["w3"]]); L["ffn_norm"] = None
rows(m["w3"], 0, None, self.S[p + "mlp"]); scale(m["w2"], 1 / self.S[p + "mlp"])
scale(self.v_norm, self.VS)
scale(self.patch, self.PATCH)
for L in self.vl:
scale(L["out_proj"], self.VS); scale(L["mlp"]["w2"], self.VS)
for n in (self.l2, self.l2b): # the text embeddings are scaled by TS in the graph (an input now)
scale(n, self.TS)
for L in self.ll:
scale(L["post_operator_norm"], self.TS); scale(L["post_ffn_norm"], self.TS)
scale(L["out_proj"] if L["kind"] == "conv" else L["o_proj"], self.PS["conv" if L["kind"] == "conv" else "attn"])
scale(L["mlp"]["w2"], self.PS["mlp"])
# projector LayerNorm affine into the merge conv: W(x*w + b) = (W*w) x + W b
W, w_, b_ = g(self.l1).float(), g(self.ln_w).float().reshape(1, -1, 1, 1), g(self.ln_b).float().reshape(-1)
setp(self.l1b, g(self.l1b).float() + torch.einsum("ochw,c->o", W, b_)); setp(self.l1, W * w_)
self.ln_w = self.ln_b = None
def g(self, n):
return unwrap(self, n)
def _t(self, x):
if self.taps is not None:
self.taps.append(x)
return x
def k(self, name):
return self.g(self.c[name]) if name in self.c else self.g(name)
# ---- primitives (BC1S) ----
def norm(self, x, w, eps="eps"): # RMSNorm over channels (dim 1); "eps_v": the vision stream, carried at VS x
if eps in ("eps_v", "eps_t", "eps_p"):
x = x * self.k({"eps_v": "inv_vs", "eps_t": "inv_ts", "eps_p": "inv_ps"}[eps]) # squares at the true scale
eps = "eps"
y = x * torch.rsqrt((x * x).mean(1, keepdim=True) + self.k(eps))
return y if w is None else y * self.g(w)
def head_norm(self, x, w): # per-head RMSNorm, x [1, H, hd, S]; pre-scaled by 8 (eps by 64): fp16 squares stay normal
y = x * self.k("pre")
return y * torch.rsqrt((y * y).mean(2, keepdim=True) + self.k("eps_pre")) * self.g(w)
def proj(self, x, w, b=None):
return F.conv2d(x, self.g(w), None if b is None else self.g(b))
def sigmoid(self, x): # the ANE's own sigmoid is coarse (SiLU 52 dB vs 84 dB this way, single-op exports); tanh is not
return (self.k("one") + torch.tanh(x * self.k("half"))) * self.k("half")
def silu(self, x):
return x * self.sigmoid(x)
# The MLP's hidden channels in chunks, w2's partial products summed: 1.4x faster on the ANE for the whole model
# (MLP 8.0 -> 5.5 ms; dev/mlp_variants.py, speed.py). Each chunk is its own weight, so a quantizer sees plain convs.
MLP_SPLIT = 8
def split_mlps(self):
for L in self.vl + self.ll:
p = L["mlp"]
n = self.g(p["w1"]).shape[0] // self.MLP_SPLIT
for k, cut in (("w1", lambda w, s: w[s]), ("w3", lambda w, s: w[s]), ("w2", lambda w, s: w[:, s])):
w = self.g(p[k])
p[k + "s"] = []
for i in range(self.MLP_SPLIT):
name = "%s_%d" % (p[k], i)
self.add_module(name, _W(cut(w, slice(i * n, (i + 1) * n)).clone().contiguous()))
p[k + "s"].append(name)
delattr(self, p[k])
def mlp(self, x, p):
out = None
for w1, w3, w2 in zip(p["w1s"], p["w3s"], p["w2s"]):
y = self.proj(self.silu(self.proj(x, w1)) * self.proj(x, w3), w2)
out = y if out is None else out + y
return out
@staticmethod
def rope(x, cos, sin): # x [1, H, hd, S], cos/sin [1, 1, hd, S]
half = x.shape[2] // 2
a, b = x[:, :, :half], x[:, :, half:]
return x * cos + torch.cat([-b, a], dim=2) * sin
@staticmethod
def attention(q, k, v, H, Hk, hd, mask=None):
"""q [1, H*hd, 1, Sq] (already scaled), k/v [1, Hk*hd, 1, Sk]; mask [1, Sk, 1, Sq] additive."""
qs = q.split(hd, dim=1)
ks = k.transpose(1, 3).split(hd, dim=3) # [1, Sk, 1, hd]
vs = v.transpose(1, 3).split(hd, dim=3)
out = []
for h in range(H):
j = h // (H // Hk)
s = torch.einsum("bchq,bkhc->bkhq", qs[h], ks[j]) # [1, Sk, 1, Sq]
if mask is not None:
s = s + mask
s = torch.softmax(s, dim=1)
out.append(torch.einsum("bkhq,bkhc->bchq", s, vs[j])) # [1, hd, 1, Sq]
return torch.cat(out, dim=1)
# ---- model ----
def vision(self, image):
vc, gh, gw = self.vc, self.gh, self.gw
D, H, Hk = vc["hidden_size"], vc["num_attention_heads"], vc["num_key_value_heads"]
hd, N = D // H, gh * gw
x = F.conv2d(image.to(self.g(self.patch).dtype), self.g(self.patch), stride=16) # [1, D, gh, gw]
x = self._t(self.norm(x, self.v_norm, "eps_p"))
for L in self.vl:
h = self.norm(x, L["operator_norm"], "eps_v")
if L["kind"] == "conv":
b, c, v = self.proj(h, L["in_proj"]).chunk(3, dim=1)
h = self.proj(c * F.conv2d(b * v, self.g(L["spatial"]), padding=1, groups=D), L["out_proj"])
else:
f = h.reshape(1, D, 1, N)
q = self.head_norm(self.proj(f, L["q_proj"]).reshape(1, H, hd, N), L["q_layernorm"])
k = self.head_norm(self.proj(f, L["k_proj"]).reshape(1, Hk, hd, N), L["k_layernorm"])
q = self.rope(q, self.g(self.v_cos), self.g(self.v_sin)) * self.k("v_scale")
k = self.rope(k, self.g(self.v_cos), self.g(self.v_sin))
o = self.attention(q.reshape(1, H * hd, 1, N), k.reshape(1, Hk * hd, 1, N), self.proj(f, L["v_proj"]), H, Hk, hd)
h = self.proj(o, L["out_proj"]).reshape(1, D, gh, gw)
x = x + h
x = self._t(x + self.mlp(self.norm(x, L["ffn_norm"], "eps_v"), L["mlp"]))
x = x * self.k("inv_vs")
xc = x - x.mean(1, keepdim=True)
x = xc * torch.rsqrt((xc * xc).mean(1, keepdim=True) + self.k("eps_ln")) # LayerNorm, affine folded into l1
x = F.gelu(F.conv2d(x, self.g(self.l1), self.g(self.l1b), stride=2)) # [1, 4096, gh/2, gw/2]
x = F.conv2d(x, self.g(self.l2), self.g(self.l2b)) # [1, 2048, gh/2, gw/2]
return self._t(x.reshape(1, x.shape[1], 1, -1)) # [1, 2048, 1, 220], blocks in raster order
def forward(self, image, text, head, attend, slots):
return self.language(self.vision(image), text, head, attend, slots)
def layer(self, L, x, full):
tc, T = self.tc, self.T
D, H, Hk, hd = tc["hidden_size"], tc["num_attention_heads"], tc["num_key_value_heads"], tc["head_dim"]
h = self.norm(x, L["operator_norm"], "eps_t")
if L["kind"] == "conv":
b, c, v = self.proj(h, L["in_proj"]).chunk(3, dim=1)
h = self.proj(c * F.conv2d(F.pad(b * v, (6, 0)), self.g(L["conv"]), groups=D), L["out_proj"])
else:
kind = L["kind"]
cos, sin = self.g("p_" + kind + "_cos"), self.g("p_" + kind + "_sin")
q = self.head_norm(self.proj(h, L["q_proj"]).reshape(1, H, hd, T), L["q_norm"])
k = self.head_norm(self.proj(h, L["k_proj"]).reshape(1, Hk, hd, T), L["k_norm"])
q = self.rope(q, cos, sin) * self.k("t_scale")
k = self.rope(k, cos, sin)
o = self.attention(q.reshape(1, H * hd, 1, T), k.reshape(1, Hk * hd, 1, T), self.proj(h, L["v_proj"]), H, Hk, hd,
full if kind == "full_attention" else self.g("p_sliding_attention_mask"))
g = self.sigmoid(self.proj(h, L["gate_proj"])) # [1, H, 1, T]
h = self.proj((o.reshape(1, H, hd, T) * g.reshape(1, H, 1, T)).reshape(1, D, 1, T), L["o_proj"])
x = x + self.norm(h, L["post_operator_norm"], "eps_pconv" if L["kind"] == "conv" else "eps_pattn")
return self._t(x + self.norm(self.mlp(self.norm(x, L["ffn_norm"], "eps_t"), L["mlp"]), L["post_ffn_norm"], "eps_pmlp"))
def language(self, feats, text, head, attend, slots):
"""feats [1, 2048, 1, 220] (the projector's output) and the prompt -> logits [1, U_MAX, 1, S_MAX]"""
tc, T = self.tc, self.T
D = tc["hidden_size"]
h16 = torch.float16
t = text.to(h16).transpose(0, 1).reshape(1, D, 1, -1) * self.k("ts") # the LM stream is carried at TS x
x = torch.cat([t[:, :, :, :IMAGE_START], feats.to(h16), t[:, :, :, IMAGE_START:]], dim=3) # [1, D, 1, T]
full = ((attend.to(h16).transpose(0, 1) - self.k("one")) * self.k("neg")).reshape(1, T, 1, T) # [1, key, 1, query]
for L in self.ll:
x = self.layer(L, x, full)
x = torch.einsum("bdht,st->bdhs", x, slots.to(h16)) # [1, D, 1, S_MAX]
out = torch.einsum("bdhs,ud->buhs", self.norm(x, self.final_norm, "eps_t"), head.to(h16)) # [1, U_MAX, 1, S_MAX]
return out if self.taps is None else (out, *self.taps)
class D1ANEPart(nn.Module):
"""One piece of D1ANE, for a Neural Engine that can't compile the whole 3.1 GB function at once (iOS: "Could not
retrieve data from weight file"): text layers [lo, hi), the first piece with the vision and the embeddings, the last
with the readout. Pieces pass the LM stream x [1, D, 1, T_MAX] (fp16, carried at TS x) from one to the next:
first: image, text, attend -> x middle: x, attend -> x last: x, attend, slots, head -> logits
"""
def __init__(self, ane: D1ANE, lo, hi):
super().__init__()
self.ane, self.lo, self.hi = ane, lo, hi
self.first, self.last = lo == 0, hi == len(ane.ll)
def forward(self, image=None, text=None, attend=None, x=None, slots=None, head=None):
a, h16 = self.ane, torch.float16
T, D = a.T, a.tc["hidden_size"]
if self.first:
t = text.to(h16).transpose(0, 1).reshape(1, D, 1, -1) * a.k("ts")
x = torch.cat([t[:, :, :, :IMAGE_START], a.vision(image).to(h16), t[:, :, :, IMAGE_START:]], dim=3)
else:
x = x.to(h16)
full = ((attend.to(h16).transpose(0, 1) - a.k("one")) * a.k("neg")).reshape(1, T, 1, T)
for L in a.ll[self.lo:self.hi]:
x = a.layer(L, x, full)
if not self.last:
return x
x = torch.einsum("bdht,st->bdhs", x, slots.to(h16))
return torch.einsum("bdhs,ud->buhs", a.norm(x, a.final_norm, "eps_t"), head.to(h16))