d1-2B-gestures-CoreAI / code /d1_common.py
ivanfioravanti's picture
Prompt as inputs (no baked questions), split Neural Engine model for iOS, card and code
7011ac6 verified
Raw History Blame Contribute Delete
8.94 kB
"""What every backend shares: the constant prompt layout for our four questions at 640x352, and the host-side
readout that turns the exported function's [12, U] logits into system_one's answers (mlx-vlm's own code).
meta = build_meta(engine) # needs the mlx-vlm engine (tokenizer, config); cached to prompt.json
answers = readout(engine, logits) # logits [12, U] as float32
"""
import json, os
import numpy as np
from PIL import Image
HERE = os.path.dirname(os.path.abspath(__file__))
META = os.path.join(HERE, "prompt.json")
W, H = 640, 360 # what index.html sends; the processor makes it 640x352
def crop16x9(w, h):
"""The centre 16:9 rectangle of a w x h picture, (left, top, right, bottom) in pixels."""
if w * 9 > h * 16:
nw = h * 16 // 9
return (w - nw) // 2, 0, (w - nw) // 2 + nw, h
if w * 9 < h * 16:
nh = w * 9 // 16
return 0, (h - nh) // 2, w, (h - nh) // 2 + nh
return 0, 0, w, h
def frame(img):
"""Any picture as the page would send it: centre-cropped to 16:9, 640x360 RGB."""
img = img.convert("RGB")
c = crop16x9(*img.size)
if c != (0, 0) + img.size:
img = img.crop(c)
return img.resize((W, H), Image.Resampling.BICUBIC)
def uncrop_box(box, w, h):
"""A box in fractions of frame()'s crop, as fractions of the original w x h picture."""
l, t, r, b = crop16x9(w, h)
return {"top": (t + box["top"] * (b - t)) / h, "bottom": (t + box["bottom"] * (b - t)) / h,
"left": (l + box["left"] * (r - l)) / w, "right": (l + box["right"] * (r - l)) / w}
def engine_parts(engine, questions):
from mlx_vlm.models.d1_lfm3_vl import d1 as D, prompt as P
qs = list(engine.named(questions).values())
return D, P, qs
def build_meta(engine, questions, write=True):
D, P, qs = engine_parts(engine, questions)
tok = engine.tokenizer
markup, parts = engine.media.encode(D.media_list([frame(Image.new("RGB", (W, H), "gray"))]))
grid = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
prefix = P.prefix_text(tok, None, engine.bos, engine.state_style, engine.system, markup)
layouts = [P.suffix_layout(tok, q, engine.lead) for q in qs]
first = tok.encode(prefix + layouts[0][0], add_special_tokens=False)
ids, state_len, question, ends = D.pack_set(tok, prefix, [t for t, _ in layouts], P.question_header(tok, None),
[o for _, o in layouts], first)
slots, tables = [], []
for j, (q, at) in enumerate(zip(qs, ends)):
for r, pos in zip(P.question_slots(q, tok), at):
tokens, forms, _ = P.readout_table(tok, r)
slots.append({"question": j, "pos": pos, "kind": type(r).__name__, "tempered": not P.typed(q)})
tables.append([[tokens[i] for i in row] for row in forms.tolist()])
union = list(dict.fromkeys(t for tab in tables for row in tab for t in row))
at = {t: i for i, t in enumerate(union)}
for s, tab in zip(slots, tables):
s["forms"] = [[at[t] for t in row] for row in tab]
image_pos = [i for i, t in enumerate(ids) if t == engine.model.config.image_token_id]
assert image_pos == list(range(image_pos[0], image_pos[0] + len(image_pos))), "image tokens not contiguous"
meta = {"ids": ids, "state_len": state_len, "question": question, "grid_thw": grid,
"image_start": image_pos[0], "image_len": len(image_pos), "union": union, "slots": slots,
"sliding_window": engine.model.config.text_config.sliding_window}
if write:
with open(META, "w") as f:
json.dump(meta, f)
return meta
def load_meta():
with open(META) as f:
return json.load(f)
# The exported function takes the prompt as inputs, padded to these sizes: any QUESTIONS whose prompt fits run on the
# same .aimodel. The image sits at IMAGE_START (the system prefix before it does not depend on the questions).
T_MAX, S_MAX, U_MAX, IMAGE_START, IMAGE_LEN = 640, 16, 1152, 5, 220
def prompt_problem(meta):
"""Why these questions don't fit the export, or None."""
T, S, U = len(meta["ids"]), len(meta["slots"]), len(meta["union"])
if (meta["image_start"], meta["image_len"]) != (IMAGE_START, IMAGE_LEN):
return "the image is at %d (%d tokens), the export expects %d (%d)" % (meta["image_start"], meta["image_len"], IMAGE_START, IMAGE_LEN)
if T > T_MAX or S > S_MAX or U > U_MAX:
return ("QUESTIONS need %d tokens, %d answer slots, %d readout tokens; the Core AI export fits %d, %d, %d"
% (T, S, U, T_MAX, S_MAX, U_MAX))
return None
def prompt_inputs(meta, emb):
"""The function's prompt inputs for `meta` (build_meta), fp16 numpy; `emb` [vocab, D] the input embeddings:
text [T_MAX - IMAGE_LEN, D] (the text tokens' embeddings, the image left out), head [U_MAX, D] (the readout tokens'),
attend [T_MAX, T_MAX] (1 where query row i reads key j in full attention), slots [S_MAX, T_MAX] (one-hot rows)."""
why = prompt_problem(meta)
if why:
raise ValueError(why)
ids, T, S, U = meta["ids"], len(meta["ids"]), len(meta["slots"]), len(meta["union"])
s, n = meta["image_start"], meta["image_len"]
D = emb.shape[1]
text = np.zeros((T_MAX - n, D), np.float16)
text[:T - n] = np.asarray(emb[ids[:s] + ids[s + n:]], np.float16)
head = np.zeros((U_MAX, D), np.float16)
head[:U] = np.asarray(emb[meta["union"]], np.float16)
p = np.arange(T_MAX)
q = np.array(meta["question"] + [-2] * (T_MAX - T)) # padding: its own group, after everything (causal layers)
sl, real = meta["state_len"], p < T
# d1's set mask (d1.set_masks): the state reads and is read both ways, each question reads itself; the state does
# not read the padding (nothing real does)
full = ((p[:, None] < sl) & real[None]) | (p[None] < sl) | (q[:, None] == q[None])
slots = np.zeros((S_MAX, T_MAX), np.float16)
slots[np.arange(S), [x["pos"] for x in meta["slots"]]] = 1
return {"text": text, "head": head, "attend": full.astype(np.float16), "slots": slots}
def embeddings():
"""d1-2B's input embedding table [128000, 2048] fp16 (memory-mapped): from the PyTorch weights here, else the
Core AI repo's copy (prompt inputs need the exact embeddings, not the MLX model's 8-bit ones)."""
path = os.path.join(HERE, "out", "embed_tokens_fp16.npy")
if not os.path.exists(path):
from huggingface_hub import hf_hub_download
path = hf_hub_download("ivanfioravanti/d1-2B-gestures-CoreAI", "embed_tokens_fp16.npy")
return np.load(path, mmap_mode="r")
def pixels(engine, img):
"""The processor's own pixel_values for a 640x360 frame, folded back into the [1, 3, 352, 640] image the exported
graph takes (exact: the patches are a permutation of it, both temporal copies equal)."""
from mlx_vlm.models.d1_lfm3_vl.media import media_list
_, parts = engine.media.encode(media_list([img]))
pv = np.array(parts[0]["pixel_values"], dtype=np.float32)
t, gh, gw = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
x = pv.reshape(gh // 2, gw // 2, 2, 2, 3, 2, 16, 16)[:, :, :, :, :, 0] # (bh, bw, mr, mc, C, ph, pw)
return x.transpose(4, 0, 2, 5, 1, 3, 6).reshape(1, 3, gh * 16, gw * 16)
def mlx_logits(engine, img, meta=None):
"""The MLX reference's own [12, U] logits for a frame (the same pass system_one makes), to compare numerically."""
import mlx.core as mx
from mlx_vlm.models.d1_lfm3_vl.media import media_list
meta = meta or load_meta()
model = engine.model
_, parts = engine.media.encode(media_list([img]))
feats = [model.d1_features(p) for p in parts]
hidden = model.d1_hidden(meta["ids"], meta["state_len"], meta["question"], feats)
at = mx.array([s["pos"] for s in meta["slots"]])
logits = (hidden[at] @ model.language_model.model.embed_tokens(mx.array(meta["union"])).T).astype(mx.float32)
return np.array(logits)
def readout(engine, questions, logits, meta=None):
"""system_one's answers from the exported logits, via mlx-vlm's own probabilities / calibration / box code."""
from mlx_vlm.models.d1_lfm3_vl.prompt import Box, Boxes, Choice, probabilities, question_slots
meta = meta or load_meta()
named = engine.named(questions)
qs = list(named.values())
logits = np.asarray(logits, dtype=np.float64)
own = [[] for _ in qs]
k = 0
for j, q in enumerate(qs):
for r in question_slots(q, engine.tokenizer):
s = meta["slots"][k]
values = logits[k][np.array(s["forms"])].max(-1).tolist()
own[j].append(probabilities(r, values, engine.calibration if s["tempered"] else None))
k += 1
out = {}
for (name, q), probs in zip(named.items(), own):
p = probs if isinstance(q, (Box, Boxes)) else probs[0]
out[name] = engine.answer(None, q, p)
return out