File size: 8,941 Bytes
7a102c1 7011ac6 7a102c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | """What every backend shares: the constant prompt layout for our four questions at 640x352, and the host-side
readout that turns the exported function's [12, U] logits into system_one's answers (mlx-vlm's own code).
meta = build_meta(engine) # needs the mlx-vlm engine (tokenizer, config); cached to prompt.json
answers = readout(engine, logits) # logits [12, U] as float32
"""
import json, os
import numpy as np
from PIL import Image
HERE = os.path.dirname(os.path.abspath(__file__))
META = os.path.join(HERE, "prompt.json")
W, H = 640, 360 # what index.html sends; the processor makes it 640x352
def crop16x9(w, h):
"""The centre 16:9 rectangle of a w x h picture, (left, top, right, bottom) in pixels."""
if w * 9 > h * 16:
nw = h * 16 // 9
return (w - nw) // 2, 0, (w - nw) // 2 + nw, h
if w * 9 < h * 16:
nh = w * 9 // 16
return 0, (h - nh) // 2, w, (h - nh) // 2 + nh
return 0, 0, w, h
def frame(img):
"""Any picture as the page would send it: centre-cropped to 16:9, 640x360 RGB."""
img = img.convert("RGB")
c = crop16x9(*img.size)
if c != (0, 0) + img.size:
img = img.crop(c)
return img.resize((W, H), Image.Resampling.BICUBIC)
def uncrop_box(box, w, h):
"""A box in fractions of frame()'s crop, as fractions of the original w x h picture."""
l, t, r, b = crop16x9(w, h)
return {"top": (t + box["top"] * (b - t)) / h, "bottom": (t + box["bottom"] * (b - t)) / h,
"left": (l + box["left"] * (r - l)) / w, "right": (l + box["right"] * (r - l)) / w}
def engine_parts(engine, questions):
from mlx_vlm.models.d1_lfm3_vl import d1 as D, prompt as P
qs = list(engine.named(questions).values())
return D, P, qs
def build_meta(engine, questions, write=True):
D, P, qs = engine_parts(engine, questions)
tok = engine.tokenizer
markup, parts = engine.media.encode(D.media_list([frame(Image.new("RGB", (W, H), "gray"))]))
grid = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
prefix = P.prefix_text(tok, None, engine.bos, engine.state_style, engine.system, markup)
layouts = [P.suffix_layout(tok, q, engine.lead) for q in qs]
first = tok.encode(prefix + layouts[0][0], add_special_tokens=False)
ids, state_len, question, ends = D.pack_set(tok, prefix, [t for t, _ in layouts], P.question_header(tok, None),
[o for _, o in layouts], first)
slots, tables = [], []
for j, (q, at) in enumerate(zip(qs, ends)):
for r, pos in zip(P.question_slots(q, tok), at):
tokens, forms, _ = P.readout_table(tok, r)
slots.append({"question": j, "pos": pos, "kind": type(r).__name__, "tempered": not P.typed(q)})
tables.append([[tokens[i] for i in row] for row in forms.tolist()])
union = list(dict.fromkeys(t for tab in tables for row in tab for t in row))
at = {t: i for i, t in enumerate(union)}
for s, tab in zip(slots, tables):
s["forms"] = [[at[t] for t in row] for row in tab]
image_pos = [i for i, t in enumerate(ids) if t == engine.model.config.image_token_id]
assert image_pos == list(range(image_pos[0], image_pos[0] + len(image_pos))), "image tokens not contiguous"
meta = {"ids": ids, "state_len": state_len, "question": question, "grid_thw": grid,
"image_start": image_pos[0], "image_len": len(image_pos), "union": union, "slots": slots,
"sliding_window": engine.model.config.text_config.sliding_window}
if write:
with open(META, "w") as f:
json.dump(meta, f)
return meta
def load_meta():
with open(META) as f:
return json.load(f)
# The exported function takes the prompt as inputs, padded to these sizes: any QUESTIONS whose prompt fits run on the
# same .aimodel. The image sits at IMAGE_START (the system prefix before it does not depend on the questions).
T_MAX, S_MAX, U_MAX, IMAGE_START, IMAGE_LEN = 640, 16, 1152, 5, 220
def prompt_problem(meta):
"""Why these questions don't fit the export, or None."""
T, S, U = len(meta["ids"]), len(meta["slots"]), len(meta["union"])
if (meta["image_start"], meta["image_len"]) != (IMAGE_START, IMAGE_LEN):
return "the image is at %d (%d tokens), the export expects %d (%d)" % (meta["image_start"], meta["image_len"], IMAGE_START, IMAGE_LEN)
if T > T_MAX or S > S_MAX or U > U_MAX:
return ("QUESTIONS need %d tokens, %d answer slots, %d readout tokens; the Core AI export fits %d, %d, %d"
% (T, S, U, T_MAX, S_MAX, U_MAX))
return None
def prompt_inputs(meta, emb):
"""The function's prompt inputs for `meta` (build_meta), fp16 numpy; `emb` [vocab, D] the input embeddings:
text [T_MAX - IMAGE_LEN, D] (the text tokens' embeddings, the image left out), head [U_MAX, D] (the readout tokens'),
attend [T_MAX, T_MAX] (1 where query row i reads key j in full attention), slots [S_MAX, T_MAX] (one-hot rows)."""
why = prompt_problem(meta)
if why:
raise ValueError(why)
ids, T, S, U = meta["ids"], len(meta["ids"]), len(meta["slots"]), len(meta["union"])
s, n = meta["image_start"], meta["image_len"]
D = emb.shape[1]
text = np.zeros((T_MAX - n, D), np.float16)
text[:T - n] = np.asarray(emb[ids[:s] + ids[s + n:]], np.float16)
head = np.zeros((U_MAX, D), np.float16)
head[:U] = np.asarray(emb[meta["union"]], np.float16)
p = np.arange(T_MAX)
q = np.array(meta["question"] + [-2] * (T_MAX - T)) # padding: its own group, after everything (causal layers)
sl, real = meta["state_len"], p < T
# d1's set mask (d1.set_masks): the state reads and is read both ways, each question reads itself; the state does
# not read the padding (nothing real does)
full = ((p[:, None] < sl) & real[None]) | (p[None] < sl) | (q[:, None] == q[None])
slots = np.zeros((S_MAX, T_MAX), np.float16)
slots[np.arange(S), [x["pos"] for x in meta["slots"]]] = 1
return {"text": text, "head": head, "attend": full.astype(np.float16), "slots": slots}
def embeddings():
"""d1-2B's input embedding table [128000, 2048] fp16 (memory-mapped): from the PyTorch weights here, else the
Core AI repo's copy (prompt inputs need the exact embeddings, not the MLX model's 8-bit ones)."""
path = os.path.join(HERE, "out", "embed_tokens_fp16.npy")
if not os.path.exists(path):
from huggingface_hub import hf_hub_download
path = hf_hub_download("ivanfioravanti/d1-2B-gestures-CoreAI", "embed_tokens_fp16.npy")
return np.load(path, mmap_mode="r")
def pixels(engine, img):
"""The processor's own pixel_values for a 640x360 frame, folded back into the [1, 3, 352, 640] image the exported
graph takes (exact: the patches are a permutation of it, both temporal copies equal)."""
from mlx_vlm.models.d1_lfm3_vl.media import media_list
_, parts = engine.media.encode(media_list([img]))
pv = np.array(parts[0]["pixel_values"], dtype=np.float32)
t, gh, gw = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
x = pv.reshape(gh // 2, gw // 2, 2, 2, 3, 2, 16, 16)[:, :, :, :, :, 0] # (bh, bw, mr, mc, C, ph, pw)
return x.transpose(4, 0, 2, 5, 1, 3, 6).reshape(1, 3, gh * 16, gw * 16)
def mlx_logits(engine, img, meta=None):
"""The MLX reference's own [12, U] logits for a frame (the same pass system_one makes), to compare numerically."""
import mlx.core as mx
from mlx_vlm.models.d1_lfm3_vl.media import media_list
meta = meta or load_meta()
model = engine.model
_, parts = engine.media.encode(media_list([img]))
feats = [model.d1_features(p) for p in parts]
hidden = model.d1_hidden(meta["ids"], meta["state_len"], meta["question"], feats)
at = mx.array([s["pos"] for s in meta["slots"]])
logits = (hidden[at] @ model.language_model.model.embed_tokens(mx.array(meta["union"])).T).astype(mx.float32)
return np.array(logits)
def readout(engine, questions, logits, meta=None):
"""system_one's answers from the exported logits, via mlx-vlm's own probabilities / calibration / box code."""
from mlx_vlm.models.d1_lfm3_vl.prompt import Box, Boxes, Choice, probabilities, question_slots
meta = meta or load_meta()
named = engine.named(questions)
qs = list(named.values())
logits = np.asarray(logits, dtype=np.float64)
own = [[] for _ in qs]
k = 0
for j, q in enumerate(qs):
for r in question_slots(q, engine.tokenizer):
s = meta["slots"][k]
values = logits[k][np.array(s["forms"])].max(-1).tolist()
own[j].append(probabilities(r, values, engine.calibration if s["tempered"] else None))
k += 1
out = {}
for (name, q), probs in zip(named.items(), own):
p = probs if isinstance(q, (Box, Boxes)) else probs[0]
out[name] = engine.answer(None, q, p)
return out
|