File size: 8,941 Bytes
7a102c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7011ac6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7a102c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
"""What every backend shares: the constant prompt layout for our four questions at 640x352, and the host-side
readout that turns the exported function's [12, U] logits into system_one's answers (mlx-vlm's own code).

    meta = build_meta(engine)                   # needs the mlx-vlm engine (tokenizer, config); cached to prompt.json
    answers = readout(engine, logits)           # logits [12, U] as float32
"""
import json, os
import numpy as np
from PIL import Image

HERE = os.path.dirname(os.path.abspath(__file__))
META = os.path.join(HERE, "prompt.json")
W, H = 640, 360  # what index.html sends; the processor makes it 640x352


def crop16x9(w, h):
    """The centre 16:9 rectangle of a w x h picture, (left, top, right, bottom) in pixels."""
    if w * 9 > h * 16:
        nw = h * 16 // 9
        return (w - nw) // 2, 0, (w - nw) // 2 + nw, h
    if w * 9 < h * 16:
        nh = w * 9 // 16
        return 0, (h - nh) // 2, w, (h - nh) // 2 + nh
    return 0, 0, w, h


def frame(img):
    """Any picture as the page would send it: centre-cropped to 16:9, 640x360 RGB."""
    img = img.convert("RGB")
    c = crop16x9(*img.size)
    if c != (0, 0) + img.size:
        img = img.crop(c)
    return img.resize((W, H), Image.Resampling.BICUBIC)


def uncrop_box(box, w, h):
    """A box in fractions of frame()'s crop, as fractions of the original w x h picture."""
    l, t, r, b = crop16x9(w, h)
    return {"top": (t + box["top"] * (b - t)) / h, "bottom": (t + box["bottom"] * (b - t)) / h,
            "left": (l + box["left"] * (r - l)) / w, "right": (l + box["right"] * (r - l)) / w}


def engine_parts(engine, questions):
    from mlx_vlm.models.d1_lfm3_vl import d1 as D, prompt as P
    qs = list(engine.named(questions).values())
    return D, P, qs


def build_meta(engine, questions, write=True):
    D, P, qs = engine_parts(engine, questions)
    tok = engine.tokenizer
    markup, parts = engine.media.encode(D.media_list([frame(Image.new("RGB", (W, H), "gray"))]))
    grid = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
    prefix = P.prefix_text(tok, None, engine.bos, engine.state_style, engine.system, markup)
    layouts = [P.suffix_layout(tok, q, engine.lead) for q in qs]
    first = tok.encode(prefix + layouts[0][0], add_special_tokens=False)
    ids, state_len, question, ends = D.pack_set(tok, prefix, [t for t, _ in layouts], P.question_header(tok, None),
                                                [o for _, o in layouts], first)
    slots, tables = [], []
    for j, (q, at) in enumerate(zip(qs, ends)):
        for r, pos in zip(P.question_slots(q, tok), at):
            tokens, forms, _ = P.readout_table(tok, r)
            slots.append({"question": j, "pos": pos, "kind": type(r).__name__, "tempered": not P.typed(q)})
            tables.append([[tokens[i] for i in row] for row in forms.tolist()])
    union = list(dict.fromkeys(t for tab in tables for row in tab for t in row))
    at = {t: i for i, t in enumerate(union)}
    for s, tab in zip(slots, tables):
        s["forms"] = [[at[t] for t in row] for row in tab]
    image_pos = [i for i, t in enumerate(ids) if t == engine.model.config.image_token_id]
    assert image_pos == list(range(image_pos[0], image_pos[0] + len(image_pos))), "image tokens not contiguous"
    meta = {"ids": ids, "state_len": state_len, "question": question, "grid_thw": grid,
            "image_start": image_pos[0], "image_len": len(image_pos), "union": union, "slots": slots,
            "sliding_window": engine.model.config.text_config.sliding_window}
    if write:
        with open(META, "w") as f:
            json.dump(meta, f)
    return meta


def load_meta():
    with open(META) as f:
        return json.load(f)


# The exported function takes the prompt as inputs, padded to these sizes: any QUESTIONS whose prompt fits run on the
# same .aimodel. The image sits at IMAGE_START (the system prefix before it does not depend on the questions).
T_MAX, S_MAX, U_MAX, IMAGE_START, IMAGE_LEN = 640, 16, 1152, 5, 220


def prompt_problem(meta):
    """Why these questions don't fit the export, or None."""
    T, S, U = len(meta["ids"]), len(meta["slots"]), len(meta["union"])
    if (meta["image_start"], meta["image_len"]) != (IMAGE_START, IMAGE_LEN):
        return "the image is at %d (%d tokens), the export expects %d (%d)" % (meta["image_start"], meta["image_len"], IMAGE_START, IMAGE_LEN)
    if T > T_MAX or S > S_MAX or U > U_MAX:
        return ("QUESTIONS need %d tokens, %d answer slots, %d readout tokens; the Core AI export fits %d, %d, %d"
                % (T, S, U, T_MAX, S_MAX, U_MAX))
    return None


def prompt_inputs(meta, emb):
    """The function's prompt inputs for `meta` (build_meta), fp16 numpy; `emb` [vocab, D] the input embeddings:
    text [T_MAX - IMAGE_LEN, D] (the text tokens' embeddings, the image left out), head [U_MAX, D] (the readout tokens'),
    attend [T_MAX, T_MAX] (1 where query row i reads key j in full attention), slots [S_MAX, T_MAX] (one-hot rows)."""
    why = prompt_problem(meta)
    if why:
        raise ValueError(why)
    ids, T, S, U = meta["ids"], len(meta["ids"]), len(meta["slots"]), len(meta["union"])
    s, n = meta["image_start"], meta["image_len"]
    D = emb.shape[1]
    text = np.zeros((T_MAX - n, D), np.float16)
    text[:T - n] = np.asarray(emb[ids[:s] + ids[s + n:]], np.float16)
    head = np.zeros((U_MAX, D), np.float16)
    head[:U] = np.asarray(emb[meta["union"]], np.float16)
    p = np.arange(T_MAX)
    q = np.array(meta["question"] + [-2] * (T_MAX - T))  # padding: its own group, after everything (causal layers)
    sl, real = meta["state_len"], p < T
    # d1's set mask (d1.set_masks): the state reads and is read both ways, each question reads itself; the state does
    # not read the padding (nothing real does)
    full = ((p[:, None] < sl) & real[None]) | (p[None] < sl) | (q[:, None] == q[None])
    slots = np.zeros((S_MAX, T_MAX), np.float16)
    slots[np.arange(S), [x["pos"] for x in meta["slots"]]] = 1
    return {"text": text, "head": head, "attend": full.astype(np.float16), "slots": slots}


def embeddings():
    """d1-2B's input embedding table [128000, 2048] fp16 (memory-mapped): from the PyTorch weights here, else the
    Core AI repo's copy (prompt inputs need the exact embeddings, not the MLX model's 8-bit ones)."""
    path = os.path.join(HERE, "out", "embed_tokens_fp16.npy")
    if not os.path.exists(path):
        from huggingface_hub import hf_hub_download
        path = hf_hub_download("ivanfioravanti/d1-2B-gestures-CoreAI", "embed_tokens_fp16.npy")
    return np.load(path, mmap_mode="r")


def pixels(engine, img):
    """The processor's own pixel_values for a 640x360 frame, folded back into the [1, 3, 352, 640] image the exported
    graph takes (exact: the patches are a permutation of it, both temporal copies equal)."""
    from mlx_vlm.models.d1_lfm3_vl.media import media_list
    _, parts = engine.media.encode(media_list([img]))
    pv = np.array(parts[0]["pixel_values"], dtype=np.float32)
    t, gh, gw = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
    x = pv.reshape(gh // 2, gw // 2, 2, 2, 3, 2, 16, 16)[:, :, :, :, :, 0]  # (bh, bw, mr, mc, C, ph, pw)
    return x.transpose(4, 0, 2, 5, 1, 3, 6).reshape(1, 3, gh * 16, gw * 16)


def mlx_logits(engine, img, meta=None):
    """The MLX reference's own [12, U] logits for a frame (the same pass system_one makes), to compare numerically."""
    import mlx.core as mx
    from mlx_vlm.models.d1_lfm3_vl.media import media_list
    meta = meta or load_meta()
    model = engine.model
    _, parts = engine.media.encode(media_list([img]))
    feats = [model.d1_features(p) for p in parts]
    hidden = model.d1_hidden(meta["ids"], meta["state_len"], meta["question"], feats)
    at = mx.array([s["pos"] for s in meta["slots"]])
    logits = (hidden[at] @ model.language_model.model.embed_tokens(mx.array(meta["union"])).T).astype(mx.float32)
    return np.array(logits)


def readout(engine, questions, logits, meta=None):
    """system_one's answers from the exported logits, via mlx-vlm's own probabilities / calibration / box code."""
    from mlx_vlm.models.d1_lfm3_vl.prompt import Box, Boxes, Choice, probabilities, question_slots
    meta = meta or load_meta()
    named = engine.named(questions)
    qs = list(named.values())
    logits = np.asarray(logits, dtype=np.float64)
    own = [[] for _ in qs]
    k = 0
    for j, q in enumerate(qs):
        for r in question_slots(q, engine.tokenizer):
            s = meta["slots"][k]
            values = logits[k][np.array(s["forms"])].max(-1).tolist()
            own[j].append(probabilities(r, values, engine.calibration if s["tempered"] else None))
            k += 1
    out = {}
    for (name, q), probs in zip(named.items(), own):
        p = probs if isinstance(q, (Box, Boxes)) else probs[0]
        out[name] = engine.answer(None, q, p)
    return out