File size: 4,390 Bytes
7011ac6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | """Everything the iPhone / iPad app (ios/) needs besides the .aimodel, for QUESTIONS: the prompt inputs
(ios/D1Kit/Sources/D1Kit/prompt/*.f16, d1_common.prompt_inputs), the readout layout (d1.json), and the Swift tests'
fixtures (MLX's logits, pixels and answers on the parity frames) under ios/D1Kit/Tests/D1KitTests/Fixtures.
Edit QUESTIONS, run this, rebuild the app: the models stay the same.
.venv/bin/python export_app.py
d1.json is the prompt.json of the export in the order Swift reads it: per answer slot, its question, the token forms each
option scores as its best (indexes into the logits' columns) and its temperature; per box, the calibration tables.
"""
import json, os
import numpy as np
from PIL import Image, ImageOps
from mlx_vlm import load
from mlx_vlm.models.d1_lfm3_vl import prompt as P
from mlx_vlm.models.d1_lfm3_vl.box import edge_grid
from d1_common import S_MAX, U_MAX, build_meta, embeddings, frame, mlx_logits, pixels, prompt_inputs, readout
from questions import QUESTIONS
HERE = os.path.dirname(os.path.abspath(__file__))
KIT = os.path.join(HERE, "..", "ios", "D1Kit")
FIX = os.path.join(KIT, "Tests", "D1KitTests", "Fixtures")
model, _ = load("LiquidAI/d1-2B-MLX-8bit")
eng = model.engine
meta = build_meta(eng, QUESTIONS, write=False)
cal, tok = eng.calibration, eng.tokenizer
grid, scale = edge_grid(tok)
assert scale == 1000 and getattr(tok, "d1_box_edges", "median") == "median", "the Swift readout reads thousandths at the median"
questions, k = [], 0
for name, q in eng.named(QUESTIONS).items():
rs = P.question_slots(q, tok)
entry = {"name": name, "type": q.type, "slots": list(range(k, k + len(rs)))}
if q.type == "choice":
assert not q.multiple
entry["keys"] = list(q.criteria)
questions.append(entry)
for r in rs:
s = meta["slots"][k]
s["temperature"] = cal.temperature(r) if s["tempered"] else 1.0
k += 1
assert k == len(meta["slots"])
prompt = prompt_inputs(meta, embeddings())
os.makedirs(os.path.join(KIT, "Sources", "D1Kit", "prompt"), exist_ok=True)
for k, v in prompt.items():
v.tofile(os.path.join(KIT, "Sources", "D1Kit", "prompt", k + ".f16"))
layout = {
"input_tokens": len(meta["ids"]), "rows": S_MAX, "columns": U_MAX, "width": 640, "height": 352,
"prompt": {k: list(v.shape) for k, v in prompt.items()},
"questions": questions,
"slots": [{"forms": s["forms"], "temperature": s["temperature"]} for s in meta["slots"]],
"edge_grid": {"low": grid.low, "high": grid.high, "size": grid.size},
"tables": {n: cal.tables.get(n, []) for n in ("found:box", "box")},
}
os.makedirs(os.path.join(KIT, "Sources", "D1Kit"), exist_ok=True)
with open(os.path.join(KIT, "Sources", "D1Kit", "d1.json"), "w") as f:
json.dump(layout, f, separators=(",", ":"))
# fixtures: the parity frames, each as the 640x360 16:9 frame Swift reads, MLX's logits and both answers
S = Image.open(os.path.join(HERE, "..", "screenshot.png")).convert("RGB")
w, h = S.size
frames = {"screenshot": S, "screenshot_flip": ImageOps.mirror(S),
"screenshot_center60": S.crop((int(w * .2), int(h * .2), int(w * .8), int(h * .8))),
"blank_gray": Image.new("RGB", (640, 360), "gray")}
ivan = "/Users/ifioravanti/projects/local-ai-demos/ivan.jpg"
if os.path.exists(ivan):
frames["ivan"] = Image.open(ivan).convert("RGB")
os.makedirs(FIX, exist_ok=True)
out = []
for name, img in frames.items():
f = frame(img)
f.save(os.path.join(FIX, name + ".png"))
pixels(eng, f).astype(np.float32).tofile(os.path.join(FIX, name + ".f32")) # the processor's own input tensor
logits = mlx_logits(eng, f, meta)
padded = np.zeros((S_MAX, U_MAX), np.float32) # as the exported function gives them
padded[:logits.shape[0], :logits.shape[1]] = logits
out.append({"name": name, "logits": padded.astype(float).tolist(),
"readout": readout(eng, QUESTIONS, logits, meta), # the same logits through mlx-vlm: Swift must match exactly
"mlx": model.system_one(None, QUESTIONS, media=[f])["answers"]}) # MLX end to end: what the app should say
print(name, out[-1]["mlx"]["hand"]["choice"], out[-1]["mlx"]["face"]["choice"], flush=True)
with open(os.path.join(FIX, "fixtures.json"), "w") as fh:
json.dump(out, fh, default=float)
print("wrote d1.json (%d slots) and %d fixtures" % (len(layout["slots"]), len(out)))
|