File size: 2,091 Bytes
7011ac6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """The Neural Engine function in pieces (D1ANEPart), for the iPhone's Neural Engine: N .aimodel folders run in order.
.venv/bin/python export_ane_split.py [N=4] [out/d1_ane_prompt_split_fp16] # -> out/.../part0.aimodel ... part{N-1}
"""
import gc, os, shutil, sys, time
from pathlib import Path
import torch
from coreai_models.export.macos import export_to_coreai
from d1_common import embeddings, load_meta, prompt_inputs
from d1_torch import D1Fixed
from d1_ane import D1ANE, D1ANEPart
n = int(sys.argv[1]) if len(sys.argv) > 1 else 4
out = Path(sys.argv[2] if len(sys.argv) > 2 else "out/d1_ane_prompt_split_fp16")
meta = load_meta()
_, gh, gw = meta["grid_thw"]
p = {k: torch.from_numpy(v) for k, v in prompt_inputs(meta, embeddings()).items()}
ane = D1ANE(D1Fixed(meta, torch.float16)).eval()
layers = len(ane.ll)
# the first piece also holds the vision (0.38 GB, about 3 text layers' worth): it gets 3 layers fewer
cuts = [0] + [round(3 + (layers + 3) * i / n) - 3 for i in range(1, n)] + [layers]
x = None
out.mkdir(parents=True, exist_ok=True)
for i, (lo, hi) in enumerate(zip(cuts, cuts[1:])):
part = D1ANEPart(ane, lo, hi).eval()
if part.first:
ref, names = (torch.zeros(1, 3, gh * 16, gw * 16), p["text"], p["attend"]), ("image", "text", "attend")
elif part.last:
ref, names = (x, p["attend"], p["slots"], p["head"]), ("x", "attend", "slots", "head")
else:
ref, names = (x, p["attend"]), ("x", "attend")
with torch.no_grad():
y = part(**dict(zip(names, ref)))
t = time.time()
program = export_to_coreai(part, dict(zip(names, ref)), input_names=names, output_names=("logits" if part.last else "x",))
program.optimize()
dst = out / ("part%d.aimodel" % i)
if dst.exists():
shutil.rmtree(dst)
program.save_asset(dst)
size = sum(f.stat().st_size for f in dst.iterdir()) / 1e9
print("part%d: layers %d-%d%s, %.2f GB, %.0fs" % (i, lo, hi - 1, " + vision" if part.first else " + readout" if part.last else "", size, time.time() - t), flush=True)
x, program = y, None
gc.collect()
|