"""Image smoke: a synthetic image (red square on white, text-free) -> the vision tower runs, the answers are sane, the per-image encoding cache works (2nd request on the same image does not re-encode), and results are identical between a System One image part and the images= argument. With --ref compares against stored reference probabilities.""" import argparse, json, time import numpy as np from tests.common import REPO, agreement, answer_probs def make_images(): from PIL import Image, ImageDraw a = Image.new("RGB", (640, 480), "white"); ImageDraw.Draw(a).rectangle([200, 150, 440, 330], fill="red") b = Image.new("RGB", (512, 512), "black"); d = ImageDraw.Draw(b) for i in range(0, 512, 64): d.line([(i, 0), (i, 512)], fill="white", width=4); d.line([(0, i), (512, i)], fill="white", width=4) return {"red_square": a, "grid": b} QS = {"color": {"type": "choice", "instructions": "What is the dominant colour of the shape in the image?", "criteria": {"red": "red", "green": "green", "blue": "blue", "none": "no coloured shape"}}, "grid": {"type": "noul", "instructions": "Does the image show a grid of straight lines?"}, "photo": {"type": "noul", "instructions": "Is this a photograph of a real-world scene?"}} def run(device=None, precision="bf16", ref=None, verbose=True): from julia2 import Runtime rt = Runtime(REPO, device=device, precision=precision) assert rt.vision is not None, "no vision tower in this checkpoint" imgs = make_images(); res = {"device": str(rt.device), "precision": precision, "answers": {}} for name, img in imgs.items(): t0 = time.time(); out = rt.predict_questions("", QS, images=[img]); t1 = time.time() - t0 t0 = time.time(); out2 = rt.predict_questions([{"type": "image", "image": img}, {"type": "text", "text": ""}], QS); t2 = time.time() - t0 assert len(rt._img_cache) >= 1 and t2 <= t1 * 1.5 + 0.05, "image cache not used" p1 = [answer_probs(a) for a in out["answers"].values()]; p2 = [answer_probs(a) for a in out2["answers"].values()] assert agreement(p1, p2)["maxdp_max"] < 1e-6, "image part vs images= differ" res["answers"][name] = {k: (a.get("choice") if a["type"] == "choice" else round(a["noul"], 3)) for k, a in out["answers"].items()} res[name + "_ms_first"] = round(t1 * 1000); res[name + "_ms_cached"] = round(t2 * 1000) assert res["answers"]["red_square"]["color"] == "red", res assert res["answers"]["grid"]["grid"] > 0.5, res if ref: stored = json.loads(open(ref).read()) if isinstance(ref, str) else ref probs = [answer_probs(a) for name in imgs for a in rt.predict_questions("", QS, images=[imgs[name]])["answers"].values()] res["vs_ref"] = agreement(probs, stored) if verbose: print(json.dumps(res)) return res def save_ref(path, device=None): from julia2 import Runtime rt = Runtime(REPO, device=device); imgs = make_images() probs = [answer_probs(a) for name in imgs for a in rt.predict_questions("", QS, images=[imgs[name]])["answers"].values()] json.dump(probs, open(path, "w")) if __name__ == "__main__": ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16"); ap.add_argument("--ref"); ap.add_argument("--save_ref") a = ap.parse_args() if a.save_ref: save_ref(a.save_ref, a.device) else: run(a.device, a.precision, a.ref)