Julia-2 / runtime /tests /test_image.py
Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db
Raw History Blame Contribute Delete
3.48 kB
"""Image smoke: a synthetic image (red square on white, text-free) -> the vision tower runs, the answers are sane, the
per-image encoding cache works (2nd request on the same image does not re-encode), and results are identical between
a System One image part and the images= argument. With --ref <json> compares against stored reference probabilities."""
import argparse, json, time
import numpy as np
from tests.common import REPO, agreement, answer_probs
def make_images():
from PIL import Image, ImageDraw
a = Image.new("RGB", (640, 480), "white"); ImageDraw.Draw(a).rectangle([200, 150, 440, 330], fill="red")
b = Image.new("RGB", (512, 512), "black"); d = ImageDraw.Draw(b)
for i in range(0, 512, 64):
d.line([(i, 0), (i, 512)], fill="white", width=4); d.line([(0, i), (512, i)], fill="white", width=4)
return {"red_square": a, "grid": b}
QS = {"color": {"type": "choice", "instructions": "What is the dominant colour of the shape in the image?", "criteria": {"red": "red", "green": "green", "blue": "blue", "none": "no coloured shape"}},
"grid": {"type": "noul", "instructions": "Does the image show a grid of straight lines?"},
"photo": {"type": "noul", "instructions": "Is this a photograph of a real-world scene?"}}
def run(device=None, precision="bf16", ref=None, verbose=True):
from julia2 import Runtime
rt = Runtime(REPO, device=device, precision=precision)
assert rt.vision is not None, "no vision tower in this checkpoint"
imgs = make_images(); res = {"device": str(rt.device), "precision": precision, "answers": {}}
for name, img in imgs.items():
t0 = time.time(); out = rt.predict_questions("", QS, images=[img]); t1 = time.time() - t0
t0 = time.time(); out2 = rt.predict_questions([{"type": "image", "image": img}, {"type": "text", "text": ""}], QS); t2 = time.time() - t0
assert len(rt._img_cache) >= 1 and t2 <= t1 * 1.5 + 0.05, "image cache not used"
p1 = [answer_probs(a) for a in out["answers"].values()]; p2 = [answer_probs(a) for a in out2["answers"].values()]
assert agreement(p1, p2)["maxdp_max"] < 1e-6, "image part vs images= differ"
res["answers"][name] = {k: (a.get("choice") if a["type"] == "choice" else round(a["noul"], 3)) for k, a in out["answers"].items()}
res[name + "_ms_first"] = round(t1 * 1000); res[name + "_ms_cached"] = round(t2 * 1000)
assert res["answers"]["red_square"]["color"] == "red", res
assert res["answers"]["grid"]["grid"] > 0.5, res
if ref:
stored = json.loads(open(ref).read()) if isinstance(ref, str) else ref
probs = [answer_probs(a) for name in imgs for a in rt.predict_questions("", QS, images=[imgs[name]])["answers"].values()]
res["vs_ref"] = agreement(probs, stored)
if verbose:
print(json.dumps(res))
return res
def save_ref(path, device=None):
from julia2 import Runtime
rt = Runtime(REPO, device=device); imgs = make_images()
probs = [answer_probs(a) for name in imgs for a in rt.predict_questions("", QS, images=[imgs[name]])["answers"].values()]
json.dump(probs, open(path, "w"))
if __name__ == "__main__":
ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16"); ap.add_argument("--ref"); ap.add_argument("--save_ref")
a = ap.parse_args()
if a.save_ref:
save_ref(a.save_ref, a.device)
else:
run(a.device, a.precision, a.ref)