Julia-2-Flash / runtime /tests /test_text.py
vianahnz's picture
Julia 2 Flash
b61b4c4 verified
Raw History Blame Contribute Delete
1.7 kB
"""Text answers == reference (j2 HF path on base_v2, 300 fixed val rows): argmax agreement must be >= 99%, max|dp| reported.
python -m tests.test_text [--device cpu] [--precision int8] [--n 300]"""
import argparse, json, time
from tests.common import REPO, text_ref, agreement, to_rows
def run(device=None, precision="bf16", n=300, verbose=True):
from julia2 import Runtime
rt = Runtime(REPO, device=device, precision=precision)
ref = text_ref(n); rows = to_rows(ref)
t0 = time.time(); probs, _ = rt._probs_text(rows); dt = time.time() - t0
res = agreement(probs, [r["ref"] for r in ref]); res.update({"device": str(rt.device), "precision": precision, "sec": round(dt, 2)})
# shared-state path (multi-question) on the first 5 states: must equal the per-row path
def crit(r):
return dict(r["options"]) if r["type"] == "choice" else ({} if r["type"] == "noul" else [d for _, d in r["options"]])
req = [{"state": r["state"], "questions": {f"q{i}": {"type": r["type"], "instructions": r["question"], "criteria": crit(r)}}} for i, r in enumerate(ref[:5])]
out = rt.predict_batch(req, shared=True)
sh = [[a["noul"], 1 - a["noul"]] if a["type"] == "noul" else list(a["probabilities"].values()) for o in out for a in o["answers"].values()]
res["shared_vs_ref"] = agreement(sh, [r["ref"] for r in ref[:5]])
if verbose:
print(json.dumps(res))
assert res["argmax_agree"] >= 0.99, res
return res
if __name__ == "__main__":
ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16"); ap.add_argument("--n", type=int, default=300)
a = ap.parse_args(); run(a.device, a.precision, a.n)