"""Text answers == reference (j2 HF path on base_v2, 300 fixed val rows): argmax agreement must be >= 99%, max|dp| reported. python -m tests.test_text [--device cpu] [--precision int8] [--n 300]""" import argparse, json, time from tests.common import REPO, text_ref, agreement, to_rows def run(device=None, precision="bf16", n=300, verbose=True): from julia2 import Runtime rt = Runtime(REPO, device=device, precision=precision) ref = text_ref(n); rows = to_rows(ref) t0 = time.time(); probs, _ = rt._probs_text(rows); dt = time.time() - t0 res = agreement(probs, [r["ref"] for r in ref]); res.update({"device": str(rt.device), "precision": precision, "sec": round(dt, 2)}) # shared-state path (multi-question) on the first 5 states: must equal the per-row path def crit(r): return dict(r["options"]) if r["type"] == "choice" else ({} if r["type"] == "noul" else [d for _, d in r["options"]]) req = [{"state": r["state"], "questions": {f"q{i}": {"type": r["type"], "instructions": r["question"], "criteria": crit(r)}}} for i, r in enumerate(ref[:5])] out = rt.predict_batch(req, shared=True) sh = [[a["noul"], 1 - a["noul"]] if a["type"] == "noul" else list(a["probabilities"].values()) for o in out for a in o["answers"].values()] res["shared_vs_ref"] = agreement(sh, [r["ref"] for r in ref[:5]]) if verbose: print(json.dumps(res)) assert res["argmax_agree"] >= 0.99, res return res if __name__ == "__main__": ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16"); ap.add_argument("--n", type=int, default=300) a = ap.parse_args(); run(a.device, a.precision, a.n)