Julia-2-Flash / runtime /tests /common.py
vianahnz's picture
Julia 2 Flash
b61b4c4 verified
Raw History Blame Contribute Delete
1.16 kB
"""Shared fixtures for the runtime tests: repo path, reference sets, agreement metrics."""
import json
import os
import sys
from pathlib import Path
import numpy as np
HERE = Path(__file__).resolve().parent
REPO = Path(os.environ.get("JULIA2_REPO", HERE.parent.parent))
sys.path.insert(0, str(HERE.parent))
def text_ref(n=None):
rows = json.loads((HERE / "text_ref.json").read_text())
return rows[:n] if n else rows
def agreement(probs, refs):
"""-> dict(argmax agreement, max|dp| p50/p95/max)."""
ag = float(np.mean([int(np.argmax(a) == np.argmax(b)) for a, b in zip(probs, refs)]))
dp = [max(abs(x - y) for x, y in zip(a, b)) for a, b in zip(probs, refs)]
return {"argmax_agree": round(ag, 4), "maxdp_p50": round(float(np.median(dp)), 4), "maxdp_p95": round(float(np.percentile(dp, 95)), 4), "maxdp_max": round(float(max(dp)), 4), "n": len(dp)}
def to_rows(ref):
return [{"state": r["state"], "type": r["type"], "question": r["question"], "options": [tuple(o) for o in r["options"]]} for r in ref]
def answer_probs(a):
return [a["noul"], 1 - a["noul"]] if a["type"] == "noul" else list(a["probabilities"].values())