"""Shared fixtures for the runtime tests: repo path, reference sets, agreement metrics.""" import json import os import sys from pathlib import Path import numpy as np HERE = Path(__file__).resolve().parent REPO = Path(os.environ.get("JULIA2_REPO", HERE.parent.parent)) sys.path.insert(0, str(HERE.parent)) def text_ref(n=None): rows = json.loads((HERE / "text_ref.json").read_text()) return rows[:n] if n else rows def agreement(probs, refs): """-> dict(argmax agreement, max|dp| p50/p95/max).""" ag = float(np.mean([int(np.argmax(a) == np.argmax(b)) for a, b in zip(probs, refs)])) dp = [max(abs(x - y) for x, y in zip(a, b)) for a, b in zip(probs, refs)] return {"argmax_agree": round(ag, 4), "maxdp_p50": round(float(np.median(dp)), 4), "maxdp_p95": round(float(np.percentile(dp, 95)), 4), "maxdp_max": round(float(max(dp)), 4), "n": len(dp)} def to_rows(ref): return [{"state": r["state"], "type": r["type"], "question": r["question"], "options": [tuple(o) for o in r["options"]]} for r in ref] def answer_probs(a): return [a["noul"], 1 - a["noul"]] if a["type"] == "noul" else list(a["probabilities"].values())