"""Score a Laya checkpoint on a task's locked test set, through the same call a user makes. Predictions go through `laya.load(...).predict_batch`, so the numbers are the ones a user of the published model gets, temperature and all, not the numbers of a separate scoring path. Two coverage numbers are reported and they mean different things: - `gate`: Laya's own threshold, fitted during the fine-tune on training rows held back from training, then applied to the test once. This is the claim: decided before the test was read. - `curve`: the most the model could answer at a given accuracy, with the cut chosen on the test itself. It describes this test and is optimistic for new data. Accuracy is also split two ways the card has to be honest about: test rows that appear word for word in training (`seen`) against the rest, and rows whose text Laya cut short (from `layatasks.fit`) against the rest. Run it with `python -m layatasks.evaluate --model --name `, or `--model base` for untuned Laya. `--dev` scores the rows a run kept out of its training (`dev.jsonl` beside the model) instead, and leaves the test unread. `--dev --fit-gate` also refits the gate on those rows, for when Laya's 400 calibration rows are too few to fit one: Laya's rule adds one error as a margin, so at a 1% target it needs about 100 answers in a row with no mistake. The cut is written into the model's config with a record of where it came from, before the test is read. """ from __future__ import annotations import argparse import json import os import time from collections import Counter from pathlib import Path from typing import Any, Dict, Iterable, List, Optional from layatasks import metrics from layatasks import task as tasks from layatasks.common import data_dir, normalise, read_json, read_jsonl, run_dir, write_json from layatasks.data import check_test_lock from layatasks.train import base_dir CURVE_TARGETS = (0.95, 0.98, 0.99) def gate_threshold(model_dir: str) -> Optional[float]: """The threshold Laya's fine-tune fitted for questions with 11 or more options, if it fitted one.""" with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f: cfg = json.load(f) gates = (cfg.get("training") or {}).get("abstention_thresholds") or {} return gates.get("choice:11+", gates.get("default")) GATE_BUCKET = "choice:11+" def fit_gate(preds: List[Dict[str, Any]], target_error: float) -> float: """Laya's own cut, with its one-error margin, on predictions from rows the run never trained on. Confidences are rounded to 4 decimals first, as Laya reports them and as its own fit does. """ from laya.calibrate import _select_abstention_threshold pairs = [(round(p["confidence"], 4), int(p["pred"] == p["gold"])) for p in preds] return _select_abstention_threshold(pairs, target_error, conservative=True) def write_gate(model_dir: str, threshold: float, n: int, target_error: float) -> None: """Put the refitted cut where Laya's fine-tune put its own, and keep the one it replaces.""" path = os.path.join(model_dir, "rl_agent_config.json") with open(path, encoding="utf-8") as f: cfg = json.load(f) training = cfg.setdefault("training", {}) old = training.get("abstention_thresholds") or {} key = next(iter(old), GATE_BUCKET) training["abstention_thresholds"] = {key: threshold} training["abstention_fit"] = {"rows": "dev", "n": n, "target_error": target_error, "conservative": True, "replaces": old} with open(path, "w", encoding="utf-8", newline="\n") as f: json.dump(cfg, f, indent=2) f.write("\n") def predict(agent, task, rows: List[Dict[str, Any]], batch_size: int) -> List[Dict[str, Any]]: question = rows[0]["questions"] if any(r["questions"] != question for r in rows): raise ValueError("every test row must ask the same question") out = agent.predict_batch([r["state"] for r in rows], question, batch_size=batch_size, max_len=task.max_len, head_max_len=task.head_max_len) preds = [] for i, (row, res) in enumerate(zip(rows, out)): ans = res["answers"][task.question_id] probs = ans["probabilities"] gold = row["expected"][task.question_id] preds.append({"i": i, "gold": gold, "pred": ans["choice"], "confidence": max(probs.values()), "p_gold": probs[gold]}) return preds def latency(agent, task, rows: List[Dict[str, Any]], n: int) -> Dict[str, float]: """Milliseconds for one text alone, the way a live request would arrive. Median and 95th percentile.""" question = rows[0]["questions"] for r in rows[:5]: agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len) times = [] for r in rows[:n]: t = time.perf_counter() agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len) times.append((time.perf_counter() - t) * 1000) times.sort() return {"n": len(times), "median_ms": times[len(times) // 2], "p95_ms": times[int(0.95 * (len(times) - 1))]} def accuracy_of(preds: Iterable[Dict[str, Any]]) -> Dict[str, Any]: preds = list(preds) if not preds: return {"n": 0, "right": 0, "accuracy": None} k = sum(1 for p in preds if p["pred"] == p["gold"]) return {"n": len(preds), "right": k, "accuracy": k / len(preds)} def split_by(preds: List[Dict[str, Any]], flags: Iterable[bool], yes: str, no: str) -> Dict[str, Any]: flags = list(flags) return {yes: accuracy_of(p for p, f in zip(preds, flags) if f), no: accuracy_of(p for p, f in zip(preds, flags) if not f)} def summarise(preds: List[Dict[str, Any]], gate: Optional[float]) -> Dict[str, Any]: correct = [p["pred"] == p["gold"] for p in preds] conf = [p["confidence"] for p in preds] n, k = len(preds), sum(correct) lo, hi = metrics.wilson(k, n) by_label: Dict[str, List[int]] = {} for p, ok in zip(preds, correct): by_label.setdefault(p["gold"], [0, 0]) by_label[p["gold"]][0] += int(ok) by_label[p["gold"]][1] += 1 per_label = {g: {"accuracy": r / t, "n": t} for g, (r, t) in by_label.items()} confusions = Counter((p["gold"], p["pred"]) for p, ok in zip(preds, correct) if not ok) return { "n": n, "right": k, "accuracy": k / n, "ci95": [lo, hi], "mean_confidence": sum(conf) / n, "ece": metrics.ece(conf, correct), "gate": metrics.selective(conf, correct, gate) if gate is not None else None, "curve": [metrics.coverage_at_accuracy(conf, correct, t) for t in CURVE_TARGETS], "weakest_labels": sorted(per_label.items(), key=lambda kv: (kv[1]["accuracy"], -kv[1]["n"]))[:10], "top_confusions": [{"gold": g, "pred": p, "count": c} for (g, p), c in confusions.most_common(10)], } def main() -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("task", choices=tasks.TASKS) parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya") parser.add_argument("--name", required=True, help="run name; results go to $LAYA_TASKS_HOME//runs//") parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--latency", type=int, default=200, help="texts to time one at a time (0 skips it)") parser.add_argument("--device", default="cuda") parser.add_argument("--dev", action="store_true", help="score the run's dev rows (dev.jsonl beside the model) instead of the test") parser.add_argument("--fit-gate", action="store_true", help="with --dev: refit the gate on the dev rows and write it into the model's config") args = parser.parse_args() if args.fit_gate and not args.dev: parser.error("--fit-gate needs --dev: a gate fitted on the test would be scored on the rows that chose it") import laya module = tasks.load(args.task) task = module.TASK if task.test_sha256 is None: raise SystemExit("%s: the test file is not locked yet; run layatasks.data and pin its hash first" % task.name) folder = data_dir(task.name) model_dir = base_dir() if args.model == "base" else args.model if args.dev: rows = read_jsonl(Path(model_dir).parent / "dev.jsonl") test_sha = None else: test_sha = check_test_lock(task, folder / "test.jsonl") rows = read_jsonl(folder / "test.jsonl") agent = laya.load(model_dir, device=args.device) gate = gate_threshold(model_dir) t = time.perf_counter() preds = predict(agent, task, rows, args.batch_size) seconds = time.perf_counter() - t if args.fit_gate: old_gate, gate = gate, fit_gate(preds, task.target_error) write_gate(model_dir, gate, len(preds), task.target_error) print("gate refitted on %d dev rows at %.0f%% target error: %s -> %.4f (written to the model's config; the" " dev numbers below use the rows that chose it, so only the test checks it)" % (len(preds), 100 * task.target_error, old_gate, gate)) result = {"task": task.name, "name": args.name, "model": str(args.model), "test_sha256": test_sha, "rows": "dev" if args.dev else "test", "max_len": task.max_len, "head_max_len": task.head_max_len, "batched": {"batch_size": args.batch_size, "seconds": seconds, "ms_per_text": seconds * 1000 / len(rows)}, **summarise(preds, gate)} if not args.dev: seen_texts = {normalise(r["state"]) for r in read_jsonl(folder / "train.jsonl")} seen = [normalise(r["state"]) in seen_texts for r in rows] result["by_seen"] = split_by(preds, seen, "seen_in_train", "unseen") fit_path = folder / "fit.json" if fit_path.exists() and not args.dev: fit = read_json(fit_path) if fit.get("max_len") == task.max_len and fit.get("head_max_len") == task.head_max_len: cut = set(fit["test"]["truncated_rows"]) result["by_truncation"] = split_by(preds, (i in cut for i in range(len(preds))), "cut_short", "whole") result["task_scores"] = tasks.extra_scores(module, preds, gate) if args.latency and not args.dev: result["single"] = latency(agent, task, rows, args.latency) out = run_dir(task.name, args.name) out.mkdir(parents=True, exist_ok=True) suffix = "_dev" if args.dev else "" write_json(out / ("eval%s.json" % suffix), result) with open(out / ("predictions%s.jsonl" % suffix), "w", encoding="utf-8", newline="\n") as f: for p in preds: f.write(json.dumps(p) + "\n") lo, hi = result["ci95"] print("%s/%s%s: %d / %d right = %.2f%% (95%% CI %.2f to %.2f%%), mean confidence %.3f, ECE %.3f" % (task.name, args.name, " (dev)" if args.dev else "", result["right"], result["n"], 100 * result["accuracy"], 100 * lo, 100 * hi, result["mean_confidence"], result["ece"])) if result["gate"]: g = result["gate"] print("gate %.4f (fitted in training): answers %.1f%% of texts, %s right" % (g["threshold"], 100 * g["coverage"], "n/a" if g["accuracy"] is None else "%.2f%%" % (100 * g["accuracy"]))) for c in result["curve"]: if c: print("curve, cut on the test: %.0f%% accuracy at %.1f%% coverage" % (100 * c["target"], 100 * c["coverage"])) for key in ("by_seen", "by_truncation"): if key in result: print("%s: %s" % (key, ", ".join("%s %s of %d" % (k, "n/a" if v["accuracy"] is None else "%.2f%%" % (100 * v["accuracy"]), v["n"]) for k, v in result[key].items()))) print("task scores: %s" % json.dumps(result["task_scores"])) if "single" in result: print("one text: median %.1f ms, p95 %.1f ms" % (result["single"]["median_ms"], result["single"]["p95_ms"])) if __name__ == "__main__": main()