Text Classification
Laya
Safetensors
English
banking77
intent-classification
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-banking77 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-banking77 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download code/banking77/evaluate.py from VibeHash/laya-banking77: direct link, hf CLI and curl.
- Browser
- Download file 6.78 kB
-
https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/evaluate.py
- Command line
-
hf download hf://VibeHash/laya-banking77/code/banking77/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/evaluate.py
6.78 kB
| """Score a Laya checkpoint on the locked Banking77 test set, through the same call a user makes. | |
| Predictions go through `laya.load(...).predict_batch`, so the numbers are the ones a user of the published model | |
| gets, temperature and all, not the numbers of a separate scoring path. | |
| Two coverage numbers are reported and they mean different things: | |
| - `gate`: Laya's own threshold, fitted during the fine-tune on training messages held back from training, then | |
| applied to the test once. This is the claim: decided before the test was read. | |
| - `curve`: the most the model could answer at a given accuracy, with the cut chosen on the test itself. It | |
| describes this test and is optimistic for new data. | |
| Run it with `python -m banking77.evaluate --model <dir> --name <run>` or `--model base` for untuned Laya. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| from collections import Counter | |
| from pathlib import Path | |
| from typing import Any, Dict, List | |
| from banking77 import metrics | |
| from banking77.data import QUESTION_ID, check_test_lock, data_dir, home, read_jsonl | |
| from banking77.train import HEAD_MAX_LEN, base_dir | |
| CURVE_TARGETS = (0.95, 0.98, 0.99) | |
| def gate_threshold(model_dir: str) -> float | None: | |
| """The threshold Laya's fine-tune fitted for questions with 11 or more options, if it fitted one.""" | |
| with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f: | |
| cfg = json.load(f) | |
| gates = (cfg.get("training") or {}).get("abstention_thresholds") or {} | |
| return gates.get("choice:11+", gates.get("default")) | |
| def predict(agent, rows: List[Dict[str, Any]], batch_size: int) -> List[Dict[str, Any]]: | |
| question = rows[0]["questions"] | |
| if any(r["questions"] != question for r in rows): | |
| raise ValueError("every test row must ask the same question") | |
| out = agent.predict_batch([r["state"] for r in rows], question, batch_size=batch_size, | |
| head_max_len=HEAD_MAX_LEN) | |
| preds = [] | |
| for i, (row, res) in enumerate(zip(rows, out)): | |
| ans = res["answers"][QUESTION_ID] | |
| probs = ans["probabilities"] | |
| gold = row["expected"][QUESTION_ID] | |
| preds.append({"i": i, "gold": gold, "pred": ans["choice"], "confidence": max(probs.values()), | |
| "p_gold": probs[gold]}) | |
| return preds | |
| def latency(agent, rows: List[Dict[str, Any]], n: int) -> Dict[str, float]: | |
| """Milliseconds for one message alone, the way a live request would arrive. Median and 95th percentile.""" | |
| question = rows[0]["questions"] | |
| for r in rows[:5]: | |
| agent.predict(r["state"], question, head_max_len=HEAD_MAX_LEN) | |
| times = [] | |
| for r in rows[:n]: | |
| t = time.perf_counter() | |
| agent.predict(r["state"], question, head_max_len=HEAD_MAX_LEN) | |
| times.append((time.perf_counter() - t) * 1000) | |
| times.sort() | |
| return {"n": len(times), "median_ms": times[len(times) // 2], "p95_ms": times[int(0.95 * (len(times) - 1))]} | |
| def summarise(preds: List[Dict[str, Any]], gate: float | None) -> Dict[str, Any]: | |
| correct = [p["pred"] == p["gold"] for p in preds] | |
| conf = [p["confidence"] for p in preds] | |
| n, k = len(preds), sum(correct) | |
| lo, hi = metrics.wilson(k, n) | |
| by_intent: Dict[str, List[int]] = {} | |
| for p, ok in zip(preds, correct): | |
| by_intent.setdefault(p["gold"], [0, 0]) | |
| by_intent[p["gold"]][0] += int(ok) | |
| by_intent[p["gold"]][1] += 1 | |
| per_intent = {g: r / t for g, (r, t) in by_intent.items()} | |
| confusions = Counter((p["gold"], p["pred"]) for p, ok in zip(preds, correct) if not ok) | |
| return { | |
| "n": n, "right": k, "accuracy": k / n, "ci95": [lo, hi], | |
| "mean_confidence": sum(conf) / n, "ece": metrics.ece(conf, correct), | |
| "gate": metrics.selective(conf, correct, gate) if gate is not None else None, | |
| "curve": [metrics.coverage_at_accuracy(conf, correct, t) for t in CURVE_TARGETS], | |
| "weakest_intents": sorted(per_intent.items(), key=lambda kv: kv[1])[:8], | |
| "top_confusions": [{"gold": g, "pred": p, "count": c} for (g, p), c in confusions.most_common(10)], | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya") | |
| parser.add_argument("--name", required=True, help="run name; results go to $LAYA_B77_HOME/runs/<name>/") | |
| parser.add_argument("--batch-size", type=int, default=32) | |
| parser.add_argument("--latency", type=int, default=200, help="messages to time one at a time (0 skips it)") | |
| parser.add_argument("--device", default="cuda") | |
| args = parser.parse_args() | |
| import laya | |
| test_path = data_dir() / "test.jsonl" | |
| test_sha = check_test_lock(test_path) | |
| rows = read_jsonl(test_path) | |
| model_dir = base_dir() if args.model == "base" else args.model | |
| agent = laya.load(model_dir, device=args.device) | |
| gate = gate_threshold(model_dir) | |
| t = time.perf_counter() | |
| preds = predict(agent, rows, args.batch_size) | |
| seconds = time.perf_counter() - t | |
| result = {"name": args.name, "model": str(args.model), "test_sha256": test_sha, | |
| "head_max_len": HEAD_MAX_LEN, "batched": {"batch_size": args.batch_size, "seconds": seconds, | |
| "ms_per_message": seconds * 1000 / len(rows)}, | |
| **summarise(preds, gate)} | |
| if args.latency: | |
| result["single"] = latency(agent, rows, args.latency) | |
| out = home() / "runs" / args.name | |
| out.mkdir(parents=True, exist_ok=True) | |
| with open(out / "eval.json", "w", encoding="utf-8") as f: | |
| json.dump(result, f, indent=2) | |
| with open(out / "predictions.jsonl", "w", encoding="utf-8", newline="\n") as f: | |
| for p in preds: | |
| f.write(json.dumps(p) + "\n") | |
| lo, hi = result["ci95"] | |
| print("%s: %d / %d right = %.2f%% (95%% CI %.2f to %.2f%%), mean confidence %.3f, ECE %.3f" | |
| % (args.name, result["right"], result["n"], 100 * result["accuracy"], 100 * lo, 100 * hi, | |
| result["mean_confidence"], result["ece"])) | |
| if result["gate"]: | |
| g = result["gate"] | |
| print("gate %.4f (fitted in training): answers %.1f%% of messages, %s right" | |
| % (g["threshold"], 100 * g["coverage"], | |
| "n/a" if g["accuracy"] is None else "%.2f%%" % (100 * g["accuracy"]))) | |
| for c in result["curve"]: | |
| if c: | |
| print("curve, cut on the test: %.0f%% accuracy at %.1f%% coverage" % (100 * c["target"], 100 * c["coverage"])) | |
| if "single" in result: | |
| print("one message: median %.1f ms, p95 %.1f ms" % (result["single"]["median_ms"], result["single"]["p95_ms"])) | |
| if __name__ == "__main__": | |
| main() | |