laya-banking77 / code /banking77 /offtopic.py
VibeHash's picture
Add laya-banking77: Laya fine-tuned on Banking77, its scores and the code that made it
4e09985 verified
Raw History Blame Contribute Delete
4.46 kB
"""Measure how often messages that belong to none of the 77 intents get past the confidence threshold.
Banking77 has no "none of these" option, so the model always picks one of the 77 intents. The threshold is the only
guard against a message about something else. This script asks the model about the 1,000 test queries that CLINC150
labels out of scope (questions about the stock market, screen brightness, maths and so on) and counts how many it
would have answered by itself.
A few of those queries read like banking questions, so a high-confidence answer is not always wrong. The count is the
number a deployer should plan for, not an exact error rate.
Run it with `python -m banking77.offtopic --model <dir> --name <run>`.
"""
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
from typing import List
from banking77.data import data_dir, home, read_jsonl, sha256_file
from banking77.evaluate import gate_threshold
from banking77.train import HEAD_MAX_LEN, base_dir
REPO = "clinc/clinc_oos"
REVISION = "155b9c710419136e17307b80d0a13e68cd46b4ec"
FILE = "plus/test-00000-of-00001.parquet"
SHA256 = "3e60e45b25bf86543aa5df8ba4fcc674114164e6184f0197690648c2908d0102"
LICENSE = "CC BY 3.0"
ATTRIBUTION = ("Larson et al., An Evaluation Dataset for Intent Classification and Out-of-Scope Prediction, "
"EMNLP-IJCNLP 2019. Dataset: clinc/clinc_oos on Hugging Face, CC BY 3.0.")
def out_of_scope_texts() -> List[str]:
"""The out-of-scope queries of CLINC150's test split, in file order, from the pinned and hashed file."""
import pyarrow.parquet as pq
from huggingface_hub import hf_hub_download
path = Path(hf_hub_download(REPO, FILE, repo_type="dataset", revision=REVISION))
got = sha256_file(path)
if got != SHA256:
raise RuntimeError("%s: SHA-256 is %s, expected %s" % (FILE, got, SHA256))
table = pq.read_table(path)
names = json.loads(table.schema.metadata[b"huggingface"])["info"]["features"]["intent"]["names"]
oos = names.index("oos")
return [r["text"] for r in table.to_pylist() if r["intent"] == oos]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya")
parser.add_argument("--name", required=True, help="run name; results go to $LAYA_B77_HOME/runs/<name>/")
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--device", default="cuda")
args = parser.parse_args()
import laya
question = read_jsonl(data_dir() / "test.jsonl")[0]["questions"]
model_dir = base_dir() if args.model == "base" else args.model
gate = gate_threshold(model_dir)
if gate is None:
raise SystemExit("%s has no fitted threshold, so there is nothing to test" % model_dir)
agent = laya.load(model_dir, device=args.device)
texts = out_of_scope_texts()
out = agent.predict_batch(texts, question, batch_size=args.batch_size, head_max_len=HEAD_MAX_LEN)
conf = [max(r["answers"]["intent"]["probabilities"].values()) for r in out]
picks = [r["answers"]["intent"]["choice"] for r in out]
passed = [(t, p, c) for t, p, c in zip(texts, picks, conf) if c >= gate]
result = {
"name": args.name, "source": {"repo": REPO, "revision": REVISION, "file": FILE, "sha256": SHA256,
"license": LICENSE, "attribution": ATTRIBUTION},
"threshold": gate, "queries": len(texts), "passed": len(passed), "passed_share": len(passed) / len(texts),
"mean_confidence": sum(conf) / len(conf),
"passed_as": Counter(p for _, p, _ in passed).most_common(10),
"passed_examples": [{"text": t, "answered_as": p, "confidence": c}
for t, p, c in sorted(passed, key=lambda x: -x[2])[:15]],
}
dest = home() / "runs" / args.name
dest.mkdir(parents=True, exist_ok=True)
with open(dest / "offtopic.json", "w", encoding="utf-8") as f:
json.dump(result, f, indent=2)
print("%s: %d of %d out-of-scope queries pass the %.4f threshold (%.1f%%); mean confidence %.3f"
% (args.name, len(passed), len(texts), gate, 100 * result["passed_share"], result["mean_confidence"]))
for p, n in result["passed_as"][:5]:
print(" answered as %s: %d" % (p, n))
if __name__ == "__main__":
main()