Text Classification
Laya
Safetensors
English
banking77
intent-classification
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-banking77 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-banking77 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download code/banking77/data.py from VibeHash/laya-banking77: direct link, hf CLI and curl.
- Browser
- Download file 7.23 kB
-
https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/data.py
- Command line
-
hf download hf://VibeHash/laya-banking77/code/banking77/data.py
-
curl -L -o data.py https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/data.py
7.23 kB
| """Download Banking77 at a pinned revision, check it, and write the files Laya trains and is scored on. | |
| The source is pinned by commit and by SHA-256, so a rerun reads the same bytes. The test file this script writes is | |
| locked by its own SHA-256 as well. If anything about how a test question is written changes (the option texts, the | |
| instruction, the order), the hash changes and the script refuses to go on, because scores made before and after the | |
| change would no longer compare. | |
| Run it with `python -m banking77.data`. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from collections import Counter | |
| from pathlib import Path | |
| from typing import Any, Dict, Iterable, List, Sequence | |
| REPO = "PolyAI/banking77" | |
| REVISION = "1846984189f2bc0ac51c6fc3a600fa9e1714e0c6" | |
| LICENSE = "CC BY 4.0" | |
| ATTRIBUTION = ("Casanueva et al., Efficient Intent Detection with Dual Sentence Encoders, NLP4ConvAI 2020. " | |
| "Dataset: PolyAI/banking77 on Hugging Face, CC BY 4.0.") | |
| FILES = { | |
| "train": ("data/train-00000-of-00001.parquet", | |
| "8dc89b87a188509c42cce5a086e8f7c1ff4a225bdd9ae3f23e6cdfa5806e363a"), | |
| "test": ("data/test-00000-of-00001.parquet", | |
| "1f40f777c33f0edfa6c530ff84fe2878db6490838c12d22d036f626948905341"), | |
| } | |
| QUESTION_ID = "intent" | |
| INSTRUCTIONS = "Which request is this bank customer making?" | |
| # SHA-256 of test.jsonl as this script writes it. Set once, on the first build, before any model was scored. | |
| TEST_SHA256 = "ebc2da4677c6a810d0384e696a427c2177cad289248ed59fe3ef8366e187404b" | |
| def home() -> Path: | |
| """Where data, models and runs live. Outside the repository, because none of them belong in git.""" | |
| return Path(os.environ.get("LAYA_B77_HOME", "~/.laya-banking77")).expanduser() | |
| def data_dir() -> Path: | |
| return home() / "data" / "v1" | |
| def sha256_file(path: Path) -> str: | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| for block in iter(lambda: f.read(1 << 20), b""): | |
| h.update(block) | |
| return h.hexdigest() | |
| def option_text(name: str) -> str: | |
| """The option as Laya reads it: the dataset's label name in plain words. | |
| Laya reads its options as text, so `card_arrival` becomes `card arrival`. One label in the source starts with a | |
| capital (`Refund_not_showing_up`); lowercasing every name keeps them consistent. | |
| """ | |
| return name.replace("_", " ").strip().lower() | |
| def question(names: Sequence[str]) -> Dict[str, Any]: | |
| """The one question every message gets: pick one of the 77 intents, in the dataset's label order.""" | |
| options = [option_text(n) for n in names] | |
| if len(set(options)) != len(options): | |
| dupes = [o for o, c in Counter(options).items() if c > 1] | |
| raise ValueError("two labels read the same once written as text: %r" % (dupes,)) | |
| return {"type": "choice", "instructions": INSTRUCTIONS, "criteria": options} | |
| def rows(records: Iterable[Dict[str, Any]], names: Sequence[str]) -> List[Dict[str, Any]]: | |
| """Laya's `{state, questions, expected}` rows, one a message, in the source's order.""" | |
| q = question(names) | |
| return [{"state": r["text"], "questions": {QUESTION_ID: q}, | |
| "expected": {QUESTION_ID: option_text(names[r["label"]])}} for r in records] | |
| def normalise(text: str) -> str: | |
| return " ".join(text.lower().split()) | |
| def overlap(train_texts: Iterable[str], test_texts: Iterable[str]) -> int: | |
| """How many test messages also appear in training, ignoring case and spacing.""" | |
| seen = {normalise(t) for t in train_texts} | |
| return sum(1 for t in test_texts if normalise(t) in seen) | |
| def write_jsonl(path: Path, items: Iterable[Dict[str, Any]]) -> None: | |
| tmp = path.with_suffix(path.suffix + ".tmp") | |
| with open(tmp, "w", encoding="utf-8", newline="\n") as f: | |
| for item in items: | |
| f.write(json.dumps(item, ensure_ascii=False, sort_keys=True) + "\n") | |
| os.replace(tmp, path) | |
| def read_jsonl(path: Path) -> List[Dict[str, Any]]: | |
| with open(path, encoding="utf-8") as f: | |
| return [json.loads(line) for line in f if line.strip()] | |
| def download() -> Dict[str, Path]: | |
| """The two parquet files at the pinned revision, each checked against its SHA-256.""" | |
| from huggingface_hub import hf_hub_download | |
| paths = {} | |
| for split, (filename, expected) in FILES.items(): | |
| path = Path(hf_hub_download(REPO, filename, repo_type="dataset", revision=REVISION)) | |
| got = sha256_file(path) | |
| if got != expected: | |
| raise RuntimeError("%s: SHA-256 is %s, expected %s" % (filename, got, expected)) | |
| paths[split] = path | |
| return paths | |
| def read_parquet(path: Path): | |
| """`(records, label names)`. The names come from the file's own metadata, in label-id order.""" | |
| import pyarrow.parquet as pq | |
| table = pq.read_table(path) | |
| features = json.loads(table.schema.metadata[b"huggingface"])["info"]["features"] | |
| return table.to_pylist(), list(features["label"]["names"]) | |
| def check_test_lock(path: Path) -> str: | |
| """Refuse a test file whose hash moved. Returns the hash.""" | |
| got = sha256_file(path) | |
| if TEST_SHA256 is not None and got != TEST_SHA256: | |
| raise RuntimeError("test.jsonl is %s, locked at %s. The test changed; earlier scores no longer compare." | |
| % (got, TEST_SHA256)) | |
| return got | |
| def build(out: Path) -> Dict[str, Any]: | |
| out.mkdir(parents=True, exist_ok=True) | |
| sources = download() | |
| train_records, names = read_parquet(sources["train"]) | |
| test_records, test_names = read_parquet(sources["test"]) | |
| if names != test_names: | |
| raise RuntimeError("train and test name their labels differently") | |
| write_jsonl(out / "train.jsonl", rows(train_records, names)) | |
| write_jsonl(out / "test.jsonl", rows(test_records, names)) | |
| test_sha = check_test_lock(out / "test.jsonl") | |
| labels = {"names": names, "options": [option_text(n) for n in names]} | |
| with open(out / "labels.json", "w", encoding="utf-8") as f: | |
| json.dump(labels, f, indent=2) | |
| manifest = { | |
| "source": {"repo": REPO, "revision": REVISION, "license": LICENSE, "attribution": ATTRIBUTION, | |
| "files": {s: {"path": p, "sha256": h} for s, (p, h) in FILES.items()}}, | |
| "question": {"id": QUESTION_ID, "instructions": INSTRUCTIONS, "options": len(names)}, | |
| "train": {"rows": len(train_records), "per_label_min": min(Counter(r["label"] for r in train_records).values()), | |
| "per_label_max": max(Counter(r["label"] for r in train_records).values())}, | |
| "test": {"rows": len(test_records), "sha256": test_sha, | |
| "also_in_train": overlap((r["text"] for r in train_records), (r["text"] for r in test_records))}, | |
| } | |
| with open(out / "manifest.json", "w", encoding="utf-8") as f: | |
| json.dump(manifest, f, indent=2) | |
| return manifest | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| parser.add_argument("--out", type=Path, default=None, help="output folder (default: $LAYA_B77_HOME/data/v1)") | |
| args = parser.parse_args() | |
| manifest = build(args.out or data_dir()) | |
| print(json.dumps(manifest, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |