File size: 7,228 Bytes
4e09985
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
"""Download Banking77 at a pinned revision, check it, and write the files Laya trains and is scored on.

The source is pinned by commit and by SHA-256, so a rerun reads the same bytes. The test file this script writes is
locked by its own SHA-256 as well. If anything about how a test question is written changes (the option texts, the
instruction, the order), the hash changes and the script refuses to go on, because scores made before and after the
change would no longer compare.

Run it with `python -m banking77.data`.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
from collections import Counter
from pathlib import Path
from typing import Any, Dict, Iterable, List, Sequence

REPO = "PolyAI/banking77"
REVISION = "1846984189f2bc0ac51c6fc3a600fa9e1714e0c6"
LICENSE = "CC BY 4.0"
ATTRIBUTION = ("Casanueva et al., Efficient Intent Detection with Dual Sentence Encoders, NLP4ConvAI 2020. "
               "Dataset: PolyAI/banking77 on Hugging Face, CC BY 4.0.")
FILES = {
    "train": ("data/train-00000-of-00001.parquet",
              "8dc89b87a188509c42cce5a086e8f7c1ff4a225bdd9ae3f23e6cdfa5806e363a"),
    "test": ("data/test-00000-of-00001.parquet",
             "1f40f777c33f0edfa6c530ff84fe2878db6490838c12d22d036f626948905341"),
}

QUESTION_ID = "intent"
INSTRUCTIONS = "Which request is this bank customer making?"

# SHA-256 of test.jsonl as this script writes it. Set once, on the first build, before any model was scored.
TEST_SHA256 = "ebc2da4677c6a810d0384e696a427c2177cad289248ed59fe3ef8366e187404b"


def home() -> Path:
    """Where data, models and runs live. Outside the repository, because none of them belong in git."""
    return Path(os.environ.get("LAYA_B77_HOME", "~/.laya-banking77")).expanduser()


def data_dir() -> Path:
    return home() / "data" / "v1"


def sha256_file(path: Path) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for block in iter(lambda: f.read(1 << 20), b""):
            h.update(block)
    return h.hexdigest()


def option_text(name: str) -> str:
    """The option as Laya reads it: the dataset's label name in plain words.

    Laya reads its options as text, so `card_arrival` becomes `card arrival`. One label in the source starts with a
    capital (`Refund_not_showing_up`); lowercasing every name keeps them consistent.
    """
    return name.replace("_", " ").strip().lower()


def question(names: Sequence[str]) -> Dict[str, Any]:
    """The one question every message gets: pick one of the 77 intents, in the dataset's label order."""
    options = [option_text(n) for n in names]
    if len(set(options)) != len(options):
        dupes = [o for o, c in Counter(options).items() if c > 1]
        raise ValueError("two labels read the same once written as text: %r" % (dupes,))
    return {"type": "choice", "instructions": INSTRUCTIONS, "criteria": options}


def rows(records: Iterable[Dict[str, Any]], names: Sequence[str]) -> List[Dict[str, Any]]:
    """Laya's `{state, questions, expected}` rows, one a message, in the source's order."""
    q = question(names)
    return [{"state": r["text"], "questions": {QUESTION_ID: q},
             "expected": {QUESTION_ID: option_text(names[r["label"]])}} for r in records]


def normalise(text: str) -> str:
    return " ".join(text.lower().split())


def overlap(train_texts: Iterable[str], test_texts: Iterable[str]) -> int:
    """How many test messages also appear in training, ignoring case and spacing."""
    seen = {normalise(t) for t in train_texts}
    return sum(1 for t in test_texts if normalise(t) in seen)


def write_jsonl(path: Path, items: Iterable[Dict[str, Any]]) -> None:
    tmp = path.with_suffix(path.suffix + ".tmp")
    with open(tmp, "w", encoding="utf-8", newline="\n") as f:
        for item in items:
            f.write(json.dumps(item, ensure_ascii=False, sort_keys=True) + "\n")
    os.replace(tmp, path)


def read_jsonl(path: Path) -> List[Dict[str, Any]]:
    with open(path, encoding="utf-8") as f:
        return [json.loads(line) for line in f if line.strip()]


def download() -> Dict[str, Path]:
    """The two parquet files at the pinned revision, each checked against its SHA-256."""
    from huggingface_hub import hf_hub_download

    paths = {}
    for split, (filename, expected) in FILES.items():
        path = Path(hf_hub_download(REPO, filename, repo_type="dataset", revision=REVISION))
        got = sha256_file(path)
        if got != expected:
            raise RuntimeError("%s: SHA-256 is %s, expected %s" % (filename, got, expected))
        paths[split] = path
    return paths


def read_parquet(path: Path):
    """`(records, label names)`. The names come from the file's own metadata, in label-id order."""
    import pyarrow.parquet as pq

    table = pq.read_table(path)
    features = json.loads(table.schema.metadata[b"huggingface"])["info"]["features"]
    return table.to_pylist(), list(features["label"]["names"])


def check_test_lock(path: Path) -> str:
    """Refuse a test file whose hash moved. Returns the hash."""
    got = sha256_file(path)
    if TEST_SHA256 is not None and got != TEST_SHA256:
        raise RuntimeError("test.jsonl is %s, locked at %s. The test changed; earlier scores no longer compare."
                           % (got, TEST_SHA256))
    return got


def build(out: Path) -> Dict[str, Any]:
    out.mkdir(parents=True, exist_ok=True)
    sources = download()
    train_records, names = read_parquet(sources["train"])
    test_records, test_names = read_parquet(sources["test"])
    if names != test_names:
        raise RuntimeError("train and test name their labels differently")
    write_jsonl(out / "train.jsonl", rows(train_records, names))
    write_jsonl(out / "test.jsonl", rows(test_records, names))
    test_sha = check_test_lock(out / "test.jsonl")
    labels = {"names": names, "options": [option_text(n) for n in names]}
    with open(out / "labels.json", "w", encoding="utf-8") as f:
        json.dump(labels, f, indent=2)
    manifest = {
        "source": {"repo": REPO, "revision": REVISION, "license": LICENSE, "attribution": ATTRIBUTION,
                   "files": {s: {"path": p, "sha256": h} for s, (p, h) in FILES.items()}},
        "question": {"id": QUESTION_ID, "instructions": INSTRUCTIONS, "options": len(names)},
        "train": {"rows": len(train_records), "per_label_min": min(Counter(r["label"] for r in train_records).values()),
                  "per_label_max": max(Counter(r["label"] for r in train_records).values())},
        "test": {"rows": len(test_records), "sha256": test_sha,
                 "also_in_train": overlap((r["text"] for r in train_records), (r["text"] for r in test_records))},
    }
    with open(out / "manifest.json", "w", encoding="utf-8") as f:
        json.dump(manifest, f, indent=2)
    return manifest


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--out", type=Path, default=None, help="output folder (default: $LAYA_B77_HOME/data/v1)")
    args = parser.parse_args()
    manifest = build(args.out or data_dir())
    print(json.dumps(manifest, indent=2))


if __name__ == "__main__":
    main()