File size: 5,404 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
"""Download a task's source at a pinned revision, check it, and write the files Laya trains and is scored on.

The source is pinned by commit and by SHA-256, so a rerun reads the same bytes. The test file this script writes is
locked by its own SHA-256 as well. If anything about how a test question is written changes (the option texts, the
instruction, the order), the hash changes and the script refuses to go on, because scores made before and after the
change would no longer compare.

Run it with `python -m layatasks.data <task>`.
"""

from __future__ import annotations

import argparse
import json
from collections import Counter
from pathlib import Path
from typing import Any, Dict, Iterable, List, Sequence, Tuple

from layatasks import task as tasks
from layatasks.common import data_dir, normalise, sha256_file, write_json, write_jsonl


def overlap(train_texts: Iterable[str], test_texts: Iterable[str]) -> int:
    """How many test texts also appear in training, ignoring case and spacing."""
    seen = {normalise(t) for t in train_texts}
    return sum(1 for t in test_texts if normalise(t) in seen)


def download(task) -> Dict[str, Path]:
    """Each source file at the pinned revision, checked against its SHA-256."""
    from huggingface_hub import hf_hub_download

    paths = {}
    for split, (filename, expected) in task.files.items():
        path = Path(hf_hub_download(task.repo, filename, repo_type="dataset", revision=task.revision))
        got = sha256_file(path)
        if got != expected:
            raise RuntimeError("%s: SHA-256 is %s, expected %s" % (filename, got, expected))
        paths[split] = path
    return paths


def read_parquet(path: Path, label_column: str) -> Tuple[List[Dict[str, Any]], List[str]]:
    """`(records, label names)`. The names come from the file's own metadata, in label-id order."""
    import pyarrow.parquet as pq

    table = pq.read_table(path)
    features = json.loads(table.schema.metadata[b"huggingface"])["info"]["features"]
    return table.to_pylist(), list(features[label_column]["names"])


def check_test_lock(task, path: Path) -> str:
    """Refuse a test file whose hash moved. Returns the hash."""
    got = sha256_file(path)
    if task.test_sha256 is not None and got != task.test_sha256:
        raise RuntimeError("%s is %s, locked at %s. The test changed; earlier scores no longer compare."
                           % (path.name, got, task.test_sha256))
    return got


def label_counts(records: Sequence[Dict[str, Any]], names: Sequence[str], column: str) -> Dict[str, int]:
    counts = Counter(r[column] for r in records)
    return {names[i]: counts.get(i, 0) for i in range(len(names))}


def build(task, out: Path) -> Dict[str, Any]:
    out.mkdir(parents=True, exist_ok=True)
    sources = download(task)
    train_records, names = read_parquet(sources["train"], task.label_column)
    test_records, test_names = read_parquet(sources["test"], task.label_column)
    if names != test_names:
        raise RuntimeError("train and test name their labels differently")
    write_jsonl(out / "train.jsonl", task.rows(train_records, names))
    write_jsonl(out / "test.jsonl", task.rows(test_records, names))
    test_sha = check_test_lock(task, out / "test.jsonl")
    write_json(out / "labels.json", {"names": names, "options": task.options(names)})
    train_counts = label_counts(train_records, names, task.label_column)
    test_counts = label_counts(test_records, names, task.label_column)
    manifest = {
        "task": task.name,
        "source": {"repo": task.repo, "revision": task.revision, "license": task.license,
                   "attribution": task.attribution,
                   "files": {s: {"path": p, "sha256": h} for s, (p, h) in task.files.items()}},
        "question": {"id": task.question_id, "instructions": task.instructions, "options": len(names)},
        "train": {"rows": len(train_records), "per_label_min": min(train_counts.values()),
                  "per_label_max": max(train_counts.values()), "per_label": train_counts},
        "test": {"rows": len(test_records), "sha256": test_sha, "locked": task.test_sha256 is not None,
                 "per_label_min": min(test_counts.values()), "per_label_max": max(test_counts.values()),
                 "per_label": test_counts,
                 "also_in_train": overlap((r[task.text_column] for r in train_records),
                                          (r[task.text_column] for r in test_records))},
    }
    write_json(out / "manifest.json", manifest)
    return manifest


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("task", choices=tasks.TASKS)
    parser.add_argument("--out", type=Path, default=None, help="output folder (default: $LAYA_TASKS_HOME/<task>/data/v1)")
    args = parser.parse_args()
    task = tasks.load(args.task).TASK
    manifest = build(task, args.out or data_dir(task.name))
    short = {k: ({kk: vv for kk, vv in v.items() if kk != "per_label"} if isinstance(v, dict) else v)
             for k, v in manifest.items()}
    print(json.dumps(short, indent=2))
    if not manifest["test"]["locked"]:
        print("test.jsonl is not locked yet: set test_sha256 = %r in layatasks/tasks/%s.py before scoring any model"
              % (manifest["test"]["sha256"], task.name))


if __name__ == "__main__":
    main()