File size: 12,207 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
"""Score a Laya checkpoint on a task's locked test set, through the same call a user makes.

Predictions go through `laya.load(...).predict_batch`, so the numbers are the ones a user of the published model
gets, temperature and all, not the numbers of a separate scoring path.

Two coverage numbers are reported and they mean different things:

- `gate`: Laya's own threshold, fitted during the fine-tune on training rows held back from training, then applied
  to the test once. This is the claim: decided before the test was read.
- `curve`: the most the model could answer at a given accuracy, with the cut chosen on the test itself. It
  describes this test and is optimistic for new data.

Accuracy is also split two ways the card has to be honest about: test rows that appear word for word in training
(`seen`) against the rest, and rows whose text Laya cut short (from `layatasks.fit`) against the rest.

Run it with `python -m layatasks.evaluate <task> --model <dir> --name <run>`, or `--model base` for untuned Laya.
`--dev` scores the rows a run kept out of its training (`dev.jsonl` beside the model) instead, and leaves the test
unread. `--dev --fit-gate` also refits the gate on those rows, for when Laya's 400 calibration rows are too few to
fit one: Laya's rule adds one error as a margin, so at a 1% target it needs about 100 answers in a row with no
mistake. The cut is written into the model's config with a record of where it came from, before the test is read.
"""

from __future__ import annotations

import argparse
import json
import os
import time
from collections import Counter
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional

from layatasks import metrics
from layatasks import task as tasks
from layatasks.common import data_dir, normalise, read_json, read_jsonl, run_dir, write_json
from layatasks.data import check_test_lock
from layatasks.train import base_dir

CURVE_TARGETS = (0.95, 0.98, 0.99)


def gate_threshold(model_dir: str) -> Optional[float]:
    """The threshold Laya's fine-tune fitted for questions with 11 or more options, if it fitted one."""
    with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f:
        cfg = json.load(f)
    gates = (cfg.get("training") or {}).get("abstention_thresholds") or {}
    return gates.get("choice:11+", gates.get("default"))


GATE_BUCKET = "choice:11+"


def fit_gate(preds: List[Dict[str, Any]], target_error: float) -> float:
    """Laya's own cut, with its one-error margin, on predictions from rows the run never trained on.

    Confidences are rounded to 4 decimals first, as Laya reports them and as its own fit does.
    """
    from laya.calibrate import _select_abstention_threshold

    pairs = [(round(p["confidence"], 4), int(p["pred"] == p["gold"])) for p in preds]
    return _select_abstention_threshold(pairs, target_error, conservative=True)


def write_gate(model_dir: str, threshold: float, n: int, target_error: float) -> None:
    """Put the refitted cut where Laya's fine-tune put its own, and keep the one it replaces."""
    path = os.path.join(model_dir, "rl_agent_config.json")
    with open(path, encoding="utf-8") as f:
        cfg = json.load(f)
    training = cfg.setdefault("training", {})
    old = training.get("abstention_thresholds") or {}
    key = next(iter(old), GATE_BUCKET)
    training["abstention_thresholds"] = {key: threshold}
    training["abstention_fit"] = {"rows": "dev", "n": n, "target_error": target_error, "conservative": True,
                                  "replaces": old}
    with open(path, "w", encoding="utf-8", newline="\n") as f:
        json.dump(cfg, f, indent=2)
        f.write("\n")


def predict(agent, task, rows: List[Dict[str, Any]], batch_size: int) -> List[Dict[str, Any]]:
    question = rows[0]["questions"]
    if any(r["questions"] != question for r in rows):
        raise ValueError("every test row must ask the same question")
    out = agent.predict_batch([r["state"] for r in rows], question, batch_size=batch_size,
                              max_len=task.max_len, head_max_len=task.head_max_len)
    preds = []
    for i, (row, res) in enumerate(zip(rows, out)):
        ans = res["answers"][task.question_id]
        probs = ans["probabilities"]
        gold = row["expected"][task.question_id]
        preds.append({"i": i, "gold": gold, "pred": ans["choice"], "confidence": max(probs.values()),
                      "p_gold": probs[gold]})
    return preds


def latency(agent, task, rows: List[Dict[str, Any]], n: int) -> Dict[str, float]:
    """Milliseconds for one text alone, the way a live request would arrive. Median and 95th percentile."""
    question = rows[0]["questions"]
    for r in rows[:5]:
        agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len)
    times = []
    for r in rows[:n]:
        t = time.perf_counter()
        agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len)
        times.append((time.perf_counter() - t) * 1000)
    times.sort()
    return {"n": len(times), "median_ms": times[len(times) // 2], "p95_ms": times[int(0.95 * (len(times) - 1))]}


def accuracy_of(preds: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
    preds = list(preds)
    if not preds:
        return {"n": 0, "right": 0, "accuracy": None}
    k = sum(1 for p in preds if p["pred"] == p["gold"])
    return {"n": len(preds), "right": k, "accuracy": k / len(preds)}


def split_by(preds: List[Dict[str, Any]], flags: Iterable[bool], yes: str, no: str) -> Dict[str, Any]:
    flags = list(flags)
    return {yes: accuracy_of(p for p, f in zip(preds, flags) if f),
            no: accuracy_of(p for p, f in zip(preds, flags) if not f)}


def summarise(preds: List[Dict[str, Any]], gate: Optional[float]) -> Dict[str, Any]:
    correct = [p["pred"] == p["gold"] for p in preds]
    conf = [p["confidence"] for p in preds]
    n, k = len(preds), sum(correct)
    lo, hi = metrics.wilson(k, n)
    by_label: Dict[str, List[int]] = {}
    for p, ok in zip(preds, correct):
        by_label.setdefault(p["gold"], [0, 0])
        by_label[p["gold"]][0] += int(ok)
        by_label[p["gold"]][1] += 1
    per_label = {g: {"accuracy": r / t, "n": t} for g, (r, t) in by_label.items()}
    confusions = Counter((p["gold"], p["pred"]) for p, ok in zip(preds, correct) if not ok)
    return {
        "n": n, "right": k, "accuracy": k / n, "ci95": [lo, hi],
        "mean_confidence": sum(conf) / n, "ece": metrics.ece(conf, correct),
        "gate": metrics.selective(conf, correct, gate) if gate is not None else None,
        "curve": [metrics.coverage_at_accuracy(conf, correct, t) for t in CURVE_TARGETS],
        "weakest_labels": sorted(per_label.items(), key=lambda kv: (kv[1]["accuracy"], -kv[1]["n"]))[:10],
        "top_confusions": [{"gold": g, "pred": p, "count": c} for (g, p), c in confusions.most_common(10)],
    }


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("task", choices=tasks.TASKS)
    parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya")
    parser.add_argument("--name", required=True, help="run name; results go to $LAYA_TASKS_HOME/<task>/runs/<name>/")
    parser.add_argument("--batch-size", type=int, default=32)
    parser.add_argument("--latency", type=int, default=200, help="texts to time one at a time (0 skips it)")
    parser.add_argument("--device", default="cuda")
    parser.add_argument("--dev", action="store_true",
                        help="score the run's dev rows (dev.jsonl beside the model) instead of the test")
    parser.add_argument("--fit-gate", action="store_true",
                        help="with --dev: refit the gate on the dev rows and write it into the model's config")
    args = parser.parse_args()
    if args.fit_gate and not args.dev:
        parser.error("--fit-gate needs --dev: a gate fitted on the test would be scored on the rows that chose it")

    import laya

    module = tasks.load(args.task)
    task = module.TASK
    if task.test_sha256 is None:
        raise SystemExit("%s: the test file is not locked yet; run layatasks.data and pin its hash first" % task.name)
    folder = data_dir(task.name)
    model_dir = base_dir() if args.model == "base" else args.model
    if args.dev:
        rows = read_jsonl(Path(model_dir).parent / "dev.jsonl")
        test_sha = None
    else:
        test_sha = check_test_lock(task, folder / "test.jsonl")
        rows = read_jsonl(folder / "test.jsonl")
    agent = laya.load(model_dir, device=args.device)
    gate = gate_threshold(model_dir)

    t = time.perf_counter()
    preds = predict(agent, task, rows, args.batch_size)
    seconds = time.perf_counter() - t
    if args.fit_gate:
        old_gate, gate = gate, fit_gate(preds, task.target_error)
        write_gate(model_dir, gate, len(preds), task.target_error)
        print("gate refitted on %d dev rows at %.0f%% target error: %s -> %.4f (written to the model's config; the"
              " dev numbers below use the rows that chose it, so only the test checks it)"
              % (len(preds), 100 * task.target_error, old_gate, gate))
    result = {"task": task.name, "name": args.name, "model": str(args.model), "test_sha256": test_sha,
              "rows": "dev" if args.dev else "test",
              "max_len": task.max_len, "head_max_len": task.head_max_len,
              "batched": {"batch_size": args.batch_size, "seconds": seconds,
                          "ms_per_text": seconds * 1000 / len(rows)},
              **summarise(preds, gate)}
    if not args.dev:
        seen_texts = {normalise(r["state"]) for r in read_jsonl(folder / "train.jsonl")}
        seen = [normalise(r["state"]) in seen_texts for r in rows]
        result["by_seen"] = split_by(preds, seen, "seen_in_train", "unseen")
    fit_path = folder / "fit.json"
    if fit_path.exists() and not args.dev:
        fit = read_json(fit_path)
        if fit.get("max_len") == task.max_len and fit.get("head_max_len") == task.head_max_len:
            cut = set(fit["test"]["truncated_rows"])
            result["by_truncation"] = split_by(preds, (i in cut for i in range(len(preds))), "cut_short", "whole")
    result["task_scores"] = tasks.extra_scores(module, preds, gate)
    if args.latency and not args.dev:
        result["single"] = latency(agent, task, rows, args.latency)

    out = run_dir(task.name, args.name)
    out.mkdir(parents=True, exist_ok=True)
    suffix = "_dev" if args.dev else ""
    write_json(out / ("eval%s.json" % suffix), result)
    with open(out / ("predictions%s.jsonl" % suffix), "w", encoding="utf-8", newline="\n") as f:
        for p in preds:
            f.write(json.dumps(p) + "\n")

    lo, hi = result["ci95"]
    print("%s/%s%s: %d / %d right = %.2f%% (95%% CI %.2f to %.2f%%), mean confidence %.3f, ECE %.3f"
          % (task.name, args.name, " (dev)" if args.dev else "", result["right"], result["n"],
             100 * result["accuracy"], 100 * lo, 100 * hi,
             result["mean_confidence"], result["ece"]))
    if result["gate"]:
        g = result["gate"]
        print("gate %.4f (fitted in training): answers %.1f%% of texts, %s right"
              % (g["threshold"], 100 * g["coverage"],
                 "n/a" if g["accuracy"] is None else "%.2f%%" % (100 * g["accuracy"])))
    for c in result["curve"]:
        if c:
            print("curve, cut on the test: %.0f%% accuracy at %.1f%% coverage" % (100 * c["target"], 100 * c["coverage"]))
    for key in ("by_seen", "by_truncation"):
        if key in result:
            print("%s: %s" % (key, ", ".join("%s %s of %d" % (k, "n/a" if v["accuracy"] is None else
                                                                 "%.2f%%" % (100 * v["accuracy"]), v["n"])
                                             for k, v in result[key].items())))
    print("task scores: %s" % json.dumps(result["task_scores"]))
    if "single" in result:
        print("one text: median %.1f ms, p95 %.1f ms" % (result["single"]["median_ms"], result["single"]["p95_ms"]))


if __name__ == "__main__":
    main()