Text Classification
Laya
Safetensors
English
ledgar
lexglue
legal
contracts
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-ledgar with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-ledgar with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 12,207 Bytes
779ff0b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 | """Score a Laya checkpoint on a task's locked test set, through the same call a user makes.
Predictions go through `laya.load(...).predict_batch`, so the numbers are the ones a user of the published model
gets, temperature and all, not the numbers of a separate scoring path.
Two coverage numbers are reported and they mean different things:
- `gate`: Laya's own threshold, fitted during the fine-tune on training rows held back from training, then applied
to the test once. This is the claim: decided before the test was read.
- `curve`: the most the model could answer at a given accuracy, with the cut chosen on the test itself. It
describes this test and is optimistic for new data.
Accuracy is also split two ways the card has to be honest about: test rows that appear word for word in training
(`seen`) against the rest, and rows whose text Laya cut short (from `layatasks.fit`) against the rest.
Run it with `python -m layatasks.evaluate <task> --model <dir> --name <run>`, or `--model base` for untuned Laya.
`--dev` scores the rows a run kept out of its training (`dev.jsonl` beside the model) instead, and leaves the test
unread. `--dev --fit-gate` also refits the gate on those rows, for when Laya's 400 calibration rows are too few to
fit one: Laya's rule adds one error as a margin, so at a 1% target it needs about 100 answers in a row with no
mistake. The cut is written into the model's config with a record of where it came from, before the test is read.
"""
from __future__ import annotations
import argparse
import json
import os
import time
from collections import Counter
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
from layatasks import metrics
from layatasks import task as tasks
from layatasks.common import data_dir, normalise, read_json, read_jsonl, run_dir, write_json
from layatasks.data import check_test_lock
from layatasks.train import base_dir
CURVE_TARGETS = (0.95, 0.98, 0.99)
def gate_threshold(model_dir: str) -> Optional[float]:
"""The threshold Laya's fine-tune fitted for questions with 11 or more options, if it fitted one."""
with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f:
cfg = json.load(f)
gates = (cfg.get("training") or {}).get("abstention_thresholds") or {}
return gates.get("choice:11+", gates.get("default"))
GATE_BUCKET = "choice:11+"
def fit_gate(preds: List[Dict[str, Any]], target_error: float) -> float:
"""Laya's own cut, with its one-error margin, on predictions from rows the run never trained on.
Confidences are rounded to 4 decimals first, as Laya reports them and as its own fit does.
"""
from laya.calibrate import _select_abstention_threshold
pairs = [(round(p["confidence"], 4), int(p["pred"] == p["gold"])) for p in preds]
return _select_abstention_threshold(pairs, target_error, conservative=True)
def write_gate(model_dir: str, threshold: float, n: int, target_error: float) -> None:
"""Put the refitted cut where Laya's fine-tune put its own, and keep the one it replaces."""
path = os.path.join(model_dir, "rl_agent_config.json")
with open(path, encoding="utf-8") as f:
cfg = json.load(f)
training = cfg.setdefault("training", {})
old = training.get("abstention_thresholds") or {}
key = next(iter(old), GATE_BUCKET)
training["abstention_thresholds"] = {key: threshold}
training["abstention_fit"] = {"rows": "dev", "n": n, "target_error": target_error, "conservative": True,
"replaces": old}
with open(path, "w", encoding="utf-8", newline="\n") as f:
json.dump(cfg, f, indent=2)
f.write("\n")
def predict(agent, task, rows: List[Dict[str, Any]], batch_size: int) -> List[Dict[str, Any]]:
question = rows[0]["questions"]
if any(r["questions"] != question for r in rows):
raise ValueError("every test row must ask the same question")
out = agent.predict_batch([r["state"] for r in rows], question, batch_size=batch_size,
max_len=task.max_len, head_max_len=task.head_max_len)
preds = []
for i, (row, res) in enumerate(zip(rows, out)):
ans = res["answers"][task.question_id]
probs = ans["probabilities"]
gold = row["expected"][task.question_id]
preds.append({"i": i, "gold": gold, "pred": ans["choice"], "confidence": max(probs.values()),
"p_gold": probs[gold]})
return preds
def latency(agent, task, rows: List[Dict[str, Any]], n: int) -> Dict[str, float]:
"""Milliseconds for one text alone, the way a live request would arrive. Median and 95th percentile."""
question = rows[0]["questions"]
for r in rows[:5]:
agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len)
times = []
for r in rows[:n]:
t = time.perf_counter()
agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len)
times.append((time.perf_counter() - t) * 1000)
times.sort()
return {"n": len(times), "median_ms": times[len(times) // 2], "p95_ms": times[int(0.95 * (len(times) - 1))]}
def accuracy_of(preds: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
preds = list(preds)
if not preds:
return {"n": 0, "right": 0, "accuracy": None}
k = sum(1 for p in preds if p["pred"] == p["gold"])
return {"n": len(preds), "right": k, "accuracy": k / len(preds)}
def split_by(preds: List[Dict[str, Any]], flags: Iterable[bool], yes: str, no: str) -> Dict[str, Any]:
flags = list(flags)
return {yes: accuracy_of(p for p, f in zip(preds, flags) if f),
no: accuracy_of(p for p, f in zip(preds, flags) if not f)}
def summarise(preds: List[Dict[str, Any]], gate: Optional[float]) -> Dict[str, Any]:
correct = [p["pred"] == p["gold"] for p in preds]
conf = [p["confidence"] for p in preds]
n, k = len(preds), sum(correct)
lo, hi = metrics.wilson(k, n)
by_label: Dict[str, List[int]] = {}
for p, ok in zip(preds, correct):
by_label.setdefault(p["gold"], [0, 0])
by_label[p["gold"]][0] += int(ok)
by_label[p["gold"]][1] += 1
per_label = {g: {"accuracy": r / t, "n": t} for g, (r, t) in by_label.items()}
confusions = Counter((p["gold"], p["pred"]) for p, ok in zip(preds, correct) if not ok)
return {
"n": n, "right": k, "accuracy": k / n, "ci95": [lo, hi],
"mean_confidence": sum(conf) / n, "ece": metrics.ece(conf, correct),
"gate": metrics.selective(conf, correct, gate) if gate is not None else None,
"curve": [metrics.coverage_at_accuracy(conf, correct, t) for t in CURVE_TARGETS],
"weakest_labels": sorted(per_label.items(), key=lambda kv: (kv[1]["accuracy"], -kv[1]["n"]))[:10],
"top_confusions": [{"gold": g, "pred": p, "count": c} for (g, p), c in confusions.most_common(10)],
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("task", choices=tasks.TASKS)
parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya")
parser.add_argument("--name", required=True, help="run name; results go to $LAYA_TASKS_HOME/<task>/runs/<name>/")
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--latency", type=int, default=200, help="texts to time one at a time (0 skips it)")
parser.add_argument("--device", default="cuda")
parser.add_argument("--dev", action="store_true",
help="score the run's dev rows (dev.jsonl beside the model) instead of the test")
parser.add_argument("--fit-gate", action="store_true",
help="with --dev: refit the gate on the dev rows and write it into the model's config")
args = parser.parse_args()
if args.fit_gate and not args.dev:
parser.error("--fit-gate needs --dev: a gate fitted on the test would be scored on the rows that chose it")
import laya
module = tasks.load(args.task)
task = module.TASK
if task.test_sha256 is None:
raise SystemExit("%s: the test file is not locked yet; run layatasks.data and pin its hash first" % task.name)
folder = data_dir(task.name)
model_dir = base_dir() if args.model == "base" else args.model
if args.dev:
rows = read_jsonl(Path(model_dir).parent / "dev.jsonl")
test_sha = None
else:
test_sha = check_test_lock(task, folder / "test.jsonl")
rows = read_jsonl(folder / "test.jsonl")
agent = laya.load(model_dir, device=args.device)
gate = gate_threshold(model_dir)
t = time.perf_counter()
preds = predict(agent, task, rows, args.batch_size)
seconds = time.perf_counter() - t
if args.fit_gate:
old_gate, gate = gate, fit_gate(preds, task.target_error)
write_gate(model_dir, gate, len(preds), task.target_error)
print("gate refitted on %d dev rows at %.0f%% target error: %s -> %.4f (written to the model's config; the"
" dev numbers below use the rows that chose it, so only the test checks it)"
% (len(preds), 100 * task.target_error, old_gate, gate))
result = {"task": task.name, "name": args.name, "model": str(args.model), "test_sha256": test_sha,
"rows": "dev" if args.dev else "test",
"max_len": task.max_len, "head_max_len": task.head_max_len,
"batched": {"batch_size": args.batch_size, "seconds": seconds,
"ms_per_text": seconds * 1000 / len(rows)},
**summarise(preds, gate)}
if not args.dev:
seen_texts = {normalise(r["state"]) for r in read_jsonl(folder / "train.jsonl")}
seen = [normalise(r["state"]) in seen_texts for r in rows]
result["by_seen"] = split_by(preds, seen, "seen_in_train", "unseen")
fit_path = folder / "fit.json"
if fit_path.exists() and not args.dev:
fit = read_json(fit_path)
if fit.get("max_len") == task.max_len and fit.get("head_max_len") == task.head_max_len:
cut = set(fit["test"]["truncated_rows"])
result["by_truncation"] = split_by(preds, (i in cut for i in range(len(preds))), "cut_short", "whole")
result["task_scores"] = tasks.extra_scores(module, preds, gate)
if args.latency and not args.dev:
result["single"] = latency(agent, task, rows, args.latency)
out = run_dir(task.name, args.name)
out.mkdir(parents=True, exist_ok=True)
suffix = "_dev" if args.dev else ""
write_json(out / ("eval%s.json" % suffix), result)
with open(out / ("predictions%s.jsonl" % suffix), "w", encoding="utf-8", newline="\n") as f:
for p in preds:
f.write(json.dumps(p) + "\n")
lo, hi = result["ci95"]
print("%s/%s%s: %d / %d right = %.2f%% (95%% CI %.2f to %.2f%%), mean confidence %.3f, ECE %.3f"
% (task.name, args.name, " (dev)" if args.dev else "", result["right"], result["n"],
100 * result["accuracy"], 100 * lo, 100 * hi,
result["mean_confidence"], result["ece"]))
if result["gate"]:
g = result["gate"]
print("gate %.4f (fitted in training): answers %.1f%% of texts, %s right"
% (g["threshold"], 100 * g["coverage"],
"n/a" if g["accuracy"] is None else "%.2f%%" % (100 * g["accuracy"])))
for c in result["curve"]:
if c:
print("curve, cut on the test: %.0f%% accuracy at %.1f%% coverage" % (100 * c["target"], 100 * c["coverage"]))
for key in ("by_seen", "by_truncation"):
if key in result:
print("%s: %s" % (key, ", ".join("%s %s of %d" % (k, "n/a" if v["accuracy"] is None else
"%.2f%%" % (100 * v["accuracy"]), v["n"])
for k, v in result[key].items())))
print("task scores: %s" % json.dumps(result["task_scores"]))
if "single" in result:
print("one text: median %.1f ms, p95 %.1f ms" % (result["single"]["median_ms"], result["single"]["p95_ms"]))
if __name__ == "__main__":
main()
|