Text Classification
Laya
Safetensors
English
ledgar
lexglue
legal
contracts
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-ledgar with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-ledgar with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download code/layatasks/evaluate.py from VibeHash/laya-ledgar: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/VibeHash/laya-ledgar/resolve/main/code/layatasks/evaluate.py
- Command line
-
hf download hf://VibeHash/laya-ledgar/code/layatasks/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/VibeHash/laya-ledgar/resolve/main/code/layatasks/evaluate.py
12.2 kB
| """Score a Laya checkpoint on a task's locked test set, through the same call a user makes. | |
| Predictions go through `laya.load(...).predict_batch`, so the numbers are the ones a user of the published model | |
| gets, temperature and all, not the numbers of a separate scoring path. | |
| Two coverage numbers are reported and they mean different things: | |
| - `gate`: Laya's own threshold, fitted during the fine-tune on training rows held back from training, then applied | |
| to the test once. This is the claim: decided before the test was read. | |
| - `curve`: the most the model could answer at a given accuracy, with the cut chosen on the test itself. It | |
| describes this test and is optimistic for new data. | |
| Accuracy is also split two ways the card has to be honest about: test rows that appear word for word in training | |
| (`seen`) against the rest, and rows whose text Laya cut short (from `layatasks.fit`) against the rest. | |
| Run it with `python -m layatasks.evaluate <task> --model <dir> --name <run>`, or `--model base` for untuned Laya. | |
| `--dev` scores the rows a run kept out of its training (`dev.jsonl` beside the model) instead, and leaves the test | |
| unread. `--dev --fit-gate` also refits the gate on those rows, for when Laya's 400 calibration rows are too few to | |
| fit one: Laya's rule adds one error as a margin, so at a 1% target it needs about 100 answers in a row with no | |
| mistake. The cut is written into the model's config with a record of where it came from, before the test is read. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| from collections import Counter | |
| from pathlib import Path | |
| from typing import Any, Dict, Iterable, List, Optional | |
| from layatasks import metrics | |
| from layatasks import task as tasks | |
| from layatasks.common import data_dir, normalise, read_json, read_jsonl, run_dir, write_json | |
| from layatasks.data import check_test_lock | |
| from layatasks.train import base_dir | |
| CURVE_TARGETS = (0.95, 0.98, 0.99) | |
| def gate_threshold(model_dir: str) -> Optional[float]: | |
| """The threshold Laya's fine-tune fitted for questions with 11 or more options, if it fitted one.""" | |
| with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f: | |
| cfg = json.load(f) | |
| gates = (cfg.get("training") or {}).get("abstention_thresholds") or {} | |
| return gates.get("choice:11+", gates.get("default")) | |
| GATE_BUCKET = "choice:11+" | |
| def fit_gate(preds: List[Dict[str, Any]], target_error: float) -> float: | |
| """Laya's own cut, with its one-error margin, on predictions from rows the run never trained on. | |
| Confidences are rounded to 4 decimals first, as Laya reports them and as its own fit does. | |
| """ | |
| from laya.calibrate import _select_abstention_threshold | |
| pairs = [(round(p["confidence"], 4), int(p["pred"] == p["gold"])) for p in preds] | |
| return _select_abstention_threshold(pairs, target_error, conservative=True) | |
| def write_gate(model_dir: str, threshold: float, n: int, target_error: float) -> None: | |
| """Put the refitted cut where Laya's fine-tune put its own, and keep the one it replaces.""" | |
| path = os.path.join(model_dir, "rl_agent_config.json") | |
| with open(path, encoding="utf-8") as f: | |
| cfg = json.load(f) | |
| training = cfg.setdefault("training", {}) | |
| old = training.get("abstention_thresholds") or {} | |
| key = next(iter(old), GATE_BUCKET) | |
| training["abstention_thresholds"] = {key: threshold} | |
| training["abstention_fit"] = {"rows": "dev", "n": n, "target_error": target_error, "conservative": True, | |
| "replaces": old} | |
| with open(path, "w", encoding="utf-8", newline="\n") as f: | |
| json.dump(cfg, f, indent=2) | |
| f.write("\n") | |
| def predict(agent, task, rows: List[Dict[str, Any]], batch_size: int) -> List[Dict[str, Any]]: | |
| question = rows[0]["questions"] | |
| if any(r["questions"] != question for r in rows): | |
| raise ValueError("every test row must ask the same question") | |
| out = agent.predict_batch([r["state"] for r in rows], question, batch_size=batch_size, | |
| max_len=task.max_len, head_max_len=task.head_max_len) | |
| preds = [] | |
| for i, (row, res) in enumerate(zip(rows, out)): | |
| ans = res["answers"][task.question_id] | |
| probs = ans["probabilities"] | |
| gold = row["expected"][task.question_id] | |
| preds.append({"i": i, "gold": gold, "pred": ans["choice"], "confidence": max(probs.values()), | |
| "p_gold": probs[gold]}) | |
| return preds | |
| def latency(agent, task, rows: List[Dict[str, Any]], n: int) -> Dict[str, float]: | |
| """Milliseconds for one text alone, the way a live request would arrive. Median and 95th percentile.""" | |
| question = rows[0]["questions"] | |
| for r in rows[:5]: | |
| agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len) | |
| times = [] | |
| for r in rows[:n]: | |
| t = time.perf_counter() | |
| agent.predict(r["state"], question, max_len=task.max_len, head_max_len=task.head_max_len) | |
| times.append((time.perf_counter() - t) * 1000) | |
| times.sort() | |
| return {"n": len(times), "median_ms": times[len(times) // 2], "p95_ms": times[int(0.95 * (len(times) - 1))]} | |
| def accuracy_of(preds: Iterable[Dict[str, Any]]) -> Dict[str, Any]: | |
| preds = list(preds) | |
| if not preds: | |
| return {"n": 0, "right": 0, "accuracy": None} | |
| k = sum(1 for p in preds if p["pred"] == p["gold"]) | |
| return {"n": len(preds), "right": k, "accuracy": k / len(preds)} | |
| def split_by(preds: List[Dict[str, Any]], flags: Iterable[bool], yes: str, no: str) -> Dict[str, Any]: | |
| flags = list(flags) | |
| return {yes: accuracy_of(p for p, f in zip(preds, flags) if f), | |
| no: accuracy_of(p for p, f in zip(preds, flags) if not f)} | |
| def summarise(preds: List[Dict[str, Any]], gate: Optional[float]) -> Dict[str, Any]: | |
| correct = [p["pred"] == p["gold"] for p in preds] | |
| conf = [p["confidence"] for p in preds] | |
| n, k = len(preds), sum(correct) | |
| lo, hi = metrics.wilson(k, n) | |
| by_label: Dict[str, List[int]] = {} | |
| for p, ok in zip(preds, correct): | |
| by_label.setdefault(p["gold"], [0, 0]) | |
| by_label[p["gold"]][0] += int(ok) | |
| by_label[p["gold"]][1] += 1 | |
| per_label = {g: {"accuracy": r / t, "n": t} for g, (r, t) in by_label.items()} | |
| confusions = Counter((p["gold"], p["pred"]) for p, ok in zip(preds, correct) if not ok) | |
| return { | |
| "n": n, "right": k, "accuracy": k / n, "ci95": [lo, hi], | |
| "mean_confidence": sum(conf) / n, "ece": metrics.ece(conf, correct), | |
| "gate": metrics.selective(conf, correct, gate) if gate is not None else None, | |
| "curve": [metrics.coverage_at_accuracy(conf, correct, t) for t in CURVE_TARGETS], | |
| "weakest_labels": sorted(per_label.items(), key=lambda kv: (kv[1]["accuracy"], -kv[1]["n"]))[:10], | |
| "top_confusions": [{"gold": g, "pred": p, "count": c} for (g, p), c in confusions.most_common(10)], | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| parser.add_argument("task", choices=tasks.TASKS) | |
| parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya") | |
| parser.add_argument("--name", required=True, help="run name; results go to $LAYA_TASKS_HOME/<task>/runs/<name>/") | |
| parser.add_argument("--batch-size", type=int, default=32) | |
| parser.add_argument("--latency", type=int, default=200, help="texts to time one at a time (0 skips it)") | |
| parser.add_argument("--device", default="cuda") | |
| parser.add_argument("--dev", action="store_true", | |
| help="score the run's dev rows (dev.jsonl beside the model) instead of the test") | |
| parser.add_argument("--fit-gate", action="store_true", | |
| help="with --dev: refit the gate on the dev rows and write it into the model's config") | |
| args = parser.parse_args() | |
| if args.fit_gate and not args.dev: | |
| parser.error("--fit-gate needs --dev: a gate fitted on the test would be scored on the rows that chose it") | |
| import laya | |
| module = tasks.load(args.task) | |
| task = module.TASK | |
| if task.test_sha256 is None: | |
| raise SystemExit("%s: the test file is not locked yet; run layatasks.data and pin its hash first" % task.name) | |
| folder = data_dir(task.name) | |
| model_dir = base_dir() if args.model == "base" else args.model | |
| if args.dev: | |
| rows = read_jsonl(Path(model_dir).parent / "dev.jsonl") | |
| test_sha = None | |
| else: | |
| test_sha = check_test_lock(task, folder / "test.jsonl") | |
| rows = read_jsonl(folder / "test.jsonl") | |
| agent = laya.load(model_dir, device=args.device) | |
| gate = gate_threshold(model_dir) | |
| t = time.perf_counter() | |
| preds = predict(agent, task, rows, args.batch_size) | |
| seconds = time.perf_counter() - t | |
| if args.fit_gate: | |
| old_gate, gate = gate, fit_gate(preds, task.target_error) | |
| write_gate(model_dir, gate, len(preds), task.target_error) | |
| print("gate refitted on %d dev rows at %.0f%% target error: %s -> %.4f (written to the model's config; the" | |
| " dev numbers below use the rows that chose it, so only the test checks it)" | |
| % (len(preds), 100 * task.target_error, old_gate, gate)) | |
| result = {"task": task.name, "name": args.name, "model": str(args.model), "test_sha256": test_sha, | |
| "rows": "dev" if args.dev else "test", | |
| "max_len": task.max_len, "head_max_len": task.head_max_len, | |
| "batched": {"batch_size": args.batch_size, "seconds": seconds, | |
| "ms_per_text": seconds * 1000 / len(rows)}, | |
| **summarise(preds, gate)} | |
| if not args.dev: | |
| seen_texts = {normalise(r["state"]) for r in read_jsonl(folder / "train.jsonl")} | |
| seen = [normalise(r["state"]) in seen_texts for r in rows] | |
| result["by_seen"] = split_by(preds, seen, "seen_in_train", "unseen") | |
| fit_path = folder / "fit.json" | |
| if fit_path.exists() and not args.dev: | |
| fit = read_json(fit_path) | |
| if fit.get("max_len") == task.max_len and fit.get("head_max_len") == task.head_max_len: | |
| cut = set(fit["test"]["truncated_rows"]) | |
| result["by_truncation"] = split_by(preds, (i in cut for i in range(len(preds))), "cut_short", "whole") | |
| result["task_scores"] = tasks.extra_scores(module, preds, gate) | |
| if args.latency and not args.dev: | |
| result["single"] = latency(agent, task, rows, args.latency) | |
| out = run_dir(task.name, args.name) | |
| out.mkdir(parents=True, exist_ok=True) | |
| suffix = "_dev" if args.dev else "" | |
| write_json(out / ("eval%s.json" % suffix), result) | |
| with open(out / ("predictions%s.jsonl" % suffix), "w", encoding="utf-8", newline="\n") as f: | |
| for p in preds: | |
| f.write(json.dumps(p) + "\n") | |
| lo, hi = result["ci95"] | |
| print("%s/%s%s: %d / %d right = %.2f%% (95%% CI %.2f to %.2f%%), mean confidence %.3f, ECE %.3f" | |
| % (task.name, args.name, " (dev)" if args.dev else "", result["right"], result["n"], | |
| 100 * result["accuracy"], 100 * lo, 100 * hi, | |
| result["mean_confidence"], result["ece"])) | |
| if result["gate"]: | |
| g = result["gate"] | |
| print("gate %.4f (fitted in training): answers %.1f%% of texts, %s right" | |
| % (g["threshold"], 100 * g["coverage"], | |
| "n/a" if g["accuracy"] is None else "%.2f%%" % (100 * g["accuracy"]))) | |
| for c in result["curve"]: | |
| if c: | |
| print("curve, cut on the test: %.0f%% accuracy at %.1f%% coverage" % (100 * c["target"], 100 * c["coverage"])) | |
| for key in ("by_seen", "by_truncation"): | |
| if key in result: | |
| print("%s: %s" % (key, ", ".join("%s %s of %d" % (k, "n/a" if v["accuracy"] is None else | |
| "%.2f%%" % (100 * v["accuracy"]), v["n"]) | |
| for k, v in result[key].items()))) | |
| print("task scores: %s" % json.dumps(result["task_scores"])) | |
| if "single" in result: | |
| print("one text: median %.1f ms, p95 %.1f ms" % (result["single"]["median_ms"], result["single"]["p95_ms"])) | |
| if __name__ == "__main__": | |
| main() | |