Text Classification
Laya
Safetensors
English
ledgar
lexglue
legal
contracts
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-ledgar with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-ledgar with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 5,416 Bytes
779ff0b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | """Scores for a model that states how sure it is: accuracy with its uncertainty, calibration, and coverage.
Plain Python on purpose. These functions decide what the README claims, so they should be easy to read and to
check by hand, and they should not depend on the training stack.
"""
from __future__ import annotations
import math
from typing import Dict, List, Optional, Sequence, Tuple
Z95 = 1.959963984540054
def wilson(k: int, n: int, z: float = Z95) -> Tuple[float, float]:
"""The Wilson 95% interval for `k` right out of `n`.
It stays inside [0, 1] and behaves near 0% and 100%, where the textbook normal interval does not.
"""
if n <= 0:
raise ValueError("n must be positive")
if not 0 <= k <= n:
raise ValueError("k must be between 0 and n")
p = k / n
denom = 1 + z * z / n
centre = (p + z * z / (2 * n)) / denom
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
return max(0.0, centre - half), min(1.0, centre + half)
def ece(confidences: Sequence[float], correct: Sequence[bool], bins: int = 15) -> float:
"""Expected calibration error: the gap between stated confidence and accuracy, averaged over equal-width bins.
A model that says 0.9 and is right 90% of the time scores 0. Confidence 1.0 falls in the last bin.
"""
if len(confidences) != len(correct):
raise ValueError("confidences and correct differ in length")
n = len(confidences)
if n == 0:
raise ValueError("no predictions")
sums: List[List[float]] = [[0, 0.0, 0.0] for _ in range(bins)]
for c, ok in zip(confidences, correct):
b = min(int(c * bins), bins - 1)
sums[b][0] += 1
sums[b][1] += c
sums[b][2] += 1.0 if ok else 0.0
return sum(abs(s[1] - s[2]) for s in sums if s[0]) / n
def selective(confidences: Sequence[float], correct: Sequence[bool], threshold: float) -> Dict[str, float]:
"""What happens when the model answers only at or above `threshold` and hands the rest to a person."""
kept = [ok for c, ok in zip(confidences, correct) if c >= threshold]
n = len(confidences)
return {"threshold": threshold, "answered": len(kept), "coverage": len(kept) / n,
"accuracy": (sum(kept) / len(kept)) if kept else None,
"errors": len(kept) - sum(kept)}
def coverage_at_accuracy(confidences: Sequence[float], correct: Sequence[bool],
target: float) -> Optional[Dict[str, float]]:
"""The most messages the model could answer, most confident first, while staying at `target` accuracy or above.
The cut is chosen on the same predictions it is scored on, so it describes this data and is optimistic for new
data. Use it to draw the coverage curve; use `selective` with a threshold fixed beforehand to make a claim.
A cut may only fall between two different confidences: messages with equal confidence are kept or dropped
together, as a real threshold would. Returns None when no cut reaches the target.
"""
order = sorted(range(len(confidences)), key=lambda i: -confidences[i])
best = None
right = 0
for pos, i in enumerate(order):
right += 1 if correct[i] else 0
last_of_tie = pos + 1 == len(order) or confidences[order[pos + 1]] < confidences[i]
if last_of_tie and right / (pos + 1) >= target:
best = {"target": target, "threshold": confidences[i], "answered": pos + 1,
"coverage": (pos + 1) / len(order), "accuracy": right / (pos + 1)}
return best
def mcnemar_exact(only_a: int, only_b: int) -> float:
"""Two-sided exact McNemar p-value from the two kinds of disagreement between paired models."""
n = only_a + only_b
if n == 0:
return 1.0
k = min(only_a, only_b)
tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
return min(1.0, 2 * tail)
def micro_f1(golds: Sequence[str], preds: Sequence[str]) -> float:
"""Micro-averaged F1 over single-label predictions. Every wrong answer is one false positive and one false
negative, so it equals accuracy; it is reported under this name because LexGLUE reports it under this name."""
if len(golds) != len(preds) or not golds:
raise ValueError("golds and preds must be non-empty and equal in length")
return sum(1 for g, p in zip(golds, preds) if g == p) / len(golds)
def macro_f1(golds: Sequence[str], preds: Sequence[str]) -> float:
"""F1 of each label, averaged with equal weight, over every label that is either a gold label or a prediction.
That label set is what scikit-learn's `f1_score(average="macro")` uses when it is not told the labels, which is
how LexGLUE scores. A label that is predicted but never right counts as an F1 of 0.
"""
if len(golds) != len(preds) or not golds:
raise ValueError("golds and preds must be non-empty and equal in length")
tp: Dict[str, int] = {}
fp: Dict[str, int] = {}
fn: Dict[str, int] = {}
for g, p in zip(golds, preds):
if g == p:
tp[g] = tp.get(g, 0) + 1
else:
fp[p] = fp.get(p, 0) + 1
fn[g] = fn.get(g, 0) + 1
labels = set(golds) | set(preds)
total = 0.0
for label in labels:
t, f_p, f_n = tp.get(label, 0), fp.get(label, 0), fn.get(label, 0)
total += 2 * t / (2 * t + f_p + f_n) if t else 0.0
return total / len(labels)
|