Text Classification
Laya
Safetensors
English
ledgar
lexglue
legal
contracts
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-ledgar with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-ledgar with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 4,903 Bytes
779ff0b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """The scores the README reports, checked against values worked out by hand."""
from __future__ import annotations
import unittest
from layatasks import metrics
class WilsonTests(unittest.TestCase):
def test_half_of_ten(self):
lo, hi = metrics.wilson(5, 10)
self.assertAlmostEqual(lo, 0.2366, places=4)
self.assertAlmostEqual(hi, 0.7634, places=4)
def test_stays_inside_zero_and_one(self):
self.assertEqual(metrics.wilson(0, 20)[0], 0.0)
self.assertEqual(metrics.wilson(20, 20)[1], 1.0)
self.assertGreater(metrics.wilson(20, 20)[0], 0.8)
def test_refuses_bad_counts(self):
with self.assertRaises(ValueError):
metrics.wilson(3, 0)
with self.assertRaises(ValueError):
metrics.wilson(5, 4)
class EceTests(unittest.TestCase):
def test_honest_confidence_scores_zero(self):
conf = [0.75] * 4
correct = [True, True, True, False]
self.assertAlmostEqual(metrics.ece(conf, correct), 0.0)
def test_overconfidence_shows_as_the_gap(self):
conf = [0.9] * 10
correct = [True] * 6 + [False] * 4
self.assertAlmostEqual(metrics.ece(conf, correct), 0.3)
def test_confidence_one_falls_in_the_last_bin(self):
self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0)
class SelectiveTests(unittest.TestCase):
def test_threshold_keeps_only_confident_answers(self):
conf = [0.99, 0.95, 0.6, 0.4]
correct = [True, True, False, True]
s = metrics.selective(conf, correct, 0.9)
self.assertEqual(s["answered"], 2)
self.assertEqual(s["coverage"], 0.5)
self.assertEqual(s["accuracy"], 1.0)
self.assertEqual(s["errors"], 0)
def test_nothing_answered_has_no_accuracy(self):
self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"])
class CoverageTests(unittest.TestCase):
def test_largest_prefix_at_target(self):
conf = [0.99, 0.98, 0.97, 0.5, 0.4]
correct = [True, True, True, False, True]
c = metrics.coverage_at_accuracy(conf, correct, 0.95)
self.assertEqual(c["answered"], 3)
self.assertEqual(c["threshold"], 0.97)
def test_a_later_prefix_can_recover(self):
conf = [0.9, 0.8, 0.7, 0.6, 0.5]
correct = [True, False, True, True, True]
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5)
def test_ties_are_kept_or_dropped_together(self):
conf = [0.9, 0.9, 0.9, 0.5]
correct = [True, True, False, True]
self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95))
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4)
def test_none_when_unreachable(self):
self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5))
class McNemarTests(unittest.TestCase):
def test_no_disagreement(self):
self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0)
def test_lopsided_disagreement(self):
self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024)
def test_symmetric(self):
self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3))
class CompareTests(unittest.TestCase):
def test_counts_only_the_disagreements(self):
from layatasks.compare import compare
a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}]
b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}]
r = compare(a, b)
self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1))
def test_refuses_runs_on_different_tests(self):
from layatasks.compare import compare
with self.assertRaises(ValueError):
compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}])
class F1Tests(unittest.TestCase):
def test_micro_f1_is_accuracy_for_one_label_each(self):
self.assertAlmostEqual(metrics.micro_f1(["a", "b", "b", "c"], ["a", "b", "c", "c"]), 0.75)
def test_macro_f1_by_hand(self):
# a: tp 1, fp 0, fn 0 -> 1. b: tp 1, fp 0, fn 1 -> 2/3. c: tp 1, fp 1, fn 0 -> 2/3.
golds = ["a", "b", "b", "c"]
preds = ["a", "b", "c", "c"]
self.assertAlmostEqual(metrics.macro_f1(golds, preds), (1 + 2 / 3 + 2 / 3) / 3)
def test_a_label_only_ever_predicted_counts_as_zero(self):
# a: tp 1, fn 1 -> 2/3. z: never gold, predicted once and wrong -> 0.
self.assertAlmostEqual(metrics.macro_f1(["a", "a"], ["a", "z"]), (2 / 3 + 0) / 2)
def test_refuses_mismatched_lengths(self):
with self.assertRaises(ValueError):
metrics.macro_f1(["a"], ["a", "b"])
with self.assertRaises(ValueError):
metrics.micro_f1([], [])
if __name__ == "__main__":
unittest.main()
|