File size: 1,251 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
"""LEDGAR's option texts, settings and LexGLUE's two numbers on made-up predictions."""

from __future__ import annotations

import unittest

from layatasks.tasks import ledgar
from layatasks.tasks.ledgar import TASK, extra_scores


class OptionTests(unittest.TestCase):
    def test_label_names_in_lower_case(self):
        self.assertEqual(ledgar.option_text("Anti-Corruption Laws"), "anti-corruption laws")
        self.assertEqual(TASK.question(["Assigns", "Assignments"])["criteria"], ["assigns", "assignments"])

    def test_settings(self):
        self.assertEqual((TASK.max_len, TASK.head_max_len, TASK.epochs, TASK.target_error), (768, 512, 4, 0.01))
        self.assertEqual((TASK.loss, TASK.dev_rows), ("soft-ce", 2000))


class ScoreTests(unittest.TestCase):
    def test_micro_and_macro_f1(self):
        preds = [{"gold": g, "pred": q} for g, q in
                 [("notices", "notices"), ("notices", "notices"), ("waivers", "notices"), ("waivers", "waivers")]]
        s = extra_scores(preds, None)
        self.assertAlmostEqual(s["micro_f1"], 0.75)
        # notices: tp 2, fp 1 -> 4/5.  waivers: tp 1, fn 1 -> 2/3.
        self.assertAlmostEqual(s["macro_f1"], (4 / 5 + 2 / 3) / 2)


if __name__ == "__main__":
    unittest.main()