File size: 1,834 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
"""Refitting the gate on dev rows: Laya's own rule, its margin, and where the cut is written."""

from __future__ import annotations

import importlib.util
import json
import os
import tempfile
import unittest

from layatasks import evaluate


def preds(rows):
    return [{"confidence": c, "pred": "a", "gold": "a" if ok else "b"} for c, ok in rows]


def falling(n):
    return [(round(0.99 - i * 1e-4, 4), True) for i in range(n)]


@unittest.skipUnless(importlib.util.find_spec("laya"), "laya is not installed")
class FitGateTests(unittest.TestCase):
    def test_margin_needs_about_a_hundred_clean_answers_at_one_percent(self):
        clean = falling(99)
        self.assertEqual(evaluate.fit_gate(preds(clean), 0.01), clean[-1][0])
        self.assertEqual(evaluate.fit_gate(preds(clean[:98]), 0.01), 1.0)

    def test_cut_stops_above_the_errors(self):
        rows = falling(150) + [(0.5, False)] * 20 + [(0.4, True)] * 50
        self.assertEqual(evaluate.fit_gate(preds(rows), 0.01), rows[149][0])

    def test_written_where_laya_reads_it_with_the_old_cut_kept(self):
        with tempfile.TemporaryDirectory() as d:
            with open(os.path.join(d, "rl_agent_config.json"), "w", encoding="utf-8") as f:
                json.dump({"training": {"abstention_thresholds": {"choice:11+": 1.0}, "epochs_completed": 4}}, f)
            evaluate.write_gate(d, 0.97, 2000, 0.01)
            self.assertEqual(evaluate.gate_threshold(d), 0.97)
            with open(os.path.join(d, "rl_agent_config.json"), encoding="utf-8") as f:
                training = json.load(f)["training"]
            self.assertEqual(training["abstention_fit"]["replaces"], {"choice:11+": 1.0})
            self.assertEqual((training["abstention_fit"]["n"], training["epochs_completed"]), (2000, 4))


if __name__ == "__main__":
    unittest.main()