"""Refitting the gate on dev rows: Laya's own rule, its margin, and where the cut is written.""" from __future__ import annotations import importlib.util import json import os import tempfile import unittest from layatasks import evaluate def preds(rows): return [{"confidence": c, "pred": "a", "gold": "a" if ok else "b"} for c, ok in rows] def falling(n): return [(round(0.99 - i * 1e-4, 4), True) for i in range(n)] @unittest.skipUnless(importlib.util.find_spec("laya"), "laya is not installed") class FitGateTests(unittest.TestCase): def test_margin_needs_about_a_hundred_clean_answers_at_one_percent(self): clean = falling(99) self.assertEqual(evaluate.fit_gate(preds(clean), 0.01), clean[-1][0]) self.assertEqual(evaluate.fit_gate(preds(clean[:98]), 0.01), 1.0) def test_cut_stops_above_the_errors(self): rows = falling(150) + [(0.5, False)] * 20 + [(0.4, True)] * 50 self.assertEqual(evaluate.fit_gate(preds(rows), 0.01), rows[149][0]) def test_written_where_laya_reads_it_with_the_old_cut_kept(self): with tempfile.TemporaryDirectory() as d: with open(os.path.join(d, "rl_agent_config.json"), "w", encoding="utf-8") as f: json.dump({"training": {"abstention_thresholds": {"choice:11+": 1.0}, "epochs_completed": 4}}, f) evaluate.write_gate(d, 0.97, 2000, 0.01) self.assertEqual(evaluate.gate_threshold(d), 0.97) with open(os.path.join(d, "rl_agent_config.json"), encoding="utf-8") as f: training = json.load(f)["training"] self.assertEqual(training["abstention_fit"]["replaces"], {"choice:11+": 1.0}) self.assertEqual((training["abstention_fit"]["n"], training["epochs_completed"]), (2000, 4)) if __name__ == "__main__": unittest.main()