laya-ledgar / code /tests /test_evaluate_gate.py
VibeHash's picture
Release laya-ledgar, run ce-seed0
779ff0b verified
Raw History Blame Contribute Delete
1.83 kB
"""Refitting the gate on dev rows: Laya's own rule, its margin, and where the cut is written."""
from __future__ import annotations
import importlib.util
import json
import os
import tempfile
import unittest
from layatasks import evaluate
def preds(rows):
return [{"confidence": c, "pred": "a", "gold": "a" if ok else "b"} for c, ok in rows]
def falling(n):
return [(round(0.99 - i * 1e-4, 4), True) for i in range(n)]
@unittest.skipUnless(importlib.util.find_spec("laya"), "laya is not installed")
class FitGateTests(unittest.TestCase):
def test_margin_needs_about_a_hundred_clean_answers_at_one_percent(self):
clean = falling(99)
self.assertEqual(evaluate.fit_gate(preds(clean), 0.01), clean[-1][0])
self.assertEqual(evaluate.fit_gate(preds(clean[:98]), 0.01), 1.0)
def test_cut_stops_above_the_errors(self):
rows = falling(150) + [(0.5, False)] * 20 + [(0.4, True)] * 50
self.assertEqual(evaluate.fit_gate(preds(rows), 0.01), rows[149][0])
def test_written_where_laya_reads_it_with_the_old_cut_kept(self):
with tempfile.TemporaryDirectory() as d:
with open(os.path.join(d, "rl_agent_config.json"), "w", encoding="utf-8") as f:
json.dump({"training": {"abstention_thresholds": {"choice:11+": 1.0}, "epochs_completed": 4}}, f)
evaluate.write_gate(d, 0.97, 2000, 0.01)
self.assertEqual(evaluate.gate_threshold(d), 0.97)
with open(os.path.join(d, "rl_agent_config.json"), encoding="utf-8") as f:
training = json.load(f)["training"]
self.assertEqual(training["abstention_fit"]["replaces"], {"choice:11+": 1.0})
self.assertEqual((training["abstention_fit"]["n"], training["epochs_completed"]), (2000, 4))
if __name__ == "__main__":
unittest.main()