"""Standalone unit tests for research/decision.py (no pytest needed). Run: .venv/bin/python tests/test_decision.py """ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from research.decision import (bucket_for, decide, weighted_tally, accuracy_vs_coverage, bucket_abstention_curve) TABLE = {"HIGH": {"acc": 0.5, "n": 20}, "MEDIUM": {"acc": 0.3, "n": 10}, "LOW": {"acc": 0.1, "n": 40}, "cannot assess": {"acc": None, "n": 0}} def test_unanimous_high_vote(): votes = [{"verdict": "false", "conf": "HIGH"} for _ in range(3)] d = decide(votes, TABLE) assert d["verdict"] == "false" assert abs(d["p"] - 0.5) < 1e-9 assert d["confidence"] == "MEDIUM" # bucket_for(0.5) assert not d["abstained"] def test_unanimous_low_abstains(): votes = [{"verdict": "false", "conf": "LOW"} for _ in range(3)] d = decide(votes, TABLE, threshold=0.4) assert d["abstained"] assert d["verdict"] == "not enough information" def test_conflict_attenuated(): votes = [{"verdict": "false", "conf": "HIGH"}, {"verdict": "true", "conf": "HIGH"}] d = decide(votes, TABLE, threshold=0.3) assert d["abstained"] # p = 0.25 < 0.3 assert abs(d["p"] - 0.25) < 1e-9 def test_majority_outweighs_minority(): votes = [{"verdict": "overclaim", "conf": "HIGH"}] * 3 + \ [{"verdict": "true", "conf": "HIGH"}] d = decide(votes, TABLE) assert d["verdict"] == "overclaim" assert abs(d["p"] - 0.375) < 1e-9 # (3*0.5)/4 def test_unknown_bucket_abstains_above_zero_threshold(): votes = [{"verdict": "false", "conf": "weird"}] d = decide(votes, {}, threshold=0.1, unknown=0.0) assert d["abstained"] assert d["verdict"] == "not enough information" assert d["p"] == 0.0 # threshold 0.0 = selective prediction off: emit verdict, zero confidence d0 = decide(votes, {}, threshold=0.0, unknown=0.0) assert not d0["abstained"] and d0["confidence"] == "cannot assess" def test_weighted_tally(): votes = [{"verdict": "a", "conf": "HIGH"}, {"verdict": "a", "conf": "LOW"}, {"verdict": "b", "conf": "HIGH"}] per = weighted_tally(votes, TABLE) assert abs(per["a"] - 0.6) < 1e-9 assert abs(per["b"] - 0.5) < 1e-9 def test_accuracy_vs_coverage_monotone(): probes = [ {"votes": [{"verdict": "false", "conf": "HIGH"}], "correct": True}, {"votes": [{"verdict": "false", "conf": "HIGH"}], "correct": False}, {"votes": [{"verdict": "true", "conf": "LOW"}], "correct": True}, ] curve = accuracy_vs_coverage(probes, TABLE, thresholds=(0.0, 0.4)) t0, t4 = curve[0], curve[1] assert t0["coverage"] == 1.0 and abs(t0["accuracy"] - 2 / 3) < 1e-9 assert t4["coverage"] == 2 / 3 and t4["accuracy"] == 0.5 # LOW abstained def test_bucket_abstention_curve(): rows = [{"conf": "HIGH", "correct": True}, {"conf": "HIGH", "correct": False}, {"conf": "LOW", "correct": True}] curve = bucket_abstention_curve(rows, TABLE) assert curve[0]["coverage"] == 1.0 and abs(curve[0]["accuracy"] - 2 / 3) < 1e-9 worst = curve[1] # drop LOW (worst bucket) assert worst["coverage"] == 2 / 3 and worst["accuracy"] == 0.5 def test_bucket_for(): assert bucket_for(0.9) == "HIGH" assert bucket_for(0.5) == "MEDIUM" assert bucket_for(0.2) == "LOW" assert bucket_for(0.0) == "cannot assess" if __name__ == "__main__": fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] for fn in fns: fn() print(f"PASS {fn.__name__}") print(f"\n{len(fns)} tests passed")