File size: 3,681 Bytes
76b78ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | """Standalone unit tests for research/decision.py (no pytest needed).
Run: .venv/bin/python tests/test_decision.py
"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from research.decision import (bucket_for, decide, weighted_tally,
accuracy_vs_coverage, bucket_abstention_curve)
TABLE = {"HIGH": {"acc": 0.5, "n": 20}, "MEDIUM": {"acc": 0.3, "n": 10},
"LOW": {"acc": 0.1, "n": 40}, "cannot assess": {"acc": None, "n": 0}}
def test_unanimous_high_vote():
votes = [{"verdict": "false", "conf": "HIGH"} for _ in range(3)]
d = decide(votes, TABLE)
assert d["verdict"] == "false"
assert abs(d["p"] - 0.5) < 1e-9
assert d["confidence"] == "MEDIUM" # bucket_for(0.5)
assert not d["abstained"]
def test_unanimous_low_abstains():
votes = [{"verdict": "false", "conf": "LOW"} for _ in range(3)]
d = decide(votes, TABLE, threshold=0.4)
assert d["abstained"]
assert d["verdict"] == "not enough information"
def test_conflict_attenuated():
votes = [{"verdict": "false", "conf": "HIGH"},
{"verdict": "true", "conf": "HIGH"}]
d = decide(votes, TABLE, threshold=0.3)
assert d["abstained"] # p = 0.25 < 0.3
assert abs(d["p"] - 0.25) < 1e-9
def test_majority_outweighs_minority():
votes = [{"verdict": "overclaim", "conf": "HIGH"}] * 3 + \
[{"verdict": "true", "conf": "HIGH"}]
d = decide(votes, TABLE)
assert d["verdict"] == "overclaim"
assert abs(d["p"] - 0.375) < 1e-9 # (3*0.5)/4
def test_unknown_bucket_abstains_above_zero_threshold():
votes = [{"verdict": "false", "conf": "weird"}]
d = decide(votes, {}, threshold=0.1, unknown=0.0)
assert d["abstained"]
assert d["verdict"] == "not enough information"
assert d["p"] == 0.0
# threshold 0.0 = selective prediction off: emit verdict, zero confidence
d0 = decide(votes, {}, threshold=0.0, unknown=0.0)
assert not d0["abstained"] and d0["confidence"] == "cannot assess"
def test_weighted_tally():
votes = [{"verdict": "a", "conf": "HIGH"}, {"verdict": "a", "conf": "LOW"},
{"verdict": "b", "conf": "HIGH"}]
per = weighted_tally(votes, TABLE)
assert abs(per["a"] - 0.6) < 1e-9
assert abs(per["b"] - 0.5) < 1e-9
def test_accuracy_vs_coverage_monotone():
probes = [
{"votes": [{"verdict": "false", "conf": "HIGH"}], "correct": True},
{"votes": [{"verdict": "false", "conf": "HIGH"}], "correct": False},
{"votes": [{"verdict": "true", "conf": "LOW"}], "correct": True},
]
curve = accuracy_vs_coverage(probes, TABLE, thresholds=(0.0, 0.4))
t0, t4 = curve[0], curve[1]
assert t0["coverage"] == 1.0 and abs(t0["accuracy"] - 2 / 3) < 1e-9
assert t4["coverage"] == 2 / 3 and t4["accuracy"] == 0.5 # LOW abstained
def test_bucket_abstention_curve():
rows = [{"conf": "HIGH", "correct": True},
{"conf": "HIGH", "correct": False},
{"conf": "LOW", "correct": True}]
curve = bucket_abstention_curve(rows, TABLE)
assert curve[0]["coverage"] == 1.0 and abs(curve[0]["accuracy"] - 2 / 3) < 1e-9
worst = curve[1] # drop LOW (worst bucket)
assert worst["coverage"] == 2 / 3 and worst["accuracy"] == 0.5
def test_bucket_for():
assert bucket_for(0.9) == "HIGH"
assert bucket_for(0.5) == "MEDIUM"
assert bucket_for(0.2) == "LOW"
assert bucket_for(0.0) == "cannot assess"
if __name__ == "__main__":
fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
for fn in fns:
fn()
print(f"PASS {fn.__name__}")
print(f"\n{len(fns)} tests passed")
|