File size: 5,597 Bytes
97c39f2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """Standalone unit tests for research/verify_loop.py.
Run: .venv/bin/python tests/test_verify_loop.py
"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from research.verify_loop import plan_checks, run_checks, verify_case
def _verified_sources():
return [
{"source_id": "dot", "url": "https://records.example/dot",
"retrieved_at": "2026-08-12T12:00:00Z", "content_sha256": "a" * 64,
"independent": True, "retrievable": True,
"triage": {"independence": 3, "proximity": 3, "recency": 2, "track": 3, "interest": 3}},
{"source_id": "archive", "url": "https://archive.example/bridge",
"retrieved_at": "2026-08-12T12:01:00Z", "content_sha256": "b" * 64,
"independent": True, "retrievable": True,
"triage": {"independence": 2, "proximity": 2, "recency": 2, "track": 2, "interest": 2}},
]
def test_plan_checks_extracts_values():
checks = plan_checks('Claim: "the bridge opened in 2010" and cost $4.2M at 9:30am.')
kinds = [c["kind"] for c in checks]
assert "quote" in kinds and "number" in kinds and "time" in kinds
assert len(checks) <= 8
def test_run_checks_supports():
checks = [{"kind": "number", "value": "2010"}]
res = run_checks(checks, lambda v: "The DOT file lists the bridge opening year as 2010.")
assert res[0]["verdict"] == "supports"
def test_run_checks_refutes():
checks = [{"kind": "number", "value": "2010"}]
res = run_checks(checks, lambda v: "The DOT file lists the bridge opening year as 2012.")
assert res[0]["verdict"] == "refutes"
def test_run_checks_no_evidence():
checks = [{"kind": "quote", "value": "classified"}]
res = run_checks(checks, lambda v: "")
assert res[0]["verdict"] == "not enough information"
assert res[0]["kind"] == "no-evidence"
def test_verify_case_rule_refuted_wins():
d = verify_case('Claim: "the bridge opened in 2010"',
draft_verdict="true", draft_conf="HIGH",
retrieve=lambda v: "The DOT file lists the bridge opening year as 2012.",
require_source_policy=False)
assert d["verdict"] == "false"
assert d["confidence"] == "HIGH"
assert d["basis"] == "rule-refuted"
def test_verify_case_rule_verified():
d = verify_case('Claim: "the bridge opened in 2010"',
draft_verdict="true", draft_conf="MEDIUM",
retrieve=lambda v: "The DOT file lists the bridge opening year as 2010.",
require_source_policy=False)
assert d["verdict"] == "true"
assert d["confidence"] == "HIGH"
assert d["basis"] == "rule-verified"
def test_verify_case_unresolved_downgrades_high():
d = verify_case("Claim: the memo is significant.",
draft_verdict="overclaim", draft_conf="HIGH",
retrieve=lambda v: "", require_source_policy=False)
assert d["confidence"] == "MEDIUM"
assert d["basis"] == "draft-high-downgraded-unverified"
def test_verify_case_unresolved_keeps_low():
d = verify_case("Claim: the memo is significant.",
draft_verdict="not enough information", draft_conf="LOW",
retrieve=lambda v: "", require_source_policy=False)
assert d["verdict"] == "not enough information"
assert d["confidence"] == "LOW"
assert d["abstained"]
def test_strict_policy_overrides_an_incorrect_model_draft():
d = verify_case('Claim: "the bridge opened in 2010"',
draft_verdict="false", draft_conf="HIGH",
retrieve=lambda v: {
"evidence": "The DOT filing lists the bridge opening year as 2010.",
"sources": _verified_sources(), "claim_relation": "supports"},
require_source_policy=True)
assert d["verdict"] == "true"
assert d["confidence"] == "HIGH"
assert d["basis"] == "source-policy-verified"
assert set(d["sources"]) == {"dot", "archive"}
def test_strict_policy_fails_closed_on_a_single_untraceable_lead():
d = verify_case('Claim: "the bridge opened in 2010"',
draft_verdict="true", draft_conf="HIGH",
retrieve=lambda v: {
"evidence": "An anonymous post says the bridge opened in 2010.",
"sources": [{"source_id": "forum", "url": "https://forum.example/post",
"retrieved_at": "", "content_sha256": "bad",
"independent": True, "retrievable": True,
"triage": {"independence": 1, "proximity": 0,
"recency": 1, "track": 0, "interest": 0}}],
"claim_relation": "supports"},
require_source_policy=True)
assert d["verdict"] == "not enough information"
assert d["confidence"] == "LOW"
assert d["abstained"]
def test_verify_case_defaults_to_fail_closed_source_policy():
d = verify_case('Claim: "the bridge opened in 2010"',
draft_verdict="true", draft_conf="HIGH",
retrieve=lambda v: "A copied page says the bridge opened in 2010.")
assert d["verdict"] == "not enough information"
assert d["confidence"] == "LOW"
assert d["abstained"]
if __name__ == "__main__":
fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
for fn in fns:
fn()
print(f"PASS {fn.__name__}")
print(f"\n{len(fns)} tests passed")
|