Spaces:
Running
Running
File size: 4,510 Bytes
37e3d5a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | #!/usr/bin/env python3
"""Tests for the VLM gating layer (Plan 1.3 §3.4). Uses stub samplers — no real model, no token."""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "stage4_review"))
from vlm_gate import aggregate_samples, calibrate, evidence_consistent, gate # noqa: E402
def const_sampler(scores: dict):
return lambda i: dict(scores)
def high_all(claimed="knife"):
return {"objectness": 0.9, "semantic": 0.88, "structural": 0.86, "specular": 0.85, "claimedClass": claimed}
class VlmGateTest(unittest.TestCase):
def test_hard_gate_failure_does_not_consult_vlm(self):
eye = {"verdict": "reject", "action": "refine-code", "hardGateFailures": ["silhouette IoU 0.40 < 0.85"]}
called = {"n": 0}
def sampler(i):
called["n"] += 1
return high_all()
r = gate(eye, sampler)
self.assertFalse(r["ranVlm"])
self.assertEqual(r["verdict"], "reject")
self.assertEqual(called["n"], 0, "VLM must not be called when a hard gate failed")
def test_pass_confirmed_when_all_criteria_high(self):
eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []}
r = gate(eye, const_sampler(high_all()), geometry_class="knife")
self.assertEqual(r["verdict"], "pass")
self.assertEqual(r["action"], "continue")
self.assertTrue(r["ranVlm"])
def test_soft_reject_rescued_by_vlm(self):
# deterministic ensemble was low-confidence (no HARD failure), VLM confirms → rescue
eye = {"verdict": "low-confidence", "action": "probe", "hardGateFailures": []}
r = gate(eye, const_sampler(high_all()), geometry_class="knife")
self.assertEqual(r["verdict"], "pass")
self.assertIn("rescued", r["reason"])
def test_low_objectness_withholds_refine_spec(self):
eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []}
s = high_all(); s["objectness"] = 0.5
r = gate(eye, const_sampler(s), geometry_class="knife")
self.assertEqual(r["verdict"], "withhold")
self.assertEqual(r["action"], "refine-spec")
def test_low_structural_withholds_refine_code(self):
eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []}
s = high_all(); s["structural"] = 0.4
r = gate(eye, const_sampler(s), geometry_class="knife")
self.assertEqual(r["verdict"], "withhold")
self.assertEqual(r["action"], "refine-code")
def test_high_sample_spread_is_uncertain_probe(self):
eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []}
samples = [high_all(), {"objectness": 0.4, "semantic": 0.4, "structural": 0.4, "specular": 0.4, "claimedClass": "knife"}]
r = gate(eye, lambda i: samples[i % len(samples)], n_samples=2, geometry_class="knife")
self.assertEqual(r["verdict"], "uncertain")
self.assertEqual(r["action"], "probe")
def test_evidence_contradiction_is_uncertain(self):
eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []}
r = gate(eye, const_sampler(high_all(claimed="spoon")), geometry_class="knife")
self.assertEqual(r["verdict"], "uncertain")
self.assertEqual(r["action"], "probe")
def test_no_sampler_keeps_deterministic_verdict(self):
eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []}
r = gate(eye, None)
self.assertFalse(r["ranVlm"])
self.assertEqual(r["verdict"], "pass")
def test_calibrate_identity_and_monotonic(self):
self.assertEqual(calibrate(0.7), 0.7) # identity by default
# a map that pulls raw 0.9 down to 0.7 (anti-overconfidence)
cal = [[0.0, 0.0], [0.9, 0.7], [1.0, 0.8]]
self.assertAlmostEqual(calibrate(0.9, cal), 0.7, places=5)
self.assertLess(calibrate(0.9, cal), 0.9)
def test_evidence_consistent_unknown_geometry(self):
self.assertTrue(evidence_consistent("knife", None))
self.assertTrue(evidence_consistent("Knife", "knife"))
self.assertFalse(evidence_consistent("spoon", "knife"))
def test_aggregate_median(self):
agg = aggregate_samples([{"objectness": 0.2}, {"objectness": 0.8}, {"objectness": 0.6}])
self.assertAlmostEqual(agg["criteria"]["objectness"], 0.6, places=5)
if __name__ == "__main__":
unittest.main(verbosity=2)
|