Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """Tests for the VLM gating layer (Plan 1.3 §3.4). Uses stub samplers — no real model, no token.""" | |
| from __future__ import annotations | |
| import sys | |
| import unittest | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(ROOT / "stage4_review")) | |
| from vlm_gate import aggregate_samples, calibrate, evidence_consistent, gate # noqa: E402 | |
| def const_sampler(scores: dict): | |
| return lambda i: dict(scores) | |
| def high_all(claimed="knife"): | |
| return {"objectness": 0.9, "semantic": 0.88, "structural": 0.86, "specular": 0.85, "claimedClass": claimed} | |
| class VlmGateTest(unittest.TestCase): | |
| def test_hard_gate_failure_does_not_consult_vlm(self): | |
| eye = {"verdict": "reject", "action": "refine-code", "hardGateFailures": ["silhouette IoU 0.40 < 0.85"]} | |
| called = {"n": 0} | |
| def sampler(i): | |
| called["n"] += 1 | |
| return high_all() | |
| r = gate(eye, sampler) | |
| self.assertFalse(r["ranVlm"]) | |
| self.assertEqual(r["verdict"], "reject") | |
| self.assertEqual(called["n"], 0, "VLM must not be called when a hard gate failed") | |
| def test_pass_confirmed_when_all_criteria_high(self): | |
| eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []} | |
| r = gate(eye, const_sampler(high_all()), geometry_class="knife") | |
| self.assertEqual(r["verdict"], "pass") | |
| self.assertEqual(r["action"], "continue") | |
| self.assertTrue(r["ranVlm"]) | |
| def test_soft_reject_rescued_by_vlm(self): | |
| # deterministic ensemble was low-confidence (no HARD failure), VLM confirms → rescue | |
| eye = {"verdict": "low-confidence", "action": "probe", "hardGateFailures": []} | |
| r = gate(eye, const_sampler(high_all()), geometry_class="knife") | |
| self.assertEqual(r["verdict"], "pass") | |
| self.assertIn("rescued", r["reason"]) | |
| def test_low_objectness_withholds_refine_spec(self): | |
| eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []} | |
| s = high_all(); s["objectness"] = 0.5 | |
| r = gate(eye, const_sampler(s), geometry_class="knife") | |
| self.assertEqual(r["verdict"], "withhold") | |
| self.assertEqual(r["action"], "refine-spec") | |
| def test_low_structural_withholds_refine_code(self): | |
| eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []} | |
| s = high_all(); s["structural"] = 0.4 | |
| r = gate(eye, const_sampler(s), geometry_class="knife") | |
| self.assertEqual(r["verdict"], "withhold") | |
| self.assertEqual(r["action"], "refine-code") | |
| def test_high_sample_spread_is_uncertain_probe(self): | |
| eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []} | |
| samples = [high_all(), {"objectness": 0.4, "semantic": 0.4, "structural": 0.4, "specular": 0.4, "claimedClass": "knife"}] | |
| r = gate(eye, lambda i: samples[i % len(samples)], n_samples=2, geometry_class="knife") | |
| self.assertEqual(r["verdict"], "uncertain") | |
| self.assertEqual(r["action"], "probe") | |
| def test_evidence_contradiction_is_uncertain(self): | |
| eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []} | |
| r = gate(eye, const_sampler(high_all(claimed="spoon")), geometry_class="knife") | |
| self.assertEqual(r["verdict"], "uncertain") | |
| self.assertEqual(r["action"], "probe") | |
| def test_no_sampler_keeps_deterministic_verdict(self): | |
| eye = {"verdict": "pass", "action": "continue", "hardGateFailures": []} | |
| r = gate(eye, None) | |
| self.assertFalse(r["ranVlm"]) | |
| self.assertEqual(r["verdict"], "pass") | |
| def test_calibrate_identity_and_monotonic(self): | |
| self.assertEqual(calibrate(0.7), 0.7) # identity by default | |
| # a map that pulls raw 0.9 down to 0.7 (anti-overconfidence) | |
| cal = [[0.0, 0.0], [0.9, 0.7], [1.0, 0.8]] | |
| self.assertAlmostEqual(calibrate(0.9, cal), 0.7, places=5) | |
| self.assertLess(calibrate(0.9, cal), 0.9) | |
| def test_evidence_consistent_unknown_geometry(self): | |
| self.assertTrue(evidence_consistent("knife", None)) | |
| self.assertTrue(evidence_consistent("Knife", "knife")) | |
| self.assertFalse(evidence_consistent("spoon", "knife")) | |
| def test_aggregate_median(self): | |
| agg = aggregate_samples([{"objectness": 0.2}, {"objectness": 0.8}, {"objectness": 0.6}]) | |
| self.assertAlmostEqual(agg["criteria"]["objectness"], 0.6, places=5) | |
| if __name__ == "__main__": | |
| unittest.main(verbosity=2) | |