File size: 4,530 Bytes
87149fa 0ea997a 87149fa be97847 87149fa be97847 87149fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | """Router logic, tested with a controlled embedder rather than the real model.
Model quality is a separate concern; what matters here is that ambiguity falls
back to the full profile instead of silently answering without evidence.
"""
import numpy as np
import pytest
from router import FULL, SPECIFIC, TRIVIAL, SemanticRouter
REFERENCES = {TRIVIAL: ["GREET"], SPECIFIC: ["NARROW"], FULL: ["BROAD"]}
AXIS = {"GREET": 0, "NARROW": 1, "BROAD": 2}
def fake_embed(texts):
"""One-hot per route keyword; anything else is equidistant from all three."""
vectors = []
for text in texts:
vector = np.zeros(3, dtype=np.float32)
for token, axis in AXIS.items():
if token in text:
vector[axis] = 1.0
if not vector.any():
vector[:] = 1 / np.sqrt(3)
vectors.append(vector)
return np.array(vectors, dtype=np.float32)
@pytest.fixture
def router():
return SemanticRouter(fake_embed, references=REFERENCES)
@pytest.mark.parametrize(
"query,expected",
[("GREET", TRIVIAL), ("NARROW", SPECIFIC), ("BROAD", FULL)],
)
def test_confident_queries_reach_their_route(router, query, expected):
assert router.route(query) == expected
def test_ambiguous_queries_fall_back_to_full(router):
"""Equidistant from every route: answering without the profile is the costly error."""
assert router.route("nothing in particular") == FULL
def test_a_tie_between_trivial_and_specific_falls_back_to_full(router):
assert router.route("GREET NARROW") == FULL
@pytest.mark.parametrize("query", ["", " ", None])
def test_empty_queries_fall_back_to_full(router, query):
assert router.route(query) == FULL
def test_a_high_threshold_forces_the_safe_route():
"""An exact match scores 1.0 here, so the bar has to sit above it."""
strict = SemanticRouter(fake_embed, references=REFERENCES, min_score=1.1)
assert strict.route("GREET") == FULL
def test_a_wide_margin_requirement_forces_the_safe_route():
strict = SemanticRouter(fake_embed, references=REFERENCES, min_margin=1.1)
assert strict.route("NARROW") == FULL
def test_shipped_thresholds_leave_room_for_a_confident_match():
from router import MIN_MARGIN, MIN_SCORE
assert 0 < MIN_SCORE < 1
assert 0 < MIN_MARGIN < 1
class TestRouteLogging:
"""Answers do not reveal which tier produced them, so the log is the only
way to tell a full-profile answer from one built on five excerpts."""
def test_the_chosen_route_is_logged(self, router, caplog):
with caplog.at_level("INFO", logger="router"):
router.route("NARROW")
assert "Route specific" in caplog.text
def test_the_log_records_every_score(self, router, caplog):
with caplog.at_level("INFO", logger="router"):
router.route("NARROW")
for route in (TRIVIAL, SPECIFIC, FULL):
assert f"{route}=" in caplog.text
def test_a_fallback_says_which_rule_rejected_the_winner(self, router, caplog):
"""'Route full' alone cannot be told apart from a genuine full match."""
with caplog.at_level("INFO", logger="router"):
router.route("GREET NARROW")
assert "min_margin" in caplog.text
def test_a_low_score_fallback_names_the_score_rule(self, caplog):
strict = SemanticRouter(fake_embed, references=REFERENCES, min_score=1.1)
with caplog.at_level("INFO", logger="router"):
strict.route("GREET")
assert "min_score" in caplog.text
def test_an_unroutable_query_still_logs(self, router, caplog):
with caplog.at_level("INFO", logger="router"):
router.route(None)
assert "Route full" in caplog.text
def test_scores_cover_every_route(router):
assert set(router.scores("GREET")) == {TRIVIAL, SPECIFIC, FULL}
def test_centroids_are_normalised(router):
assert np.allclose(np.linalg.norm(router.centroids, axis=1), 1.0)
class TestRealReferenceSet:
"""The shipped reference queries must be well formed."""
def test_every_route_has_examples(self):
from router import REFERENCE_QUERIES, SKILL_ROUTES
assert set(REFERENCE_QUERIES) == {TRIVIAL, SPECIFIC, FULL, *SKILL_ROUTES}
for examples in REFERENCE_QUERIES.values():
assert len(examples) >= 5
def test_examples_are_unique_across_routes(self):
from router import REFERENCE_QUERIES
seen = [q for examples in REFERENCE_QUERIES.values() for q in examples]
assert len(seen) == len(set(seen))
|