Spaces:
Sleeping
test(llm-chain): KI-089 — update credits-election test for KI-087 NIM-first preference
Browse filesThe pre-KI-087 `test_groq_above_water_picked_in_election` asserted that
Groq wins election when both candidates are healthy and Groq has more
credits + faster latency. KI-087 (`d90f8c0`) inverted this: NIM is
always preferred when eligible, regardless of latency. The test has
been failing on main since KI-087 landed.
Replaced with two paired tests that lock in both halves of the new
contract:
1. `test_nim_preferred_over_faster_groq_when_eligible` — asserts NIM
is elected as PRIMARY even when Groq is 3x faster AND has credits.
The strategic point: NIM is the free, $0, no-daily-cap provider;
Groq + OpenRouter are emergency fallback only.
2. `test_groq_picked_when_nim_pool_empty` — fallthrough invariant.
When NO eligible NIM candidate exists (chain has only non-NIM
models, or all NIM models down/quota-exhausted), election picks
the highest-scored non-NIM candidate as PRIMARY. Locks in the
safety net so a full NIM regional outage still produces a working
brain call.
Test results:
- tests/test_credits_election.py → 12/12 pass (was 11/12 with the
pre-existing failure; now all green)
- tests/test_routing_regression.py → 15 passed, 13 subtests passed
No source changes — only test rewrites to match the KI-087 contract
that's been live since `d90f8c0`.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
@@ -173,28 +173,67 @@ class TestElectionCreditGate(unittest.TestCase):
|
|
| 173 |
self.assertEqual(primary, nim_model,
|
| 174 |
"Quota-exhausted Groq should be skipped despite faster latency.")
|
| 175 |
|
| 176 |
-
def
|
| 177 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
groq_model = "groq:llama-3.3-70b-versatile"
|
| 179 |
nim_model = "qwen/qwen3-next-80b-a3b-instruct"
|
| 180 |
gh = _healthy_now(groq_model)
|
| 181 |
-
gh.latency_ms = 100
|
| 182 |
-
gh.credits_remaining = 10000.0 #
|
| 183 |
gh.credits_unit = "tokens_day"
|
| 184 |
gh.credits_low_water = GROQ_TOKENS_LOW_WATER
|
| 185 |
gh.credits_observed_at = time.monotonic()
|
| 186 |
llm_health._STATE[groq_model] = gh
|
| 187 |
|
| 188 |
nh = _healthy_now(nim_model)
|
| 189 |
-
nh.latency_ms = 300
|
| 190 |
llm_health._STATE[nim_model] = nh
|
| 191 |
|
| 192 |
with mock.patch.object(
|
| 193 |
llm_health, "_chain_for", return_value=[groq_model, nim_model]
|
| 194 |
):
|
| 195 |
primary = llm_health.get_primary("brain")
|
| 196 |
-
self.assertEqual(
|
| 197 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
|
| 199 |
def test_none_credits_is_permissive(self) -> None:
|
| 200 |
"""Cold-start: a candidate with credits_remaining=None must be electable."""
|
|
|
|
| 173 |
self.assertEqual(primary, nim_model,
|
| 174 |
"Quota-exhausted Groq should be skipped despite faster latency.")
|
| 175 |
|
| 176 |
+
def test_nim_preferred_over_faster_groq_when_eligible(self) -> None:
|
| 177 |
+
"""KI-087 NIM-first preference: even when both candidates are healthy
|
| 178 |
+
and Groq has plenty of credits AND is 3x faster on latency, NIM must
|
| 179 |
+
still be elected as PRIMARY. Groq + OpenRouter are emergency-fallback
|
| 180 |
+
only, not picked-on-latency. Replaces the pre-KI-087
|
| 181 |
+
`test_groq_above_water_picked_in_election` which asserted the
|
| 182 |
+
opposite (Groq wins on latency)."""
|
| 183 |
groq_model = "groq:llama-3.3-70b-versatile"
|
| 184 |
nim_model = "qwen/qwen3-next-80b-a3b-instruct"
|
| 185 |
gh = _healthy_now(groq_model)
|
| 186 |
+
gh.latency_ms = 100 # faster
|
| 187 |
+
gh.credits_remaining = 10000.0 # well above water
|
| 188 |
gh.credits_unit = "tokens_day"
|
| 189 |
gh.credits_low_water = GROQ_TOKENS_LOW_WATER
|
| 190 |
gh.credits_observed_at = time.monotonic()
|
| 191 |
llm_health._STATE[groq_model] = gh
|
| 192 |
|
| 193 |
nh = _healthy_now(nim_model)
|
| 194 |
+
nh.latency_ms = 300 # slower but NIM
|
| 195 |
llm_health._STATE[nim_model] = nh
|
| 196 |
|
| 197 |
with mock.patch.object(
|
| 198 |
llm_health, "_chain_for", return_value=[groq_model, nim_model]
|
| 199 |
):
|
| 200 |
primary = llm_health.get_primary("brain")
|
| 201 |
+
self.assertEqual(
|
| 202 |
+
primary, nim_model,
|
| 203 |
+
"KI-087: NIM must beat Groq as PRIMARY even when Groq is faster + "
|
| 204 |
+
"has credits. NIM is the strategic free provider; Groq + OpenRouter "
|
| 205 |
+
"are emergency fallback only.",
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
def test_groq_picked_when_nim_pool_empty(self) -> None:
|
| 209 |
+
"""KI-087 fallthrough: when NO eligible NIM candidate exists (all NIM
|
| 210 |
+
models down / out of credits / not in chain), election falls through
|
| 211 |
+
to the best non-NIM candidate as PRIMARY. Locks in the safety net so
|
| 212 |
+
a full NIM regional outage still produces a working brain call."""
|
| 213 |
+
groq_model = "groq:llama-3.3-70b-versatile"
|
| 214 |
+
or_model = "openrouter:openai/gpt-oss-120b"
|
| 215 |
+
gh = _healthy_now(groq_model)
|
| 216 |
+
gh.latency_ms = 100
|
| 217 |
+
gh.credits_remaining = 10000.0
|
| 218 |
+
gh.credits_unit = "tokens_day"
|
| 219 |
+
gh.credits_low_water = GROQ_TOKENS_LOW_WATER
|
| 220 |
+
gh.credits_observed_at = time.monotonic()
|
| 221 |
+
llm_health._STATE[groq_model] = gh
|
| 222 |
+
|
| 223 |
+
oh = _healthy_now(or_model)
|
| 224 |
+
oh.latency_ms = 800 # slower
|
| 225 |
+
llm_health._STATE[or_model] = oh
|
| 226 |
+
|
| 227 |
+
# Note the chain has NO NIM candidates.
|
| 228 |
+
with mock.patch.object(
|
| 229 |
+
llm_health, "_chain_for", return_value=[or_model, groq_model]
|
| 230 |
+
):
|
| 231 |
+
primary = llm_health.get_primary("brain")
|
| 232 |
+
self.assertEqual(
|
| 233 |
+
primary, groq_model,
|
| 234 |
+
"KI-087 fallthrough: no NIM eligible → election picks the highest-"
|
| 235 |
+
"scored non-NIM candidate (Groq's 100ms beats OpenRouter's 800ms).",
|
| 236 |
+
)
|
| 237 |
|
| 238 |
def test_none_credits_is_permissive(self) -> None:
|
| 239 |
"""Cold-start: a candidate with credits_remaining=None must be electable."""
|