Spaces:
Sleeping
Sleeping
yc1838 commited on
Commit ·
69842d5
1
Parent(s): 388e04b
feat: apply cooldown to bound model calls
Browse files- src/lilith_agent/models.py +24 -12
- tests/test_models.py +16 -0
src/lilith_agent/models.py
CHANGED
|
@@ -376,26 +376,38 @@ class _RetryWrapper(BaseChatModel):
|
|
| 376 |
|
| 377 |
def bind_tools(self, tools: Any, **kwargs: Any):
|
| 378 |
bound = self.inner.bind_tools(tools, **kwargs)
|
| 379 |
-
|
| 380 |
-
return _BoundRetryWrapper(bound=bound)
|
| 381 |
|
| 382 |
|
| 383 |
class _BoundRetryWrapper(Runnable):
|
| 384 |
"""Wraps a tool-bound Runnable to apply retry logic to .invoke()."""
|
| 385 |
|
| 386 |
-
def __init__(self, bound):
|
| 387 |
self._bound = bound
|
|
|
|
|
|
|
| 388 |
|
| 389 |
def invoke(self, input, config=None, **kwargs):
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 399 |
|
| 400 |
async def ainvoke(self, input, config=None, **kwargs):
|
| 401 |
async for attempt in AsyncRetrying(**_async_retry_params()):
|
|
|
|
| 376 |
|
| 377 |
def bind_tools(self, tools: Any, **kwargs: Any):
|
| 378 |
bound = self.inner.bind_tools(tools, **kwargs)
|
| 379 |
+
return _BoundRetryWrapper(bound=bound, provider=self.provider, model_name=self.model_name)
|
|
|
|
| 380 |
|
| 381 |
|
| 382 |
class _BoundRetryWrapper(Runnable):
|
| 383 |
"""Wraps a tool-bound Runnable to apply retry logic to .invoke()."""
|
| 384 |
|
| 385 |
+
def __init__(self, bound, provider: str | None = None, model_name: str | None = None):
|
| 386 |
self._bound = bound
|
| 387 |
+
self._provider = provider
|
| 388 |
+
self._model_name = model_name
|
| 389 |
|
| 390 |
def invoke(self, input, config=None, **kwargs):
|
| 391 |
+
lane = _gemini_lane(self._provider, self._model_name)
|
| 392 |
+
_sleep_active_cooldown(lane)
|
| 393 |
+
try:
|
| 394 |
+
for attempt in Retrying(**_sync_retry_params()):
|
| 395 |
+
with attempt:
|
| 396 |
+
try:
|
| 397 |
+
result = self._bound.invoke(input, config=config, **kwargs)
|
| 398 |
+
except Exception as observed:
|
| 399 |
+
record_rate_limit_observation(observed)
|
| 400 |
+
raise
|
| 401 |
+
record_rate_limit_success()
|
| 402 |
+
_record_success(lane)
|
| 403 |
+
return result
|
| 404 |
+
except Exception as exc:
|
| 405 |
+
if lane is not None and is_retryable_rate_limit(exc):
|
| 406 |
+
reason = _batch_abort_reason(exc)
|
| 407 |
+
if reason is not None:
|
| 408 |
+
raise BatchAbortRateLimitError(reason=reason, original_error=str(exc)) from exc
|
| 409 |
+
raise _record_exhausted_rate_limit(lane, exc) from exc
|
| 410 |
+
raise
|
| 411 |
|
| 412 |
async def ainvoke(self, input, config=None, **kwargs):
|
| 413 |
async for attempt in AsyncRetrying(**_async_retry_params()):
|
tests/test_models.py
CHANGED
|
@@ -374,3 +374,19 @@ def test_batch_window_does_not_trigger_below_threshold():
|
|
| 374 |
record_rate_limit_success()
|
| 375 |
|
| 376 |
assert batch_rate_limit_pause_seconds() is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
record_rate_limit_success()
|
| 375 |
|
| 376 |
assert batch_rate_limit_pause_seconds() is None
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def test_bound_retry_wrapper_raises_cooldown_for_gemini_lane(monkeypatch):
|
| 380 |
+
_reset_rate_limit_state_for_tests()
|
| 381 |
+
exc = _make_genai_client_error(429)
|
| 382 |
+
monkeypatch.setattr("lilith_agent.models.time.sleep", lambda _: None)
|
| 383 |
+
wrapper = _RetryWrapper.model_construct(
|
| 384 |
+
inner=_FailingGenerateModel(exc), provider="google", model_name="gemini-3.1-pro"
|
| 385 |
+
)
|
| 386 |
+
|
| 387 |
+
bound = wrapper.bind_tools([])
|
| 388 |
+
|
| 389 |
+
with pytest.raises(RateLimitCooldownError) as raised:
|
| 390 |
+
bound.invoke([("user", "hi")])
|
| 391 |
+
|
| 392 |
+
assert raised.value.model == "gemini-3.1-pro"
|