feat: track batch rate limit window

src/lilith_agent/models.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import time
+from collections import deque
 from contextlib import contextmanager
 from contextvars import ContextVar
 from dataclasses import dataclass
@@ -112,9 +113,14 @@ _NO_THINK = "/no_think"
 _GEMINI_COOLDOWN_MODELS = {"gemini-3-flash-preview", "gemini-3.1-pro"}
 _COOLDOWN_LADDER_SECONDS = (60, 120, 300)
 _QUESTION_STREAK_LIMIT = 50
+_BATCH_WINDOW_SIZE = 100
+_BATCH_WINDOW_RATE_LIMIT_THRESHOLD = 70
+_BATCH_PAUSE_LADDER_SECONDS = (300, 600, 1200)
 _cooldown_until: dict[tuple[str, str], float] = {}
 _rate_limit_exhaustions: dict[tuple[str, str], int] = {}
 _question_rate_limit_streak: ContextVar[int | None] = ContextVar("question_rate_limit_streak", default=None)
+_batch_rate_limit_window: deque[bool] = deque(maxlen=_BATCH_WINDOW_SIZE)
+_batch_pause_count = 0
 
 
 @dataclass
@@ -149,8 +155,11 @@ class QuestionRateLimitStreakError(Exception):
 
 
 def _reset_rate_limit_state_for_tests() -> None:
+    global _batch_pause_count
     _cooldown_until.clear()
     _rate_limit_exhaustions.clear()
+    _batch_rate_limit_window.clear()
+    _batch_pause_count = 0
 
 
 @contextmanager
@@ -163,8 +172,11 @@ def rate_limit_question_scope():
 
 
 def record_rate_limit_observation(exc: BaseException) -> None:
+    if not is_retryable_rate_limit(exc):
+        return
+    _batch_rate_limit_window.append(True)
     current = _question_rate_limit_streak.get()
-    if current is None or not is_retryable_rate_limit(exc):
+    if current is None:
         return
     current += 1
     _question_rate_limit_streak.set(current)
@@ -173,10 +185,26 @@ def record_rate_limit_observation(exc: BaseException) -> None:
 
 
 def record_rate_limit_success() -> None:
+    _batch_rate_limit_window.append(False)
     if _question_rate_limit_streak.get() is not None:
         _question_rate_limit_streak.set(0)
 
 
+def batch_rate_limit_pause_seconds() -> int | None:
+    global _batch_pause_count
+    if len(_batch_rate_limit_window) < _BATCH_WINDOW_SIZE:
+        return None
+    if sum(_batch_rate_limit_window) < _BATCH_WINDOW_RATE_LIMIT_THRESHOLD:
+        return None
+    _batch_pause_count += 1
+    idx = min(_batch_pause_count - 1, len(_BATCH_PAUSE_LADDER_SECONDS) - 1)
+    return _BATCH_PAUSE_LADDER_SECONDS[idx]
+
+
+def clear_batch_rate_limit_window() -> None:
+    _batch_rate_limit_window.clear()
+
+
 def _gemini_lane(provider: str | None, model_name: str | None) -> tuple[str, str] | None:
     if provider == "google" and model_name in _GEMINI_COOLDOWN_MODELS:
         return (provider, model_name)

tests/test_models.py

@@ -13,6 +13,8 @@ from lilith_agent.models import (
     QuestionRateLimitStreakError,
     RateLimitCooldownError,
     _reset_rate_limit_state_for_tests,
+    batch_rate_limit_pause_seconds,
+    clear_batch_rate_limit_window,
     is_retryable_rate_limit,
     rate_limit_question_scope,
 )
@@ -344,3 +346,31 @@ def test_question_rate_limit_scope_resets_after_success():
         record_rate_limit_success()
         for _ in range(49):
             record_rate_limit_observation(exc)
+
+
+def test_batch_window_triggers_pause_after_70_rate_limits_in_100_outcomes():
+    _reset_rate_limit_state_for_tests()
+    exc = _make_genai_client_error(429)
+    from lilith_agent.models import record_rate_limit_observation, record_rate_limit_success
+
+    for _ in range(70):
+        record_rate_limit_observation(exc)
+    for _ in range(30):
+        record_rate_limit_success()
+
+    assert batch_rate_limit_pause_seconds() == 300
+    clear_batch_rate_limit_window()
+    assert batch_rate_limit_pause_seconds() is None
+
+
+def test_batch_window_does_not_trigger_below_threshold():
+    _reset_rate_limit_state_for_tests()
+    exc = _make_genai_client_error(429)
+    from lilith_agent.models import record_rate_limit_observation, record_rate_limit_success
+
+    for _ in range(69):
+        record_rate_limit_observation(exc)
+    for _ in range(31):
+        record_rate_limit_success()
+
+    assert batch_rate_limit_pause_seconds() is None