Fix Gemini quota circuit-breaker: detect 'quota' in 429 body
Browse filesGemini sends retry-after:5 even for daily quota exhaustion, so the
old 'wait > 30' check never fired. Now detect 'quota' in the body
and circuit-break for 8 hours, stopping the retry loop immediately.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- src/soci/engine/llm.py +16 -8
src/soci/engine/llm.py
CHANGED
|
@@ -725,11 +725,16 @@ class GeminiClient:
|
|
| 725 |
wait = float(retry_after)
|
| 726 |
except (ValueError, TypeError):
|
| 727 |
wait = 5.0
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 731 |
return ""
|
| 732 |
-
body =
|
| 733 |
logger.warning(f"Gemini 429: {body} — waiting {wait}s")
|
| 734 |
await asyncio.sleep(wait)
|
| 735 |
else:
|
|
@@ -791,11 +796,14 @@ class GeminiClient:
|
|
| 791 |
wait = float(retry_after)
|
| 792 |
except (ValueError, TypeError):
|
| 793 |
wait = 5.0
|
| 794 |
-
|
| 795 |
-
|
| 796 |
-
|
|
|
|
|
|
|
|
|
|
| 797 |
return {}
|
| 798 |
-
body =
|
| 799 |
logger.warning(f"Gemini 429 (json): {body} — waiting {wait}s")
|
| 800 |
await asyncio.sleep(wait)
|
| 801 |
else:
|
|
|
|
| 725 |
wait = float(retry_after)
|
| 726 |
except (ValueError, TypeError):
|
| 727 |
wait = 5.0
|
| 728 |
+
body_raw = e.response.text or ""
|
| 729 |
+
# Daily quota exhausted — Gemini sends retry-after:5 even for daily limits,
|
| 730 |
+
# so detect via message body and circuit-break for 8 hours.
|
| 731 |
+
if "quota" in body_raw.lower() or wait > 30:
|
| 732 |
+
circuit_wait = max(wait, 28800) # 8 hours
|
| 733 |
+
self._rate_limited_until = time.monotonic() + circuit_wait
|
| 734 |
+
body = body_raw[:200].replace("{", "(").replace("}", ")")
|
| 735 |
+
logger.warning(f"Gemini daily quota exhausted — circuit-breaking for {circuit_wait/3600:.1f}h: {body}")
|
| 736 |
return ""
|
| 737 |
+
body = body_raw[:200].replace("{", "(").replace("}", ")")
|
| 738 |
logger.warning(f"Gemini 429: {body} — waiting {wait}s")
|
| 739 |
await asyncio.sleep(wait)
|
| 740 |
else:
|
|
|
|
| 796 |
wait = float(retry_after)
|
| 797 |
except (ValueError, TypeError):
|
| 798 |
wait = 5.0
|
| 799 |
+
body_raw = e.response.text or ""
|
| 800 |
+
if "quota" in body_raw.lower() or wait > 30:
|
| 801 |
+
circuit_wait = max(wait, 28800) # 8 hours
|
| 802 |
+
self._rate_limited_until = time.monotonic() + circuit_wait
|
| 803 |
+
body = body_raw[:200].replace("{", "(").replace("}", ")")
|
| 804 |
+
logger.warning(f"Gemini daily quota exhausted — circuit-breaking for {circuit_wait/3600:.1f}h: {body}")
|
| 805 |
return {}
|
| 806 |
+
body = body_raw[:200].replace("{", "(").replace("}", ")")
|
| 807 |
logger.warning(f"Gemini 429 (json): {body} — waiting {wait}s")
|
| 808 |
await asyncio.sleep(wait)
|
| 809 |
else:
|