RayMelius Claude Sonnet 4.6 commited on
Commit
9b3bd67
·
1 Parent(s): 891d877

Fix Gemini quota circuit-breaker: detect 'quota' in 429 body

Browse files

Gemini sends retry-after:5 even for daily quota exhaustion, so the
old 'wait > 30' check never fired. Now detect 'quota' in the body
and circuit-break for 8 hours, stopping the retry loop immediately.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. src/soci/engine/llm.py +16 -8
src/soci/engine/llm.py CHANGED
@@ -725,11 +725,16 @@ class GeminiClient:
725
  wait = float(retry_after)
726
  except (ValueError, TypeError):
727
  wait = 5.0
728
- if wait > 30:
729
- self._rate_limited_until = time.monotonic() + wait
730
- logger.warning(f"Gemini quota exhausted for {wait:.0f}s")
 
 
 
 
 
731
  return ""
732
- body = e.response.text[:200].replace("{", "(").replace("}", ")")
733
  logger.warning(f"Gemini 429: {body} — waiting {wait}s")
734
  await asyncio.sleep(wait)
735
  else:
@@ -791,11 +796,14 @@ class GeminiClient:
791
  wait = float(retry_after)
792
  except (ValueError, TypeError):
793
  wait = 5.0
794
- if wait > 30:
795
- self._rate_limited_until = time.monotonic() + wait
796
- logger.warning(f"Gemini quota exhausted for {wait:.0f}s")
 
 
 
797
  return {}
798
- body = e.response.text[:200].replace("{", "(").replace("}", ")")
799
  logger.warning(f"Gemini 429 (json): {body} — waiting {wait}s")
800
  await asyncio.sleep(wait)
801
  else:
 
725
  wait = float(retry_after)
726
  except (ValueError, TypeError):
727
  wait = 5.0
728
+ body_raw = e.response.text or ""
729
+ # Daily quota exhausted — Gemini sends retry-after:5 even for daily limits,
730
+ # so detect via message body and circuit-break for 8 hours.
731
+ if "quota" in body_raw.lower() or wait > 30:
732
+ circuit_wait = max(wait, 28800) # 8 hours
733
+ self._rate_limited_until = time.monotonic() + circuit_wait
734
+ body = body_raw[:200].replace("{", "(").replace("}", ")")
735
+ logger.warning(f"Gemini daily quota exhausted — circuit-breaking for {circuit_wait/3600:.1f}h: {body}")
736
  return ""
737
+ body = body_raw[:200].replace("{", "(").replace("}", ")")
738
  logger.warning(f"Gemini 429: {body} — waiting {wait}s")
739
  await asyncio.sleep(wait)
740
  else:
 
796
  wait = float(retry_after)
797
  except (ValueError, TypeError):
798
  wait = 5.0
799
+ body_raw = e.response.text or ""
800
+ if "quota" in body_raw.lower() or wait > 30:
801
+ circuit_wait = max(wait, 28800) # 8 hours
802
+ self._rate_limited_until = time.monotonic() + circuit_wait
803
+ body = body_raw[:200].replace("{", "(").replace("}", ")")
804
+ logger.warning(f"Gemini daily quota exhausted — circuit-breaking for {circuit_wait/3600:.1f}h: {body}")
805
  return {}
806
+ body = body_raw[:200].replace("{", "(").replace("}", ")")
807
  logger.warning(f"Gemini 429 (json): {body} — waiting {wait}s")
808
  await asyncio.sleep(wait)
809
  else: