Add provider toggle (Groq/Gemini) + global rate limiter to beat free-tier limits

Default to Groq llama-3.1-8b-instant (high TPM) so the batch survives without rate-limit errors; Gemini-2.0-flash selectable via LLM_PROVIDER. Shared InMemoryRateLimiter paces all text-LLM calls. Vision/Whisper stay on Groq.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -1,6 +1,18 @@
 # --- Required secrets (fill these in) ---
 GROQ_API_KEY=
 TAVILY_API_KEY=
+# Required when LLM_PROVIDER=gemini (default). Free key: aistudio.google.com
+GOOGLE_API_KEY=
+
+# --- LLM provider for text/reasoning + judge nodes: "groq" or "gemini" ---
+# Default "groq" + llama-3.1-8b-instant works with the existing key (high TPM).
+# Switch to "gemini" for better quality once you have a free-tier GOOGLE_API_KEY
+# (created in a Google project WITHOUT billing enabled, else free tier = 0).
+LLM_PROVIDER=groq
+GROQ_TEXT_MODEL=llama-3.1-8b-instant
+GEMINI_TEXT_MODEL=gemini-2.0-flash
+# Global pace across all text-LLM calls (req/sec). 0.2 = ~12/min.
+RATE_LIMIT_RPS=0.2
 
 # --- Optional: LangSmith tracing (nice for debugging the graph) ---
 # Set TRACING=true and provide the API key to stream runs to smith.langchain.com.

gaia_agent/config.py

@@ -21,17 +21,30 @@ class Settings(BaseSettings):
     # --- Secrets ---
     groq_api_key: str = ""
     tavily_api_key: str = ""
+    google_api_key: str = ""
+
+    # --- LLM provider for the text/reasoning + judge nodes ---
+    # "groq" (default, works with the existing key) or "gemini" (better quality,
+    # needs a free-tier GOOGLE_API_KEY from a project WITHOUT billing enabled).
+    # Vision + Whisper always use Groq.
+    llm_provider: str = "groq"
 
     # --- Models (override via env) ---
-    groq_text_model: str = "llama-3.3-70b-versatile"
+    gemini_text_model: str = "gemini-2.0-flash"
+    # 8b-instant has ~30k TPM free (vs 70B's ~12k) so it survives the batch.
+    groq_text_model: str = "llama-3.1-8b-instant"
     groq_vision_model: str = "meta-llama/llama-4-scout-17b-16e-instruct"
     groq_whisper_model: str = "whisper-large-v3"
 
+    # --- Rate limiting (requests/sec across all text-LLM calls) ---
+    # Gemini 2.0-flash free tier ~15 RPM; 0.2 rps = ~12/min keeps headroom.
+    rate_limit_rps: float = 0.2
+
     # --- API + control knobs ---
     gaia_api_url: str = "https://agents-course-unit4-scoring.hf.space"
     max_judge_retries: int = 1
     recursion_limit: int = 40
-    question_timeout: int = 180
+    question_timeout: int = 300
 
 
 @lru_cache(maxsize=1)

gaia_agent/llm.py

@@ -1,33 +1,69 @@
-"""Groq chat-model factories."""
+"""Text/vision LLM factories with a shared global rate limiter.
+
+The text model is provider-switchable (Gemini or Groq). A single process-wide
+``InMemoryRateLimiter`` paces every text-LLM call so we never trip per-minute
+request limits. Vision + Whisper always use Groq (rare calls, low volume).
+"""
 
 from __future__ import annotations
 
 from functools import lru_cache
 
-from langchain_groq import ChatGroq
+from langchain_core.rate_limiters import InMemoryRateLimiter
 
 from gaia_agent.config import get_settings
 
 
+@lru_cache(maxsize=1)
+def _rate_limiter() -> InMemoryRateLimiter:
+    """Shared limiter so all text-LLM instances draw from one token bucket."""
+    rps = get_settings().rate_limit_rps
+    return InMemoryRateLimiter(
+        requests_per_second=rps,
+        check_every_n_seconds=0.1,
+        max_bucket_size=1,
+    )
+
+
 @lru_cache(maxsize=4)
-def get_text_llm(temperature: float = 0.0) -> ChatGroq:
-    """Return the Groq text/reasoning model used by most nodes."""
+def get_text_llm(temperature: float = 0.0):
+    """Return the rate-limited text/reasoning model for the configured provider."""
     s = get_settings()
-    return ChatGroq(
-        model=s.groq_text_model,
-        api_key=s.groq_api_key,
+    limiter = _rate_limiter()
+
+    if s.llm_provider.lower() == "groq":
+        from langchain_groq import ChatGroq
+
+        return ChatGroq(
+            model=s.groq_text_model,
+            api_key=s.groq_api_key,
+            temperature=temperature,
+            max_retries=3,
+            rate_limiter=limiter,
+        )
+
+    # Default: Gemini (high free-tier TPM).
+    from langchain_google_genai import ChatGoogleGenerativeAI
+
+    return ChatGoogleGenerativeAI(
+        model=s.gemini_text_model,
+        google_api_key=s.google_api_key,
         temperature=temperature,
-        max_retries=2,
+        max_retries=3,
+        rate_limiter=limiter,
     )
 
 
 @lru_cache(maxsize=1)
-def get_vision_llm() -> ChatGroq:
+def get_vision_llm():
     """Return the Groq multimodal model used for image understanding."""
+    from langchain_groq import ChatGroq
+
     s = get_settings()
     return ChatGroq(
         model=s.groq_vision_model,
         api_key=s.groq_api_key,
         temperature=0.0,
         max_retries=2,
+        rate_limiter=_rate_limiter(),
     )

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "langchain>=0.3.0",
     "langchain-core>=0.3.0",
     "langchain-groq>=0.2.0",
+    "langchain-google-genai>=2.0.0",
     "langchain-tavily>=0.1.0",
     "langchain-community>=0.3.0",
     "wikipedia>=1.4.0",

requirements.txt

@@ -8,6 +8,7 @@ langgraph>=1.0.0
 langchain>=0.3.0
 langchain-core>=0.3.0
 langchain-groq>=0.2.0
+langchain-google-genai>=2.0.0
 langchain-tavily>=0.1.0
 langchain-community>=0.3.0
 wikipedia