Cascading LLM fallback: Groq 8b-instant primary → Gemini Flash fallback

- Primary model switched from llama-3.3-70b-versatile to llama-3.1-8b-instant.
Groq's daily token quota on 8b is 5x higher than 70b.
- New module recommender/llm.py: single chat_completion_from_messages()
that tries Groq first, falls back to Gemini 1.5 Flash on rate-limit or
any other exception. Translates OpenAI-style messages -> Gemini API.
- slots.py, explain.py, chat.py: refactored to use the unified interface.
- app.py: /chat endpoint wraps the agent call in try/except so complete
LLM outage returns a friendly chat-type response instead of 500.

Requires HF Space secret: GEMINI_API_KEY (get at aistudio.google.com/apikey)

app.py

@@ -111,13 +111,28 @@ def health() -> Dict[str, Any]:
 
 @app.post("/chat")
 def chat(req: ChatIn) -> Dict[str, Any]:
-    return get_agent().respond(
-        user_id=req.user_id,
-        query=req.message,
-        conversation_history=req.history or [],
-        recent_games=req.recent_games or [],
-        top_n=req.top_n,
-    )
+    try:
+        return get_agent().respond(
+            user_id=req.user_id,
+            query=req.message,
+            conversation_history=req.history or [],
+            recent_games=req.recent_games or [],
+            top_n=req.top_n,
+        )
+    except Exception as e:
+        # LLM providers exhausted (both Groq + Gemini down / rate-limited).
+        # Return a friendly response instead of 500 so the frontend can show
+        # a nice message instead of an error stack.
+        return {
+            "type": "chat",
+            "message": (
+                "The genie is resting — both AI providers hit their limits. "
+                "Try again in a minute, or browse by genre."
+            ),
+            "slots": {"intent": "chat", "search_text": req.message},
+            "recommendations": [],
+            "error": str(e)[:200],
+        }
 
 
 @app.post("/feedback")

recommender/chat.py

@@ -11,12 +11,9 @@ discuss games / gaming / this recommender, and to redirect anything else.
 Off-topic messages are short-circuited to a canned response so we don't pay
 for LLM calls on "what's the weather".
 """
-import os
 from typing import List, Dict, Optional
 
-from groq import Groq
-
-MODEL = "llama-3.3-70b-versatile"
+from .llm import chat_completion_from_messages
 
 OFF_TOPIC_REPLY = (
     "I'm a game recommender — I only chat about games. "
@@ -37,19 +34,6 @@ STYLE:
 
 If the user is asking about specific games shown in RECENT_RECS, answer using ONLY that metadata. Do not fabricate details."""
 
-_client: Optional[Groq] = None
-
-
-def _get_client() -> Groq:
-    global _client
-    if _client is None:
-        api_key = os.environ.get("GROQ_API_KEY")
-        if not api_key:
-            raise RuntimeError("GROQ_API_KEY not set")
-        _client = Groq(api_key=api_key)
-    return _client
-
-
 def off_topic_reply() -> str:
     """No LLM call — canned polite redirect."""
     return OFF_TOPIC_REPLY
@@ -86,10 +70,9 @@ def chat_reply(
 
     messages.append({"role": "user", "content": query})
 
-    resp = _get_client().chat.completions.create(
-        model=MODEL,
-        messages=messages,
+    reply = chat_completion_from_messages(
+        messages,
         temperature=0.4,
         max_tokens=250,
     )
-    return (resp.choices[0].message.content or "").strip() or OFF_TOPIC_REPLY
+    return reply or OFF_TOPIC_REPLY

recommender/explain.py

@@ -10,24 +10,9 @@ Concepts:
   and faster; the model sees the full slate and can differentiate rationales.
 """
 import json
-import os
 from typing import List, Optional
 
-from groq import Groq
-
-MODEL = "llama-3.3-70b-versatile"
-
-_client: Optional[Groq] = None
-
-
-def _get_client() -> Groq:
-    global _client
-    if _client is None:
-        api_key = os.environ.get("GROQ_API_KEY")
-        if not api_key:
-            raise RuntimeError("GROQ_API_KEY not set in env")
-        _client = Groq(api_key=api_key)
-    return _client
+from .llm import chat_completion_from_messages
 
 
 SYSTEM_PROMPT = (
@@ -69,9 +54,8 @@ def generate_rationales(
     )
 
     try:
-        resp = _get_client().chat.completions.create(
-            model=MODEL,
-            messages=[
+        raw = chat_completion_from_messages(
+            [
                 {"role": "system", "content": SYSTEM_PROMPT},
                 {"role": "user", "content": user_prompt},
             ],
@@ -79,7 +63,7 @@ def generate_rationales(
             temperature=0.4,
             max_tokens=800,
         )
-        data = json.loads(resp.choices[0].message.content or "{}")
+        data = json.loads(raw or "{}")
         by_id = {int(r["id"]): str(r["text"]) for r in data.get("rationales", []) if "id" in r and "text" in r}
     except (json.JSONDecodeError, KeyError, ValueError):
         by_id = {}
@@ -113,16 +97,14 @@ def explain_game_for_user(game: dict, liked_games: Optional[List[dict]] = None)
         "- Paragraph 2 (50-70 words): whether it's a fit for THIS user. Reference past likes if relevant.\n"
         "- Ground every claim in the actual metadata.\n"
     )
-    resp = _get_client().chat.completions.create(
-        model=MODEL,
-        messages=[
+    return chat_completion_from_messages(
+        [
             {"role": "system", "content": EXPLAIN_GAME_SYSTEM},
             {"role": "user", "content": prompt},
         ],
         temperature=0.5,
         max_tokens=400,
     )
-    return (resp.choices[0].message.content or "").strip()
 
 
 # ---------- Genre deep-dive ----------
@@ -147,16 +129,14 @@ def explain_genre(genre_name: str, examples: Optional[List[dict]] = None) -> str
         "- Written for someone who's heard the term but doesn't know it deeply.\n"
         "- No 'in conclusion' phrases, no marketing tone."
     )
-    resp = _get_client().chat.completions.create(
-        model=MODEL,
-        messages=[
+    return chat_completion_from_messages(
+        [
             {"role": "system", "content": GENRE_EXPLAIN_SYSTEM},
             {"role": "user", "content": prompt},
         ],
         temperature=0.4,
         max_tokens=400,
     )
-    return (resp.choices[0].message.content or "").strip()
 
 
 # ---------- Taste summary ----------
@@ -187,13 +167,11 @@ def taste_summary(liked_games: List[dict]) -> str:
         "- Reference specific tags/vibes that repeat.\n"
         "- No 'overall', 'in conclusion', 'seems like'. Direct observations only."
     )
-    resp = _get_client().chat.completions.create(
-        model=MODEL,
-        messages=[
+    return chat_completion_from_messages(
+        [
             {"role": "system", "content": TASTE_SUMMARY_SYSTEM},
             {"role": "user", "content": prompt},
         ],
         temperature=0.4,
         max_tokens=200,
     )
-    return (resp.choices[0].message.content or "").strip()

recommender/llm.py

@@ -0,0 +1,149 @@
+"""
+Unified LLM interface with cascading fallback.
+
+Order of tries:
+    1. Groq llama-3.1-8b-instant   — primary. 5x the daily quota of 70B.
+    2. Google Gemini 1.5 Flash     — fallback. Separate 1M tokens/day free tier.
+
+Callers use one function: `chat_completion_from_messages(...)`.
+Same messages format both underlying APIs, no model-specific logic outside this file.
+"""
+import os
+import time
+from typing import Dict, List, Optional
+
+# Groq
+from groq import Groq
+try:
+    # Groq SDK exposes rate limit exceptions in these paths across versions.
+    from groq import RateLimitError as _GroqRateLimit  # type: ignore
+except ImportError:
+    try:
+        from groq._exceptions import RateLimitError as _GroqRateLimit  # type: ignore
+    except ImportError:
+        _GroqRateLimit = Exception  # type: ignore
+
+# Gemini
+import google.generativeai as genai
+
+# Model IDs
+GROQ_PRIMARY_MODEL = "llama-3.1-8b-instant"     # higher daily TPD than 70B
+GEMINI_MODEL       = "gemini-1.5-flash"          # free 1M tokens/day
+
+_groq_client: Optional[Groq] = None
+_gemini_configured = False
+
+
+def _get_groq() -> Groq:
+    global _groq_client
+    if _groq_client is None:
+        key = os.environ.get("GROQ_API_KEY")
+        if not key:
+            raise RuntimeError("GROQ_API_KEY not set")
+        _groq_client = Groq(api_key=key)
+    return _groq_client
+
+
+def _ensure_gemini() -> None:
+    global _gemini_configured
+    if _gemini_configured:
+        return
+    key = os.environ.get("GEMINI_API_KEY")
+    if not key:
+        raise RuntimeError("GEMINI_API_KEY not set")
+    genai.configure(api_key=key)
+    _gemini_configured = True
+
+
+def _log(msg: str) -> None:
+    print(f"[llm] {msg}", flush=True)
+
+
+def chat_completion_from_messages(
+    messages: List[Dict],
+    response_format: Optional[Dict] = None,
+    temperature: float = 0.4,
+    max_tokens: int = 500,
+) -> str:
+    """
+    Try Groq first, fall back to Gemini on rate limit or any exception.
+    Returns the response text (or empty string on complete failure).
+    """
+    # ---- 1. Groq ----
+    try:
+        resp = _get_groq().chat.completions.create(
+            model=GROQ_PRIMARY_MODEL,
+            messages=messages,
+            response_format=response_format,
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        return (resp.choices[0].message.content or "").strip()
+    except _GroqRateLimit as e:
+        _log(f"Groq rate-limited, falling back to Gemini: {e}")
+    except Exception as e:
+        _log(f"Groq errored ({type(e).__name__}), falling back to Gemini: {e}")
+
+    # ---- 2. Gemini fallback ----
+    try:
+        return _gemini_from_messages(
+            messages,
+            response_format=response_format,
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+    except Exception as e:
+        _log(f"Gemini fallback also failed ({type(e).__name__}): {e}")
+        # Return empty; callers should handle gracefully
+        return ""
+
+
+def _gemini_from_messages(
+    messages: List[Dict],
+    response_format: Optional[Dict],
+    temperature: float,
+    max_tokens: int,
+) -> str:
+    """Translate OpenAI-style messages → Gemini API call."""
+    _ensure_gemini()
+
+    # Split system content out; Gemini uses `system_instruction`, not a message role.
+    system_texts: List[str] = []
+    convo: List[Dict] = []
+    for m in messages:
+        role = m.get("role")
+        content = m.get("content") or ""
+        if role == "system":
+            system_texts.append(content)
+        elif role == "user":
+            convo.append({"role": "user", "parts": [content]})
+        elif role == "assistant":
+            convo.append({"role": "model", "parts": [content]})
+
+    system_instruction = "\n\n".join(system_texts) if system_texts else None
+
+    gen_config: Dict = {
+        "temperature": temperature,
+        "max_output_tokens": max_tokens,
+    }
+    if response_format and response_format.get("type") == "json_object":
+        gen_config["response_mime_type"] = "application/json"
+
+    model = genai.GenerativeModel(
+        model_name=GEMINI_MODEL,
+        system_instruction=system_instruction,
+        generation_config=gen_config,
+    )
+
+    if not convo:
+        return ""
+
+    if len(convo) == 1:
+        resp = model.generate_content(convo[0]["parts"][0])
+    else:
+        chat = model.start_chat(history=convo[:-1])
+        resp = chat.send_message(convo[-1]["parts"][0])
+
+    # Gemini can return responses with no text (safety filter, etc.)
+    text = getattr(resp, "text", None) or ""
+    return text.strip()

recommender/slots.py

@@ -12,24 +12,9 @@ Concepts:
   We only surface this when it says so — otherwise recommend.
 """
 import json
-import os
 from typing import Optional, List, Dict
 
-from groq import Groq
-
-MODEL = "llama-3.3-70b-versatile"
-
-_client: Optional[Groq] = None
-
-
-def _get_client() -> Groq:
-    global _client
-    if _client is None:
-        api_key = os.environ.get("GROQ_API_KEY")
-        if not api_key:
-            raise RuntimeError("GROQ_API_KEY not set in env")
-        _client = Groq(api_key=api_key)
-    return _client
+from .llm import chat_completion_from_messages
 
 
 SYSTEM_PROMPT = """You are a game recommendation assistant. Parse each user message into a JSON object with EXACTLY these fields:
@@ -93,14 +78,12 @@ def extract_slots(query: str, conversation_history: Optional[List[Dict]] = None)
         messages.extend(conversation_history[-6:])
     messages.append({"role": "user", "content": query})
 
-    resp = _get_client().chat.completions.create(
-        model=MODEL,
-        messages=messages,
+    raw = chat_completion_from_messages(
+        messages,
         response_format={"type": "json_object"},
         temperature=0.2,
         max_tokens=400,
-    )
-    raw = resp.choices[0].message.content or "{}"
+    ) or "{}"
     try:
         slots = json.loads(raw)
         if not isinstance(slots, dict):

requirements.txt

@@ -12,6 +12,7 @@ rank-bm25>=0.2.2
 
 # --- LLM ---
 groq>=0.11.0
+google-generativeai>=0.8.0
 python-dotenv>=1.0.0
 
 # --- Web (Phase 2) ---