Add multi-key Groq rotation, session cleanup, smarter FAISS chunking

app.py

@@ -27,16 +27,21 @@ from langchain_core.prompts import ChatPromptTemplate
 load_dotenv()
 
 # ── Configuration ─────────────────────────────────────────────────────────────
-GROQ_API_KEY  = os.getenv("GROQ_API_KEY", "").strip()
-GROQ_KEYS     = [GROQ_API_KEY] if GROQ_API_KEY else []
+GROQ_KEYS = [k.strip() for k in [
+    os.getenv("GROQ_API_KEY",   ""),
+    os.getenv("GROQ_API_KEY_2", ""),
+    os.getenv("GROQ_API_KEY_3", ""),
+] if k.strip()]
 BUSINESS_NAME = os.getenv("BUSINESS_NAME", "Vibhu Solutions")
 LLM_MODEL     = os.getenv("LLM_MODEL", "llama-3.1-8b-instant")
 FAISS_DIR     = "./faiss_db"
 EMBED_MODEL   = "sentence-transformers/all-MiniLM-L6-v2"
-MAX_HISTORY   = 8   # last 8 messages (~4 exchanges) per session
+MAX_HISTORY   = 8    # last 8 messages (~4 exchanges) per session
+MAX_SESSIONS  = 500  # evict oldest sessions beyond this limit
 
 # In-memory session store  {session_id: [{"role": "user"|"assistant", "content": "..."}]}
 session_memory: Dict[str, List[Dict[str, str]]] = {}
+_active_key_i: int = 0
 
 # ── Agent System Prompt (system message only — question goes in human message)
 SYSTEM_PROMPT = """\
@@ -107,7 +112,15 @@ async def lifespan(app: FastAPI):
     yield
 
 
-def _build_components(key_index: int = 0):
+def _make_llm() -> "ChatGroq":
+    return ChatGroq(
+        groq_api_key=GROQ_KEYS[_active_key_i % len(GROQ_KEYS)],
+        model_name=LLM_MODEL,
+        temperature=0.2,
+    )
+
+
+def _build_components():
     if not GROQ_KEYS:
         raise ValueError("GROQ_API_KEY not set. Add it to your .env file.")
     if not Path(FAISS_DIR).exists():
@@ -124,12 +137,8 @@ def _build_components(key_index: int = 0):
         FAISS_DIR, embeddings, allow_dangerous_deserialization=True
     )
     retriever = vectorstore.as_retriever(search_kwargs={"k": 5})
-    llm       = ChatGroq(
-        groq_api_key=GROQ_KEYS[key_index % len(GROQ_KEYS)],
-        model_name=LLM_MODEL,
-        temperature=0.2,
-    )
-    prompt = ChatPromptTemplate.from_messages([
+    llm       = _make_llm()
+    prompt    = ChatPromptTemplate.from_messages([
         ("system", SYSTEM_PROMPT),
         ("human", "{question}"),
     ])
@@ -207,7 +216,7 @@ def health():
 
 @app.post("/chat", response_model=ChatResponse)
 def chat(request: ChatRequest):
-    global _retriever, _llm, _prompt
+    global _retriever, _llm, _prompt, _active_key_i
     if not request.message.strip():
         raise HTTPException(status_code=400, detail="Message cannot be empty.")
 
@@ -216,7 +225,7 @@ def chat(request: ChatRequest):
         try:
             _retriever, _llm, _prompt = _build_components()
             print(f"✅  {BUSINESS_NAME} AI Agent is ready (loaded on first request).")
-        except (ValueError, FileNotFoundError) as e:
+        except (ValueError, FileNotFoundError):
             return ChatResponse(
                 reply=(
                     "The AI Agent is not configured yet. "
@@ -235,15 +244,12 @@ def chat(request: ChatRequest):
 
         # Build user-context block (used by RULE 4 to skip re-asking known details)
         ctx_parts = []
-        if request.user_name:
-            ctx_parts.append(f"Name  : {request.user_name}")
-        if request.user_email:
-            ctx_parts.append(f"Email : {request.user_email}")
-        if request.user_phone:
-            ctx_parts.append(f"Phone : {request.user_phone}")
+        if request.user_name:  ctx_parts.append(f"Name  : {request.user_name}")
+        if request.user_email: ctx_parts.append(f"Email : {request.user_email}")
+        if request.user_phone: ctx_parts.append(f"Phone : {request.user_phone}")
         user_ctx = "\n".join(ctx_parts) if ctx_parts else "Not provided"
 
-        # Format prompt and invoke LLM
+        # Format prompt
         formatted = _prompt.format_messages(
             business_name=BUSINESS_NAME,
             context=context,
@@ -251,13 +257,30 @@ def chat(request: ChatRequest):
             history=_format_history(history),
             user_context=user_ctx,
         )
-        reply = _llm.invoke(formatted).content
+
+        # Invoke LLM — auto-rotate key on rate limit (if multiple keys available)
+        try:
+            reply = _llm.invoke(formatted).content
+        except Exception as llm_err:
+            err_str = str(llm_err)
+            if ("rate_limit_exceeded" in err_str or "429" in err_str) and len(GROQ_KEYS) > 1:
+                _active_key_i = (_active_key_i + 1) % len(GROQ_KEYS)
+                _llm = _make_llm()
+                print(f"⚠️  Rate limit hit — rotated to Groq key #{_active_key_i + 1}")
+                reply = _llm.invoke(formatted).content
+            else:
+                raise
 
         # Persist history (capped at MAX_HISTORY messages)
         history.append({"role": "user",      "content": request.message.strip()})
         history.append({"role": "assistant", "content": reply})
         session_memory[request.session_id] = history[-MAX_HISTORY:]
 
+        # Evict oldest sessions if memory grows too large
+        if len(session_memory) > MAX_SESSIONS:
+            oldest = next(iter(session_memory))
+            del session_memory[oldest]
+
         return ChatResponse(reply=reply, session_id=request.session_id)
 
     except Exception as e:
@@ -267,8 +290,8 @@ def chat(request: ChatRequest):
         if "rate_limit_exceeded" in err or "429" in err:
             return ChatResponse(
                 reply=(
-                    "Our AI assistant is temporarily busy due to high demand. "
-                    "Please try again in a few minutes.\n\n"
+                    "Our AI assistant is temporarily busy. "
+                    "Please try again in a moment.\n\n"
                     "Or contact us directly:\n"
                     "📧 contact@vibhusolutions.com\n"
                     "📞 +91 9380345108"

ingest.py

@@ -61,7 +61,11 @@ def main():
     print(f"Loaded {len(docs)} document(s).")
 
     print("Splitting into chunks...")
-    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=800,
+        chunk_overlap=100,
+        separators=["\n\nQ:", "\n\n---\n", "\n\n", "\n", " ", ""],
+    )
     chunks = splitter.split_documents(docs)
     print(f"Created {len(chunks)} chunks.")