feat: improve chatbot search quality, implement refusal filtering.

rag/constants.py

@@ -7,8 +7,10 @@ WARMUP_INTERVAL_S = 300
 # Web search configuration — Wiki + DDG for topics, DDG only for PDF/URL materials.
 WIKI_TOP_K_RESULTS = 1
 WIKI_DOC_CONTENT_CHARS_MAX = 1200    # max chars from Wikipedia result
-DUCKDUCKGO_NUM_RESULTS = 3           # number of DDG snippet results returned per search
-DUCKDUCKGO_DOC_CONTENT_CHARS_MAX = 1200  # max chars kept from the combined DDG result block
+
+# DuckDuckGO Search
+DUCKDUCKGO_NUM_RESULTS = 5           # number of DDG snippet results returned per search
+DUCKDUCKGO_DOC_CONTENT_CHARS_MAX = 3000  # max chars kept from the combined DDG result block
 
 RAG_PROMPT_TEMPLATE_BASE = """\
 <role>

rag/rag.py

@@ -337,7 +337,10 @@ def rag_answer(
             context_parts.append(f"Wikipedia Results:\n{wiki_snippets}")
 
     # --- DuckDuckGo search: ALL material types (topics, PDFs, URLs) ---
-    ddg_snippets = direct_ddg_search(query)
+    # Enrich the search query with the subject title so follow-up / short
+    subject_title = mat.get("title") if mat and mat.get("title") else ""
+    ddg_query = f"{query} {subject_title}".strip() if subject_title else query
+    ddg_snippets = direct_ddg_search(ddg_query)
     if ddg_snippets:
         context_parts.append(f"Web Search Results (DuckDuckGo):\n{ddg_snippets}")
 
@@ -352,6 +355,18 @@ def rag_answer(
         subject=subject_title,
     )
 
+    # Safety/refusal responses must NOT be saved to memory, otherwise the
+    _REFUSAL_PREFIXES = (
+        "I can't respond on a gibberish",
+        "I can't respond on a NSFW",
+        "I can't respond on a political",
+        "I can't respond on a religious",
+    )
+
+    def _is_refusal(text: str) -> bool:
+        t = text.strip()
+        return any(t.startswith(p) for p in _REFUSAL_PREFIXES)
+
     try:
         # All queries now use the simple LLM call (no agent loop).
         # DDG results are pre-fetched and injected into context above.
@@ -368,7 +383,9 @@ def rag_answer(
         })
 
         answer = response.content
-        memory.save_context({"input": query}, {"output": answer})
+        # Only persist non-refusal answers to memory
+        if not _is_refusal(answer):
+            memory.save_context({"input": query}, {"output": answer})
         return answer, memory
     except Exception as e:
         logger.warning(f"Groq API call failed or rate-limited: {e}. Falling back to google/gemma-4-31b-it immediately.")
@@ -385,7 +402,9 @@ def rag_answer(
                 "agent_scratchpad": "",
             })
             answer = response.content
-            memory.save_context({"input": query}, {"output": answer})
+            # Only persist non-refusal answers to memory
+            if not _is_refusal(answer):
+                memory.save_context({"input": query}, {"output": answer})
             return answer, memory
         except Exception as fallback_err:
             logger.error(f"Fallback LLM call also failed: {fallback_err}")

rag/routes.py

@@ -94,8 +94,17 @@ async def ask_tutor(
     cleaned_answer = clean_summary(answer)
     elapsed = time.time() - start
 
-    # Persist assistant response
-    if body.session_id:
+    # Safety/refusal responses must not be saved to DB history either.
+    _REFUSAL_PREFIXES = (
+        "I can't respond on a gibberish",
+        "I can't respond on a NSFW",
+        "I can't respond on a political",
+        "I can't respond on a religious",
+    )
+    is_refusal = any(cleaned_answer.strip().startswith(p) for p in _REFUSAL_PREFIXES)
+
+    # Persist assistant response (only if not a refusal)
+    if body.session_id and not is_refusal:
         try:
             append_session_message(body.session_id, "assistant", cleaned_answer)
         except Exception: