refactor: replace agent-based web search with parallelized retrieval using Wikipedia and DuckDuckGo

quiz_generator/constants.py

@@ -9,11 +9,11 @@ MAX_SAMPLE_CHUNKS = 10
 RETRIEVER_K = 5
 
 # Web search configuration — Wikipedia is the primary educational source
-WIKI_TOP_K_RESULTS = 3
-WIKI_DOC_CONTENT_CHARS_MAX = 40000   # 3 × 40k = 120k  (broad topic coverage)
+WIKI_TOP_K_RESULTS = 2
+WIKI_DOC_CONTENT_CHARS_MAX = 60000  
 # arXiv adds technical depth as supplementary source
 ARXIV_TOP_K_RESULTS = 1
-ARXIV_DOC_CONTENT_CHARS_MAX = 30000  # 1 × 30k = 30k  → total ≈ 150k
+ARXIV_DOC_CONTENT_CHARS_MAX = 30000 
 
 QUIZ_PROMPT_TEMPLATE = PromptTemplate(
     input_variables=[

quiz_generator/quiz.py

@@ -126,38 +126,61 @@ def _contextual_quiz(difficulty, mcq_count, tf_count, context, material_id):
 def _web_quiz(difficulty, mcq_count, tf_count, topic_title):
     logger.info(f"Web Quiz started (topic={topic_title}, diff={difficulty})")
     try:
+        import concurrent.futures
+        from langchain_community.utilities import WikipediaAPIWrapper, DuckDuckGoSearchAPIWrapper
+
+        def fetch_wikipedia():
+            try:
+                wiki_api = WikipediaAPIWrapper(
+                    top_k_results=WIKI_TOP_K_RESULTS,
+                    doc_content_chars_max=WIKI_DOC_CONTENT_CHARS_MAX,
+                )
+                return wiki_api.run(topic_title)
+            except Exception as e:
+                logger.warning(f"Wikipedia search for '{topic_title}' failed: {e}")
+                return ""
+
+        def fetch_duckduckgo():
+            try:
+                duck_api = DuckDuckGoSearchAPIWrapper()
+                return duck_api.run(topic_title)
+            except Exception as e:
+                logger.warning(f"DuckDuckGo search for '{topic_title}' failed: {e}")
+                return ""
+
+        # Fetch in parallel
+        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+            wiki_future = executor.submit(fetch_wikipedia)
+            duck_future = executor.submit(fetch_duckduckgo)
+
+            wiki_content = wiki_future.result()
+            duck_content = duck_future.result()
+
+        all_content = []
+        if wiki_content and wiki_content.strip():
+            all_content.append(f"--- Wikipedia ---\n{wiki_content}")
+        if duck_content and duck_content.strip():
+            all_content.append(f"--- Web Search ---\n{duck_content}")
+
+        combined_context = "\n\n".join(all_content) if all_content else f"No web content found for: {topic_title}"
+
         prompt = WEB_QUIZ_PROMPT_TEMPLATE
         llm = get_quiz_llm()
-        tools = web_search_tools(
-            wiki_k=WIKI_TOP_K_RESULTS,
-            wiki_chars=WIKI_DOC_CONTENT_CHARS_MAX,
-            arxiv_k=ARXIV_TOP_K_RESULTS,
-            arxiv_chars=ARXIV_DOC_CONTENT_CHARS_MAX,
-        )
-        agent = create_tool_calling_agent(llm, tools, prompt)
 
-        executor = AgentExecutor(
-            agent=agent,
-            tools=tools,
-            verbose=False,
-            return_intermediate_steps=False,
-            handle_parsing_errors=True,
-            max_iterations=80,
-            max_execution_time=300,
-        )
-
-        safe_context = topic_title
-        response = executor.invoke({
+        chain = prompt | llm
+        response = chain.invoke({
             "topic": topic_title,
-            "context": safe_context,
+            "context": combined_context,
             "difficulty": difficulty,
             "mcq_count": mcq_count,
             "tf_count": tf_count,
             "source_type": "Web Search",
             "agent_scratchpad": "",
         })
-        logger.info("Web Quiz agent finished successfully")
-        return _parse_quiz(response)
+
+        raw_content = response.content
+        logger.info("Web Quiz chain finished successfully")
+        return _parse_quiz({"output": raw_content})
     except Exception as e:
         logger.error(f"Web Quiz failed: {str(e)}", exc_info=True)
         raise

summary_generator/constants.py

@@ -4,12 +4,12 @@ MAX_INPUT_CHARS = 150000
 MAX_COMBINED_TEXT_LEN = 160000
 
 # Web search configuration — targets ~150k total (≈ MAX_INPUT_CHARS)
-WIKI_TOP_K_RESULTS = 3
-WIKI_DOC_CONTENT_CHARS_MAX = 40000   # 3 × 40k = 120k  (80% of budget — foundational content)
+WIKI_TOP_K_RESULTS = 2
+WIKI_DOC_CONTENT_CHARS_MAX = 60000  
 
 # arXiv is supplementary — adds depth for technical/research topics
 ARXIV_TOP_K_RESULTS = 1
-ARXIV_DOC_CONTENT_CHARS_MAX = 30000  # 1 × 30k =  30k  → total ≈ 150k
+ARXIV_DOC_CONTENT_CHARS_MAX = 30000 
 
 SUMMARIZER_PROMPT_TEMPLATE = PromptTemplate(
     input_variables=["input"],

summary_generator/summary.py

@@ -1,6 +1,7 @@
 import re
 import logging
-from langchain_community.utilities import ArxivAPIWrapper, WikipediaAPIWrapper
+import concurrent.futures
+from langchain_community.utilities import WikipediaAPIWrapper, DuckDuckGoSearchAPIWrapper
 from src.rag.rag import get_llm
 from .constants import (
     SUMMARIZER_PROMPT_TEMPLATE,
@@ -8,8 +9,6 @@ from .constants import (
     MAX_INPUT_CHARS,
     WIKI_TOP_K_RESULTS,
     WIKI_DOC_CONTENT_CHARS_MAX,
-    ARXIV_TOP_K_RESULTS,
-    ARXIV_DOC_CONTENT_CHARS_MAX,
 )
 
 logger = logging.getLogger(__name__)
@@ -74,27 +73,37 @@ def web_summarizer(topic: str) -> str:
 
     all_content = []
 
-    try:
-        wiki_api = WikipediaAPIWrapper(
-            top_k_results=WIKI_TOP_K_RESULTS,
-            doc_content_chars_max=WIKI_DOC_CONTENT_CHARS_MAX,
-        )
-        wiki_content = wiki_api.run(topic)
-        if wiki_content and wiki_content.strip():
-            all_content.append(f"--- Wikipedia ---\n{wiki_content}")
-    except Exception as e:
-        logger.warning(f"Wikipedia search for '{topic}' failed: {e}")
-
-    try:
-        arxiv_api = ArxivAPIWrapper(
-            top_k_results=ARXIV_TOP_K_RESULTS,
-            doc_content_chars_max=ARXIV_DOC_CONTENT_CHARS_MAX,
-        )
-        arxiv_content = arxiv_api.run(topic)
-        if arxiv_content and arxiv_content.strip():
-            all_content.append(f"--- Arxiv ---\n{arxiv_content}")
-    except Exception as e:
-        logger.warning(f"Arxiv search for '{topic}' failed: {e}")
+    def fetch_wikipedia():
+        try:
+            wiki_api = WikipediaAPIWrapper(
+                top_k_results=WIKI_TOP_K_RESULTS,
+                doc_content_chars_max=WIKI_DOC_CONTENT_CHARS_MAX,
+            )
+            return wiki_api.run(topic)
+        except Exception as e:
+            logger.warning(f"Wikipedia search for '{topic}' failed: {e}")
+            return ""
+
+    def fetch_duckduckgo():
+        try:
+            duck_api = DuckDuckGoSearchAPIWrapper()
+            return duck_api.run(topic)
+        except Exception as e:
+            logger.warning(f"DuckDuckGo search for '{topic}' failed: {e}")
+            return ""
+
+    # Execute searches in parallel to minimize latency
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        wiki_future = executor.submit(fetch_wikipedia)
+        duck_future = executor.submit(fetch_duckduckgo)
+
+        wiki_content = wiki_future.result()
+        duck_content = duck_future.result()
+
+    if wiki_content and wiki_content.strip():
+        all_content.append(f"--- Wikipedia ---\n{wiki_content}")
+    if duck_content and duck_content.strip():
+        all_content.append(f"--- Web Search ---\n{duck_content}")
 
     if not all_content:
         logger.warning(f"No content found for topic: {topic}. Falling back to general knowledge.")