Upload 3 files

retriever.py

@@ -0,0 +1,139 @@
+import os
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import RetrievalQA
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import PromptTemplate
+from langchain.agents import initialize_agent, AgentType
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.tools import Tool
+from langchain.tools import DuckDuckGoSearchRun
+from langchain_core.documents import Document
+
+
+from dotenv import load_dotenv
+import os
+
+load_dotenv()  
+
+apikey = os.getenv("MISTRAL_API_KEY")
+
+llm = ChatOpenAI(
+    openai_api_key=apikey,
+    openai_api_base="https://api.mistral.ai/v1",
+    model="mistral-medium"
+)
+
+prompt_template = PromptTemplate(
+    input_variables=["user_prompt"],
+    template="""You are a retriever agent tasked with creating an efficient search small query to retrieve academic papers from arxiv relevant to a user’s request. Based on the user’s input prompt, generate a concise and precise search query (a string of keywords or phrases) that will be used by the function `retrieve_and_extract_papers(query: str, max_papers: int = 3) -> str` to fetch up to 3 relevant papers. The query should focus on key concepts, avoid ambiguity, and prioritize relevance to ensure the extracted text is suitable for summarization.
+
+User Input Prompt: {user_prompt}
+
+Instructions:
+1. Identify the core concepts, topics, or questions in the user prompt.
+2. Formulate a search query using relevant keywords or short phrases.
+3. Exclude overly broad or irrelevant terms to improve precision.
+4. Output only the search query as a string.
+
+Example:
+- User Prompt: "Recent advancements in large language models for natural language processing"
+- Search Query: "large language models NLP advancements"
+
+Generate the search query for the provided user prompt.
+"""
+)
+
+
+search_tool = DuckDuckGoSearchRun()
+tools = [
+    Tool(
+        name="WebSearch",
+        func=search_tool.run,
+        description="Useful for fetching up-to-date healthcare information from the web.Use it rarely"
+    ),
+]
+
+retriever_agent = initialize_agent(
+    tools=tools,
+    agent_type=AgentType.CONVERSATIONAL_REACT_DESCRIPTION,
+    llm=llm,
+    verbose=True,
+    prompt=prompt_template
+)
+
+
+
+import arxiv
+import pdfplumber
+import requests
+import os
+from typing import List
+embedding_model = HuggingFaceEmbeddings(model_name="BAAI/bge-base-en-v1.5")
+def retrieve_and_extract_papers(query: str, max_papers: int = 3) -> str:
+    """
+    Retrieves research papers from arXiv, downloads PDFs, extracts text,
+    and returns a single string with papers separated by '---pprN: Title'.
+    
+    Args:
+        query: Search query (e.g., "diffusion models 2024")
+        max_papers: Number of papers to retrieve (default: 3)
+    
+    Returns:
+        Single string with extracted text from PDFs, separated by '---pprN: Title'
+    """
+    # Initialize arXiv client
+    client = arxiv.Client()
+    search = arxiv.Search(
+        query=query,
+        max_results=max_papers,
+        sort_by=arxiv.SortCriterion.Relevance
+    )
+    
+    papers = list(client.results(search))
+    if not papers:
+        return "No papers found for the query."
+    
+    # Create temporary directory for PDFs
+    temp_dir = "temp_papers"
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    # Process each paper and collect extracted text
+    formatted_texts = []
+    for i, paper in enumerate(papers, 1):
+        try:
+            # Download PDF
+            pdf_url = paper.pdf_url
+            response = requests.get(pdf_url)
+            pdf_path = os.path.join(temp_dir, f"paper_{i}.pdf")
+            
+            with open(pdf_path, 'wb') as f:
+                f.write(response.content)
+            
+            # Extract text from PDF
+            text = ""
+            with pdfplumber.open(pdf_path) as pdf:
+                for page in pdf.pages:
+                    page_text = page.extract_text()
+                    if page_text:
+                        text += page_text + "\n"
+            
+            # Append formatted text with separator including paper title
+            separator = f"---ppr{i}: {paper.title}"
+            formatted_texts.append(f"{separator}\n{text}")
+            
+            # Clean up: Remove the downloaded PDF
+            os.remove(pdf_path)
+            
+        except Exception as e:
+            print(f"Error processing paper {i} ({paper.title}): {e}")
+            formatted_texts.append(f"---ppr{i}: {paper.title}\nError: Could not process paper.")
+    
+    # Clean up: Remove temporary directory if empty
+    if os.path.exists(temp_dir) and not os.listdir(temp_dir):
+        os.rmdir(temp_dir)
+    
+    # Join all texts into a single string
+    return "\n".join(formatted_texts)
+

summarizer.py

@@ -0,0 +1,74 @@
+import os
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import RetrievalQA
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import PromptTemplate
+from langchain.agents import initialize_agent, AgentType
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.tools import Tool
+from langchain.tools import DuckDuckGoSearchRun
+from langchain_core.documents import Document
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from dotenv import load_dotenv
+
+load_dotenv()  
+
+apikey = os.getenv("MISTRAL_API_KEY")
+
+embedding_model = HuggingFaceEmbeddings(model_name="BAAI/bge-base-en-v1.5")
+faiss_index = FAISS.load_local("faiss_index", embedding_model, allow_dangerous_deserialization=True)
+retriever = faiss_index.as_retriever(search_kwargs={"k": 3})
+
+llm = ChatOpenAI(
+    openai_api_key=apikey,
+    openai_api_base="https://api.mistral.ai/v1",
+    model="mistral-medium"
+)
+
+from langchain.prompts import PromptTemplate
+
+prompt_template = PromptTemplate(
+    input_variables=["user_prompt", "retriever_query"],
+    template="""You are a summarizer agent tasked with generating a concise summary of academic papers retrieved from a RAG pipeline. Using the user’s prompt and the retriever query used for searching, fetch relevant document content with the RAG tool and summarize the key points, findings, or insights relevant to the user’s request.
+
+User Input Prompt: {user_prompt}
+Retriever Query: {retriever_query}
+
+Instructions:
+1. Use the RAG tool to retrieve document content based on the retriever query.
+2. Analyze the user prompt to identify the focus or specific aspects of interest.
+3. Summarize the main ideas, results, or trends from the retrieved documents, excluding unnecessary details.
+4. Ensure the summary is clear, coherent, and no longer than 1500 words.
+5. If the RAG tool returns irrelevant or insufficient documents, note this briefly and provide a general response based on the prompt.
+6. Output only the summary as a string.
+
+Example:
+- User Prompt: "Recent advancements in large language models for natural language processing"
+- Retriever Query: "large language models NLP advancements"
+- Summary: "Recent advancements in large language models (LLMs) focus on improved efficiency and performance in NLP tasks. Techniques like fine-tuning and transformer architectures enhance accuracy in text generation and understanding."
+
+Generate the summary for the provided user prompt and retriever query.
+"""
+)
+search_tool = DuckDuckGoSearchRun()
+
+tools = [
+    Tool(
+    name="FAISSRetriever",
+    func=lambda query: "\n\n".join([doc.page_content for doc in retriever.invoke(query)]),
+    description="Fetches the top 3 relevant document chunks from a FAISS index containing academic paper content based on a query. Use for retrieving scholarly text for summarization."
+    ),
+]
+summarizer_agent = initialize_agent(
+    tools=tools,
+    agent_type=AgentType.CONVERSATIONAL_REACT_DESCRIPTION,
+    llm=llm,
+    verbose=True,
+    prompt=prompt_template,
+    retriever=retriever
+)
+
+

synthesizer.py

@@ -0,0 +1,90 @@
+import os
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import RetrievalQA
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import PromptTemplate
+from langchain.agents import initialize_agent, AgentType
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.tools import Tool
+from langchain.tools import DuckDuckGoSearchRun
+from langchain_core.documents import Document
+
+from dotenv import load_dotenv
+
+load_dotenv()  
+
+apikey = os.getenv("MISTRAL_API_KEY")
+
+llm = ChatOpenAI(
+    openai_api_key=apikey,
+    openai_api_base="https://api.mistral.ai/v1",
+    model="mistral-large-2411"
+)
+
+from langchain.prompts import PromptTemplate
+
+prompt_template = PromptTemplate(
+    input_variables=["user_prompt", "summaries", "critiques"],
+    template="""You are a Synthesizer Agent assisting in generating a structured research overview for a given topic.
+You are provided with:
+1. The user’s original prompt.
+2. A set of summaries extracted from multiple research papers.
+3. Critique analysis, including commentary on contradictions, outdated information, and biased claims, along with associated critique scores (from 1 to 5, where 5 is most critical).
+
+Your task is to:
+- Analyze the summaries to extract core findings and organize them into a clear, topic-wise outline.
+- Incorporate the critiques along with their scores in a dedicated section to highlight research limitations or cautionary notes.
+
+User Prompt:
+{user_prompt}
+
+Paper Summaries:
+{summaries}
+
+Critique Analysis (with scores):
+{critiques}
+
+Instructions:
+1. Begin with a concise introduction to the research topic based on the user prompt.
+2. Generate a structured outline capturing key themes, methodologies, findings, and notable contributions.
+3. Follow up with a \"Critical Reflections\" section that includes:
+   - A list of critical observations.
+   - Their corresponding critique scores.
+   - Any specific recommendations or warnings.
+4. Maintain a professional and academic tone throughout.
+5. The final response should be suitable for inclusion in a research literature review or academic discussion.
+
+Output Format:
+- **Introduction**
+- **Outline**
+  - Theme 1: ...
+  - Theme 2: ...
+- **Critical Reflections**
+  - Observation 1 (Score: X): ...
+  - Observation 2 (Score: X): ...
+  - Outdated or Contradictory Claims (if any): ...
+  - Recommendations: ...
+
+Now generate the structured synthesis below:
+"""
+)
+
+search_tool = DuckDuckGoSearchRun()
+
+tools = [
+    Tool(
+        name="WebSearch",
+        func=DuckDuckGoSearchRun().run,
+        description="Fetches up-to-date information from the web. Use for verifying claims or finding additional context."
+    ),
+]
+
+synthesizer_agent = initialize_agent(
+    tools=tools,
+    agent_type=AgentType.CONVERSATIONAL_REACT_DESCRIPTION,
+    llm=llm,
+    verbose=True,
+    prompt=prompt_template
+)