critique.py

import os
from langchain.chat_models import ChatOpenAI
from langchain.chains import RetrievalQA
from langchain.memory import ConversationBufferMemory
from langchain.prompts import PromptTemplate
from langchain.agents import initialize_agent, AgentType
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings.huggingface import HuggingFaceEmbeddings
from langchain.vectorstores import FAISS
from langchain.tools import Tool
from langchain.tools import DuckDuckGoSearchRun
from langchain_core.documents import Document

from dotenv import load_dotenv

load_dotenv()  

apikey = os.getenv("MISTRAL_API_KEY")

llm = ChatOpenAI(
    openai_api_key=apikey,
    openai_api_base="https://api.mistral.ai/v1",
    model="mistral-large-2411"
)


embedding_model = HuggingFaceEmbeddings(model_name="BAAI/bge-base-en-v1.5")
faiss_index = FAISS.load_local("faiss_index", embedding_model, allow_dangerous_deserialization=True)
retriever = faiss_index.as_retriever(search_kwargs={"k": 3})

tools = [
    Tool(
        name="FAISSRetriever",
        func=lambda query: "\n\n".join([doc.page_content for doc in retriever.invoke(query)]),
        description="Fetches the top 3 relevant document chunks from a FAISS index containing academic paper content based on a query. Use for scholarly text analysis."
    ),
    Tool(
        name="WebSearch",
        func=DuckDuckGoSearchRun().run,
        description="Fetches up-to-date information from the web. Use for verifying claims or finding additional context."
    ),
]


prompt_template = PromptTemplate(
    input_variables=["user_query", "summary"],
    template="""You are a critique agent tasked with strictly evaluating a summary of academic papers based on a user query. Using the provided user query and summary, analyze the content for accuracy, completeness, and biases. Use the FAISSRetriever and WebSearch tools to cross-reference information and validate claims. Provide a detailed critique, including recommendations, identified biases, and a relevance rating (1–5, where 5 is highly relevant and accurate).

User Query: {user_query}
Summary: {summary}

Instructions:
1. Use the FAISSRetriever tool to fetch document chunks related to the user query for additional context from academic papers.
2. Use the WebSearch tool to verify claims in the summary or find recent developments.
3. Evaluate the summary for:
   - Accuracy: Are claims supported by evidence from the tools?
   - Completeness: Does it cover key aspects of the user query?
   - Biases: Identify methodological, dataset, author, or other biases (e.g., overemphasis on positive results, lack of diverse perspectives).
4. Provide recommendations to improve the summary (e.g., additional topics, clearer explanations).
5. Assign a relevance rating (1–5) based on how well the summary addresses the user query and its reliability.
6. Be strict in identifying biases and unsupported claims.
7. Output a structured response with sections: Critique, Recommendations, Biases, Relevance Rating.

Example:
- User Query: "Advancements in diffusion models for image generation"
- Summary: "Diffusion models outperform GANs in image quality."
- Output:
  Critique: The summary claims diffusion models outperform GANs but lacks evidence or metrics.
  Recommendations: Include specific metrics (e.g., FID scores) and compare computational costs.
  Biases: Potential bias toward diffusion models; ignores GANs’ strengths in training speed.
  Relevance Rating: 2/5 (lacks depth and evidence).

Generate the critique for the provided user query and summary.
"""
)

critique_agent = initialize_agent(
    tools=tools,
    agent_type=AgentType.CONVERSATIONAL_REACT_DESCRIPTION,
    llm=llm,
    verbose=True,
    prompt=prompt_template,
)