Update agent.py

agent.py

@@ -1,213 +1,353 @@
-"""LangGraph Agent"""
+
 import os
-from dotenv import load_dotenv
-from langgraph.graph import START, StateGraph, MessagesState
-from langgraph.prebuilt import tools_condition
-from langgraph.prebuilt import ToolNode
-from langchain_google_genai import ChatGoogleGenerativeAI
-from langchain_groq import ChatGroq
-from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFaceEmbeddings
-from langchain_community.tools.tavily_search import TavilySearchResults
-from langchain_community.document_loaders import WikipediaLoader
-from langchain_community.document_loaders import ArxivLoader
-from langchain_community.vectorstores import SupabaseVectorStore
-from langchain_core.messages import SystemMessage, HumanMessage
-from langchain_core.tools import tool
-from langchain.tools.retriever import create_retriever_tool
-from supabase.client import Client, create_client
-
-load_dotenv()
-
-@tool
-def multiply(a: int, b: int) -> int:
-    """Multiply two numbers.
-    Args:
-        a: first int
-        b: second int
-    """
-    return a * b
-
-@tool
-def add(a: int, b: int) -> int:
-    """Add two numbers.
-    
-    Args:
-        a: first int
-        b: second int
-    """
-    return a + b
-
-@tool
-def subtract(a: int, b: int) -> int:
-    """Subtract two numbers.
-    
-    Args:
-        a: first int
-        b: second int
-    """
-    return a - b
-
-@tool
-def divide(a: int, b: int) -> int:
-    """Divide two numbers.
-    
-    Args:
-        a: first int
-        b: second int
-    """
-    if b == 0:
-        raise ValueError("Cannot divide by zero.")
-    return a / b
-
-@tool
-def modulus(a: int, b: int) -> int:
-    """Get the modulus of two numbers.
-    
-    Args:
-        a: first int
-        b: second int
-    """
-    return a % b
-
-@tool
-def wiki_search(query: str) -> str:
-    """Search Wikipedia for a query and return maximum 2 results.
-    
-    Args:
-        query: The search query."""
-    search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"wiki_results": formatted_search_docs}
-
-@tool
-def web_search(query: str) -> str:
-    """Search Tavily for a query and return maximum 3 results.
-    
-    Args:
-        query: The search query."""
-    search_docs = TavilySearchResults(max_results=3).invoke(query=query)
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"web_results": formatted_search_docs}
-
-@tool
-def arvix_search(query: str) -> str:
-    """Search Arxiv for a query and return maximum 3 result.
-    
-    Args:
-        query: The search query."""
-    search_docs = ArxivLoader(query=query, load_max_docs=3).load()
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"arvix_results": formatted_search_docs}
-
-
-
-# load the system prompt from the file
-with open("system_prompt.txt", "r", encoding="utf-8") as f:
-    system_prompt = f.read()
-
-# System message
-sys_msg = SystemMessage(content=system_prompt)
-
-# build a retriever
-embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") #  dim=768
-supabase: Client = create_client(
-    os.environ.get("SUPABASE_URL"), 
-    os.environ.get("SUPABASE_SERVICE_KEY"))
-vector_store = SupabaseVectorStore(
-    client=supabase,
-    embedding= embeddings,
-    table_name="documents",
-    query_name="match_documents_langchain",
-)
-create_retriever_tool = create_retriever_tool(
-    retriever=vector_store.as_retriever(),
-    name="Question Search",
-    description="A tool to retrieve similar questions from a vector store.",
-)
-
-
-
-tools = [
-    multiply,
-    add,
-    subtract,
-    divide,
-    modulus,
-    wiki_search,
-    web_search,
-    arvix_search,
-]
-
-# Build graph function
-def build_graph(provider: str = "groq"):
-    """Build the graph"""
-    # Load environment variables from .env file
-    if provider == "google":
-        # Google Gemini
-        llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0)
-    elif provider == "groq":
-        # Groq https://console.groq.com/docs/models
-        llm = ChatGroq(model="qwen-qwq-32b", temperature=0) # optional : qwen-qwq-32b gemma2-9b-it
-    elif provider == "huggingface":
-        # TODO: Add huggingface endpoint
-        llm = ChatHuggingFace(
-            llm=HuggingFaceEndpoint(
-                url="https://api-inference.huggingface.co/models/Meta-DeepLearning/llama-2-7b-chat-hf",
-                temperature=0,
-            ),
+import time
+from langchain.tools import Tool, tool
+from typing import Tuple, List
+from typing_extensions import TypedDict, Annotated, Optional
+from langgraph.graph.message import add_messages
+from langchain_core.messages import AnyMessage, HumanMessage, AIMessage, SystemMessage
+from langgraph.graph import START, StateGraph, END
+from langgraph.prebuilt import ToolNode, tools_condition
+from langchain_litellm import ChatLiteLLM
+from IPython.display import Image, display
+import asyncio
+from tools import (search_tool,
+                    download_tool,
+                    get_web_page,
+                    add,
+                    subtract,
+                    multiply,
+                    divide,
+                    power,
+                    square_root,
+                    get_information_from_wikipedia,
+                    get_information_from_arxiv,
+                    get_information_from_youtube,
+                    python_tool,
+                    get_information_from_json,
+                    get_information_from_audio,
+                    get_information_from_xml,
+                    get_information_from_docx,
+                    get_information_from_txt,
+                    get_information_from_pdf,
+                    get_information_from_csv,
+                    get_information_from_excel,
+                    get_information_from_pdb,
+                    get_information_from_image,
+                    get_information_from_pptx,
+                    get_all_files_from_zip,
+                    get_information_from_python)
+
+DELAY = 5
+TIME_SLEEP = 60/15 + DELAY
+
+GEMINI_API_KEY_1 = os.getenv("GOOGLE_API_KEY_1")
+GEMINI_API_KEY_2 = os.getenv("GOOGLE_API_KEY_2")
+GEMINI_API_KEY_3 = os.getenv("GOOGLE_API_KEY_3")
+
+chat_model_1 = ChatLiteLLM(model="gemini/gemini-2.0-flash",
+                         temperature=0,
+                         api_key=GEMINI_API_KEY_1,
+                         max_retries=10,
+                         verbose=True)
+
+chat_model_2 = ChatLiteLLM(model="gemini/gemini-2.0-flash",
+                         temperature=0,
+                         api_key=GEMINI_API_KEY_2,
+                         max_retries=10,
+                         verbose=True)
+
+chat_model_3 = ChatLiteLLM(model="gemini/gemini-2.0-flash",
+                         temperature=0,
+                         api_key=GEMINI_API_KEY_3,
+                         max_retries=10,
+                         verbose=True)
+
+class AgentState(TypedDict):
+    messages: Annotated[list[AnyMessage], add_messages]
+    question: Optional[str]
+    file_path: Optional[str]
+    task_id: Optional[str]
+    new_messages: Optional[int]
+    final_answer: Optional[str]
+    attempt: Optional[int]
+    chat_model: Optional[int]
+
+class MyAgent:
+    def __init__(self, web_tools=None):
+        print("MyAgent initialized.")
+
+        self.chat_1 = chat_model_1
+        self.chat_2 = chat_model_2
+        self.chat_3 = chat_model_3
+
+        self.tools = [search_tool,
+                    download_tool,
+                    get_web_page,
+                    add,
+                    subtract,
+                    multiply,
+                    divide,
+                    power,
+                    square_root,
+                    get_information_from_wikipedia,
+                    get_information_from_arxiv,
+                    get_information_from_youtube,
+                    python_tool,
+                    get_information_from_json,
+                    get_information_from_audio,
+                    get_information_from_xml,
+                    get_information_from_docx,
+                    get_information_from_txt,
+                    get_information_from_pdf,
+                    get_information_from_csv,
+                    get_information_from_excel,
+                    get_information_from_pdb,
+                    get_information_from_image,
+                    get_information_from_pptx,
+                    get_all_files_from_zip] + web_tools
+
+        self.chat_with_tools_1 = self.chat_1.bind_tools(self.tools, verbose=True)
+        self.chat_with_tools_2 = self.chat_2.bind_tools(self.tools, verbose=True)
+        self.chat_with_tools_3 = self.chat_3.bind_tools(self.tools, verbose=True)
+        self.chats = [self.chat_with_tools_1, self.chat_with_tools_2, self.chat_with_tools_3]
+
+        self.builder = StateGraph(AgentState)
+        self.builder.add_node("assistant", self.assistant)
+        self.builder.add_node("tools", ToolNode(self.tools))
+        self.builder.add_node("extract_data_from_file", self.extract_data_from_file)
+        self.builder.add_node("postprocess", self.postprocess)
+
+        self.builder.add_edge(START, "extract_data_from_file")
+        self.builder.add_edge("extract_data_from_file", "assistant")
+        self.builder.add_conditional_edges(
+            "assistant",
+            self.assistant_router,
+            {
+                "tools": "tools",
+                "postprocess": "postprocess"
+            }
         )
-    else:
-        raise ValueError("Invalid provider. Choose 'google', 'groq' or 'huggingface'.")
-    # Bind tools to LLM
-    llm_with_tools = llm.bind_tools(tools)
-
-    # Node
-    def assistant(state: MessagesState):
-        """Assistant node"""
-        return {"messages": [llm_with_tools.invoke(state["messages"])]}
-    
-    def retriever(state: MessagesState):
-        """Retriever node"""
-        similar_question = vector_store.similarity_search(state["messages"][0].content)
-        example_msg = HumanMessage(
-            content=f"Here I provide a similar question and answer for reference: \n\n{similar_question[0].page_content}",
+        self.builder.add_edge("tools", "assistant")
+        self.builder.add_conditional_edges(
+            "postprocess",
+            self.answer_evaluation,
+            {
+                "RETRY": "assistant",
+                "END": END
+            }
         )
-        return {"messages": [sys_msg] + state["messages"] + [example_msg]}
-
-    builder = StateGraph(MessagesState)
-    builder.add_node("retriever", retriever)
-    builder.add_node("assistant", assistant)
-    builder.add_node("tools", ToolNode(tools))
-    builder.add_edge(START, "retriever")
-    builder.add_edge("retriever", "assistant")
-    builder.add_conditional_edges(
-        "assistant",
-        tools_condition,
-    )
-    builder.add_edge("tools", "assistant")
-
-    # Compile graph
-    return builder.compile()
-
-# test
-if __name__ == "__main__":
-    question = "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia."
-    # Build the graph
-    graph = build_graph(provider="groq")
-    # Run the graph
-    messages = [HumanMessage(content=question)]
-    messages = graph.invoke({"messages": messages})
-    for m in messages["messages"]:
-        m.pretty_print()
+
+        self.agent = self.builder.compile()
+
+    async def __call__(self, question: str, file_path: str, task_id: str) -> str:
+        print("\033[1m\033[93m"+"="*150+"\033[0m")
+        print(f"QUESTION: {question}")
+        print(f"File: {file_path}")
+        prompt = f"""You are a general AI assistant. You will receive a user question and extracted data from associated files.
+Follow this process:
+1. Identify the required output type (e.g., number, string, list) and key concepts in the question.
+2. Before using any tools, check if the answer can be deduced or recalled directly. If yes, answer immediately. Never guess.
+3. If tools are needed:
+   - Create a plan with:
+     - The reasoning approach and tool sequence.
+     - A rephrased version of the question optimized for search engines (DuckDuckGo or Google).
+     - Search queries must:
+       - Be keyword-focused (avoid full sentences).
+       - Use advanced operators if helpful: `site:` for domains, `inurl:` for internal paths, `filetype:` for formats.
+       - Avoid punctuation, commas, quotes, or special characters.
+       - Cover multiple query angles if needed.
+4. Do not run any tool until the plan is complete.
+5. If a tool fails or returns no useful result:
+   - Reformulate the query with synonyms or tighter context.
+   - Retry or use a fallback tool.
+6. Analyze tool results carefully. If multiple source links appear, use `navigate_browser` to explore and extract relevant information from each.
+Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER].
+YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings.
+If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise.
+If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise.
+If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string."""
+
+        user_message = f"""Question: {question}
+Filepath: {file_path}"""
+
+        messages = [SystemMessage(content=prompt, name="SYSTEM"),
+                    HumanMessage(content=user_message, name="USER")]
+
+        response = await self.agent.ainvoke({"messages": messages,
+                                             "question": question,
+                                             "file_path": file_path,
+                                             "task_id": task_id,
+                                             "new_messages": 1,
+                                             "chat_model": 0,
+                                             "final_answer": "",
+                                             "attempt": 0},
+                                              {"recursion_limit": 100})
+
+        print("\033[1m\033[93m"+"="*150+"\033[0m")
+        return response['final_answer']
+
+    async def call_chat(self, chat, state: AgentState, max_retries=5):
+        from google.api_core.exceptions import GoogleAPICallError
+        for i in range(max_retries):
+            try:
+                return await chat.ainvoke(state["messages"])
+            except GoogleAPICallError as e:
+                if "503" in str(e) or "UNAVAILABLE" in str(e):
+                    wait = 2 ** i
+                    print(f"[Gemini] Overloaded (attempt {i+1}), retrying in {wait:.1f}s...")
+                    await asyncio.sleep(wait)
+                else:
+                    raise e
+        raise RuntimeError("Gemini failed after multiple retries")
+
+    async def assistant(self, state: AgentState):
+        new_messages = state["new_messages"]
+
+        for i in reversed(range(1, new_messages+1)):
+            print("\033[1m\033[92m"+"+"*150+"\033[0m")
+            name = state["messages"][-i].name
+            content = state["messages"][-i].content
+            print(f'\033[1m\033[96m{name}\033[0m: {content if len(content) < 5000 else content[:5000]}')
+
+        chat = self.chats[state["chat_model"]]
+        result = await self.call_chat(chat=chat, state=state)
+
+        state["chat_model"] += 1
+        state["chat_model"] %= len(self.chats)
+
+        result.name="ASSISTANT"
+        await asyncio.sleep(TIME_SLEEP)
+
+        print("\033[1m\033[92m"+"+"*150+"\033[0m")
+        content = result.content[:-2] if result.content[-2:] == '\n\n' else result.content
+        print(f'\033[1m\033[96m{result.name}\033[0m: {content}')
+        state["new_messages"] = 1
+        state["messages"].append(result)
+
+        return state
+
+    def extract_data_from_file(self, state: AgentState) -> str:
+        path = state["file_path"]
+        new_messages = state["new_messages"]
+        prompt = ""
+        messages = []
+
+        if path and "." in path:
+            ext = path.strip().split(".")[-1].lower()
+            print(f"Extension detected: {ext}")
+
+            if ext == "zip":
+                files, prompt = get_all_files_from_zip(path)
+                name = "get_all_file_from_zip"
+                messages.append(AIMessage(content=prompt, name=name))
+            else:
+                files = [path]
+
+            for file_path in files:
+                ext = file_path.strip().split(".")[-1].lower()
+                print(f"Extension detected: {ext}")
+
+                prompt = f"Information extracted from {file_path}.\n\n"
+                match ext:
+                    case "csv":
+                        content = get_information_from_csv.invoke(file_path)
+                        name = "get_information_from_csv"
+                    case "txt":
+                        content = get_information_from_txt.invoke(file_path)
+                        name = "get_information_from_txt"
+                    case "pdf":
+                        content = get_information_from_pdf.invoke(file_path)
+                        name = "get_information_from_pdf"
+                    case "json":
+                        content = get_information_from_json.invoke(file_path)
+                        name = "get_information_from_json"
+                    case "jsonld":
+                        content = get_information_from_json.invoke(file_path)
+                        name = "get_information_from_json"
+                    case "xml":
+                        content = get_information_from_xml.invoke(file_path)
+                        name = "get_information_from_xml"
+                    case "pdb":
+                        content = get_information_from_pdb.invoke(file_path)
+                        name = "get_information_from_pdb"
+                    case "mp3":
+                        content = get_information_from_audio.invoke(file_path)
+                        name = "get_information_from_audio"
+                    case "m4a":
+                        content = get_information_from_audio.invoke(file_path)
+                        name = "get_information_from_audio"
+                    case "docx":
+                        content = get_information_from_docx.invoke(file_path)
+                        name = "get_information_from_docx"
+                    case "xlsx":
+                        content = get_information_from_excel.invoke(file_path)
+                        name = "get_information_from_excel"
+                    case "xls":
+                        content = get_information_from_excel.invoke(file_path)
+                        name = "get_information_from_excel"
+                    case "png":
+                        content = get_information_from_image.invoke({"file_path": file_path, "question": state["question"]})
+                        name = "get_information_from_image"
+                    case "jpg":
+                        content = get_information_from_image.invoke({"file_path": file_path, "question": state["question"]})
+                        name = "get_information_from_image"
+                    case "py":
+                        content = get_information_from_python.invoke(file_path)
+                        name = "get_information_from_python"
+                    case "pptx":
+                        content = get_information_from_pptx.invoke(file_path)
+                        name = "get_information_from_pptx"
+                    case _:
+                        content = "Try to use some available tool to answer the user question."
+                        name = "handle_no_file"
+                prompt += f"{content}"
+                messages.append(AIMessage(content=prompt, name=name))
+                new_messages += 1
+        else:
+            prompt = "The question doesn't have an attached file."
+            name = "handle_no_file"
+
+        return {"messages": messages, "new_messages": new_messages}
+
+    def assistant_router(self, state: AgentState) -> str:
+        tool_decision = tools_condition(state)
+        if tool_decision == "tools":
+            return "tools"
+        else:
+            return "postprocess"
+
+    def postprocess(self, state: AgentState) -> AgentState:
+        last_msg = state["messages"][-1]
+        content = last_msg.content
+        index = content.find("FINAL ANSWER: ")
+        if index != -1:
+            content = content[index+len("FINAL ANSWER: "):].replace("\n", "")
+            state["final_answer"] = content
+            return state
+        else:
+            state["attempt"] += 1
+            prompt = f"""You were unable to find a satisfactory answer to the user's question.
+Now, try again, but use a different approach. You may:
+- Focus on a different angle of the question,
+- Reformulate it using alternative terminology,
+- Search for related concepts,
+- Or use a different reasoning path.
+Be creative and precise. Your goal is to uncover useful information that may have been missed previously.
+Original question:
+{state["question"]}"""
+
+            state["messages"].append(AIMessage(content=prompt, name="ASSISTANT"))
+            return state
+    def answer_evaluation(self, state: AgentState):
+        if state["final_answer"] != "":
+            return "END"
+        elif state["attempt"] >= 3:
+            state["final_answer"] = "Unable to find the answer."
+            return "END"
+        else:
+            return "RETRY"
+
+    def draw_graph(self):
+        display(Image(self.agent.get_graph().draw_mermaid_png()))
+        return