import os import requests from pydantic import BaseModel, Field from bs4 import BeautifulSoup from markdownify import markdownify as md from langchain_core.tools import tool, Tool from langchain_experimental.utilities import PythonREPL from pypdf import PdfReader from io import BytesIO from youtube_transcript_api import YouTubeTranscriptApi from pytube import extract from audio_processing import run_asr_pipeline from image_processing import use_VLM from langchain_community.tools import BraveSearch from langchain_openai import ChatOpenAI from langgraph.graph import MessagesState from langchain_core.messages import SystemMessage, HumanMessage, ToolMessage from langgraph.graph import StateGraph, START, END from langchain_community.document_loaders import WikipediaLoader from typing import Literal import json import pandas as pd @tool def multiply(a: float, b: float) -> float: """Multiplies two numbers. Args: a (float): the first number b (float): the second number """ return a * b @tool def add(a: float, b: float) -> float: """Adds two numbers. Args: a (float): the first number b (float): the second number """ return a + b @tool def subtract(a: float, b: float) -> int: """Subtracts two numbers. Args: a (float): the first number b (float): the second number """ return a - b @tool def divide(a: float, b: float) -> float: """Divides two numbers. Args: a (float): the first float number b (float): the second float number """ if b == 0: raise ValueError("Cannot divided by zero.") return a / b @tool def modulus(a: int, b: int) -> int: """Get the modulus of two numbers. Args: a (int): the first number b (int): the second number """ return a % b @tool def power(a: float, b: float) -> float: """Get the power of two numbers. Args: a (float): the first number b (float): the second number """ return a**b @tool def get_youtube_transcript(page_url: str) -> str: """Get the transcript of a YouTube video Args: page_url (str): YouTube URL of the video """ try: # get video ID from URL video_id = extract.video_id(page_url) # get transcript ytt_api = YouTubeTranscriptApi() transcript = ytt_api.fetch(video_id) # keep only text txt = '\n'.join([s.text for s in transcript.snippets]) return txt except Exception as e: return f"get_youtube_transcript failed: {e}" class PythonREPLInput(BaseModel): code: str = Field(description="The Python code string to execute.") python_repl = PythonREPL() python_repl_tool = Tool( name="python_repl", description="""A Python REPL shell (Read-Eval-Print Loop). Use this to execute single or multi-line python commands. Input should be syntactically valid Python code. Always end your code with `print(...)` to see the output. Do NOT execute code that could be harmful to the host system. You are allowed to download files from URLs. Do NOT send commands that block indefinitely (e.g., `input()`).""", func=python_repl.run, args_schema=PythonREPLInput ) @tool def get_webpage_content(page_url: str) -> str: """Load a web page and return it to markdown if possible Args: page_url (str): the URL of web page to get """ try: r = requests.get(page_url) r.raise_for_status() text = "" # special case if page is a PDF file if r.headers.get('Content-Type', '') == 'application/pdf': pdf_file = BytesIO(r.content) reader = PdfReader(pdf_file) for page in reader.pages: text += page.extract_text() else: soup = BeautifulSoup((r.text), 'html.parser') if soup.body: # convert to markdown text = md(str(soup.body)) else: # return the raw content text = r.text return text except Exception as e: return f"get_webpage_content failed: {e}" @tool def speech_recognition(file_url: str, file_extension: str) -> str: """Transcribe an audio file to text Args: file_url (str): the URL to the audio file file_extension (str): the file extension, e.g. mp3 """ print("\n\n\n####") print(file_url) print(file_extension) print("\n\n\n####") text = run_asr_pipeline(file_url, file_extension) return text @tool def query_image(query: str, image_path: str) -> str: """Ask anything about an image using a Vision Language Model Args: query (str): The query about the image, e.g. how many dogs are on the image? image_path (str): The URL to the image """ text = use_VLM(query=query, image_path=image_path) return text @tool def wiki_search(query: str) -> str: """Search Wikipedia for a query and return maximum 2 results. Args: query: The search query.""" search_docs = WikipediaLoader(query=query, load_max_docs=2).load() formatted_search_docs = "\n\n---\n\n".join( [ f'\n{doc.page_content}\n' for doc in search_docs ] ) return {"wiki_results": formatted_search_docs} @tool def analyze_excel_file(file_path: str, query: str) -> str: """ Analyze an Excel file using pandas and answer a question about it. Args: file_path (str): the path to the Excel file. query (str): Question about the data """ try: # Read the Excel file df = pd.read_excel(file_path) # Run various analyses based on the query result = ( f"Excel file loaded with {len(df)} rows and {len(df.columns)} columns.\n" ) result += f"Columns: {', '.join(df.columns)}\n\n" # Add summary statistics result += "Summary statistics:\n" result += str(df.describe()) return result except Exception as e: return f"Error analyzing Excel file: {str(e)}" tools = [ speech_recognition, get_webpage_content, python_repl_tool, get_youtube_transcript, multiply, add, subtract, power, modulus, divide, query_image, BraveSearch.from_api_key( api_key=os.getenv("BRAVE_SEARCH_API_KEY"), search_kwargs={"count": 5}), wiki_search, analyze_excel_file ] with open("system_prompt.txt", "r") as f: system_prompt = f.read() class LangGraphAgent: def __init__(self, model_name="gpt-4.1-mini", show_tools_desc=True, show_prompt=True): llm = ChatOpenAI(model=model_name, temperature=0) tools_by_name = {tool.name: tool for tool in tools} llm_with_tools = llm.bind_tools(tools) def llm_call(state: MessagesState): """LLM decides whether to call a tool or not""" return { "messages": [ llm_with_tools.invoke( [ SystemMessage( content=system_prompt ) ] + state["messages"] ) ] } def tool_node(state: dict): """Performs the tool call""" result = [] for tool_call in state["messages"][-1].tool_calls: tool = tools_by_name[tool_call["name"]] observation = tool.invoke(tool_call["args"]) result.append(ToolMessage(content=observation, tool_call_id=tool_call["id"])) return {"messages": result} def should_continue(state: MessagesState) -> Literal["environment", END]: """Decide if we should continue the loop or stop based upon whether the LLM made a tool call""" messages = state["messages"] last_message = messages[-1] # If the LLM makes a tool call, then perform an action if last_message.tool_calls: return "Action" # Otherwise, we stop (reply to the user) return END agent_builder = StateGraph(MessagesState) # Add nodes agent_builder.add_node("llm_call", llm_call) agent_builder.add_node("environment", tool_node) # Add edges to connect nodes agent_builder.add_edge(START, "llm_call") agent_builder.add_conditional_edges( "llm_call", should_continue, { # Name returned by should_continue : Name of next node to visit "Action": "environment", END: END, }, ) agent_builder.add_edge("environment", "llm_call") # Compile the agent self.agent = agent_builder.compile() if show_tools_desc: for i, tool in enumerate(llm_with_tools.kwargs['tools']): print("\n" + "="*30 + f" Tool {i+1} " + "="*30) print(json.dumps(tool[tool['type']], indent=4)) if show_prompt: print("\n" + "="*30 + f" System prompt " + "="*30) print(system_prompt) def __call__(self, question: str) -> str: print("\n\n"+"*"*50) print(f"Agent received question: {question}") print("*"*50) # Invoke messages = [HumanMessage(content=question)] messages = self.agent.invoke({"messages": messages}, {"recursion_limit": 30}) # maximum number of steps before hitting a stop condition for m in messages["messages"]: m.pretty_print() # post-process the response (keep only what's after "FINAL ANSWER:" for the exact match) response = str(messages["messages"][-1].content) try: response = response.split("FINAL ANSWER:")[-1].strip() except: print('Could not split response on "FINAL ANSWER:"') print("\n\n"+"-"*50) print(f"Agent returning with answer: {response}") return response