add agent

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

agents.py

@@ -0,0 +1,58 @@
+from tools import general_tools, file_agent_tools, data_agent_tools, math_agent_tools
+from langgraph.prebuilt import create_react_agent
+from langgraph.checkpoint.memory import MemorySaver
+from langchain_openai import ChatOpenAI
+from langgraph_supervisor import create_supervisor
+
+
+llm = ChatOpenAI(model="o4-mini")
+
+memory = MemorySaver()
+
+with open("system_prompt.txt", "r") as f:
+    prompt = f.read()
+    
+general_agent = create_react_agent(
+    model=llm,
+    tools=general_tools(),
+    checkpointer=memory,
+    prompt=prompt
+)
+
+# Create agents
+file_agent = create_react_agent(
+    model=llm,
+    tools=file_agent_tools(),
+    name="file_reader",
+    prompt="You read files. Use tools to read files."
+)
+
+math_agent = create_react_agent(
+    model=llm,
+    tools=math_agent_tools(),
+    name="calculator", 
+    prompt="You do math. Use tools for all calculations."
+)
+
+data_agent = create_react_agent(
+    model=llm,
+    tools=data_agent_tools(),
+    name="data_processor",
+    prompt="You process data. Use tools to filter and extract data."
+)
+
+prompt = """You are a supervisor. You coordinate file_reader, calculator, and data_processor to solve problems step by step. 
+Do not do calculations or file reading yourself, use the tools.
+Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. 
+YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. 
+If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. 
+If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. 
+If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+"""
+# Supervisor
+excel_supervisor = create_supervisor(
+    [file_agent, math_agent, data_agent],
+    model=llm,
+    prompt=prompt
+).compile()
+

files/1f975693-876d-457b-a649-393859e79bf3.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:200f767e732b49efef5c05d128903ee4d2c34e66fdce7f5593ac123b2e637673
+size 280868

files/99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b218c951c1f888f0bbe6f46c080f57afc7c9348fffc7ba4da35749ff1e2ac40f
+size 179304

files/f918266a-b3e0-4914-865d-4faa564f1aef.py

@@ -0,0 +1,35 @@
+from random import randint
+import time
+
+class UhOh(Exception):
+    pass
+
+class Hmm:
+    def __init__(self):
+        self.value = randint(-100, 100)
+
+    def Yeah(self):
+        if self.value == 0:
+            return True
+        else:
+            raise UhOh()
+
+def Okay():
+    while True:
+        yield Hmm()
+
+def keep_trying(go, first_try=True):
+    maybe = next(go)
+    try:
+        if maybe.Yeah():
+            return maybe.value
+    except UhOh:
+        if first_try:
+            print("Working...")
+            print("Please wait patiently...")
+        time.sleep(0.1)
+        return keep_trying(go, first_try=False)
+
+if __name__ == "__main__":
+    go = Okay()
+    print(f"{keep_trying(go)}")

qa_graph.py

@@ -0,0 +1,184 @@
+from dataclasses import dataclass
+from langgraph.graph import START, StateGraph, END
+from typing import TypedDict
+from agents import general_agent, excel_supervisor
+import os
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+os.environ["OPENAI_API_KEY"] = str(OPENAI_API_KEY)
+
+@dataclass
+class Question:
+    task_id: str
+    question: str
+    Level: str
+    file_name: str 
+    local_file_path: str|None = None
+
+def get_file_type(file_path: str) -> str:
+    """Determine file type from extension."""
+    if not file_path:
+        return "none"
+    
+    file_path = file_path.lower()
+    
+    if file_path.endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp')):
+        return "image"
+    elif file_path.endswith(('.xlsx', '.xls', '.csv')):
+        return "excel"
+    elif file_path.endswith('.py'):
+        return "python"
+    elif file_path.endswith(('.mp3', '.wav', '.m4a', '.ogg')):
+        return "audio"
+    else:
+        return "unknown"
+
+def ask_question(question: str, thread_id: str = "default") -> str:
+    """Ask the agent a question."""
+    config = {"configurable": {"thread_id": thread_id}}
+    
+    try:
+        response = general_agent.invoke(
+            {"messages": [{"role": "user", "content": question}]},
+            config=config
+        )
+        return response["messages"][-1].content
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+def ask_question_with_file(question: Question, thread_id: str = "default") -> str:
+    """Ask the agent a question, with optional file analysis."""
+    q = question.question
+    root_file = "./files"
+    file_path = root_file + "/" + question.file_name
+    if not question.file_name:
+        return ask_question(q, thread_id)
+    
+    file_type = get_file_type(file_path)
+    
+    # Create enhanced question with file guidance
+    if file_type == "image":
+        enhanced_question = f"{q}\n\nThere is an image file at '{file_path}'. Use the analyze_image tool to examine it."
+    elif file_type == "excel":
+        enhanced_question = f"{q}\n\nFile path: {file_path}"
+        result = excel_supervisor.invoke({
+            "messages": [
+                {"role": "user", "content": enhanced_question}
+            ]
+        })
+        return result["messages"][-1].content
+    elif file_type == "python":
+        enhanced_question = f"{q}\n\nThere is a Python file at '{file_path}'. Use the read_python_file tool to examine it."
+    elif file_type == "audio":
+        enhanced_question = f"{q}\n\nThere is an audio file at '{file_path}'. Use the transcribe_audio tool to process it."
+    else:
+        enhanced_question = f"{q}\n\nThere is a file at '{file_path}' but I'm not sure what type it is."
+    
+    return ask_question(enhanced_question, thread_id)
+
+test = [
+#     {
+#     "task_id": "cca530fc-4052-43b2-b130-b30968d8aa44",
+#     "question": "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.",
+#     "Level": "1",
+#     "file_name": "cca530fc-4052-43b2-b130-b30968d8aa44.png"
+#   },
+#             {
+#     "task_id": "1f975693-876d-457b-a649-393859e79bf3",
+#     "question": "Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(\n\nCould you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",
+#     "Level": "1",
+#     "file_name": "1f975693-876d-457b-a649-393859e79bf3.mp3"
+#   },
+              {
+    "task_id": "7bd855d8-463d-4ed5-93ca-5fe35145f733",
+    "question": "The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.",
+    "Level": "1",
+    "file_name": "7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx"
+  }         
+            ]
+
+questions = [Question(**item) for item in test]
+for q in questions:
+    print(q.question)
+    print(q.file_name)
+    print(q.local_file_path)
+
+
+
+# State
+class State(TypedDict):
+    question: Question
+    decision: str
+    answer: str
+    
+# NODE FUNCTIONS - These are the ones that work with LangGraph
+def ask_question_node(state: State) -> dict:
+    """Node function for questions without files."""
+    question_obj = state["question"]
+    thread_id = f"test_{question_obj.task_id}"
+    
+    # Call your existing function
+    answer = ask_question(question_obj.question, thread_id)
+    
+    # Return dict to update state
+    return {"answer": answer}
+
+def ask_question_with_file_node(state: State) -> dict:
+    """Node function for questions with files."""
+    question_obj = state["question"]
+    thread_id = f"test_{question_obj.task_id}"
+    
+    # Call your existing function
+    answer = ask_question_with_file(question_obj, thread_id)
+    
+    # Return dict to update state
+    return {"answer": answer}
+
+def router_node(state: State):
+    """Router node - returns dict to update state"""
+    if state["question"].file_name:
+        decision = "query_with_file"
+    else:
+        decision = "query"
+    
+    return {"decision": decision}
+
+def router_function(state: State):
+    """Routing function - returns string to choose path"""
+    return state["decision"]
+
+# Graph
+builder = StateGraph(State)
+
+# Use the NODE functions (not the original functions)
+builder.add_node("query_with_file", ask_question_with_file_node)
+builder.add_node("query", ask_question_node)
+builder.add_node("router", router_node)
+
+# Define edges
+builder.add_edge(START, "router")
+builder.add_conditional_edges(
+    "router",
+    router_function,
+    {
+        "query_with_file": "query_with_file",
+        "query": "query",
+    },
+)
+builder.add_edge("query_with_file", END)
+builder.add_edge("query", END)
+
+react_graph = builder.compile()
+
+if __name__ == "__main__":
+    for i, question in enumerate(questions):
+        print(f"\n{i}. {question.question}")
+        
+        # Invoke the graph and capture the result
+        result = react_graph.invoke({
+            "question": question,
+            "decision": "",
+            "answer": ""
+        })
+        
+        print(f"Answer: {result['answer']}")
+        print("-" * 50)

system_prompt.txt

@@ -0,0 +1,7 @@
+You are a general AI assistant. 
+I will ask you a question. 
+Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. 
+YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. 
+If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. 
+If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. 
+If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.

tools.py

@@ -0,0 +1,186 @@
+from langchain_core.messages import HumanMessage
+from langchain_core.tools import tool
+from langchain_community.tools import (
+    DuckDuckGoSearchRun,
+    WikipediaQueryRun,
+    ArxivQueryRun
+)
+from langchain_community.utilities import WikipediaAPIWrapper, ArxivAPIWrapper
+from langchain_openai import ChatOpenAI
+
+import base64
+import pandas as pd
+import os
+
+import os
+from huggingface_hub import InferenceClient
+import json
+import requests
+
+from dotenv import load_dotenv
+load_dotenv()
+HF_TOKEN = os.getenv("HF_TOKEN")
+client = InferenceClient(
+    provider="hf-inference",
+    api_key=HF_TOKEN,
+)
+
+llm = ChatOpenAI(model="o4-mini")
+vision_llm = ChatOpenAI(model="gpt-4o")
+
+@tool
+def analyze_image(img_path: str, question: str) -> str:
+    """Analyze an image and answer a question about it."""
+    try:
+        with open(img_path, "rb") as image_file:
+            image_bytes = image_file.read()
+        
+        image_base64 = base64.b64encode(image_bytes).decode("utf-8")
+        
+        message = [
+            HumanMessage(
+                content=[
+                    {"type": "text", "text": question},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}
+                    }
+                ]
+            )
+        ]
+        
+        response = vision_llm.invoke(message)
+        return response.content
+        
+    except Exception as e:
+        return f"Error analyzing image: {str(e)}"
+
+@tool
+def read_excel_file(file_path: str, question: str) -> str:
+    """Read and analyze an Excel file to answer a question."""
+    try:
+        # Read Excel file
+        df = pd.read_excel(file_path)
+        
+        df_dict = df.to_dict(orient='records')
+        info = json.dumps(df_dict)        
+        return info
+        
+    except Exception as e:
+        return f"Error reading Excel file: {str(e)}"
+
+@tool
+def read_python_file(file_path: str, question: str) -> str:
+    """Read and analyze a Python file to answer a question."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            code_content = f.read()
+        
+        prompt = f"""Here is Python code from a file:
+
+        ```python
+        {code_content}
+        ```
+
+        Question: {question}
+
+        Please analyze the code and answer the question."""
+        
+        response = llm.invoke([HumanMessage(content=prompt)])
+        return response.content
+        
+    except Exception as e:
+        return f"Error reading Python file: {str(e)}"
+
+@tool
+def transcribe_audio(file_path: str, question: str) -> str:
+    """Transcribe audio file."""
+    try:
+        headers = {
+            "Authorization": f"Bearer {HF_TOKEN}",
+            "Content-Type": "audio/mpeg"  # Add this line for MP3 files
+        }
+        API_URL =  "https://router.huggingface.co/hf-inference/models/openai/whisper-large-v3"
+
+        def query(filename):
+            with open(filename, "rb") as f:
+                data = f.read()
+            response = requests.request("POST", API_URL, headers=headers, data=data)
+            return json.loads(response.content.decode("utf-8"))
+
+        data = query(file_path)
+        return data
+        
+    except Exception as e:
+        return f"Error transcribing audio: {str(e)}"
+
+#### Excel supervisor agent
+
+    
+def general_tools():    
+    tools = [
+        DuckDuckGoSearchRun(),
+        WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper()),
+        ArxivQueryRun(api_wrapper=ArxivAPIWrapper()),
+        analyze_image,
+        read_python_file,
+        transcribe_audio,
+    ]
+    return tools
+
+
+# Simple file tools
+@tool
+def read_excel(file_path: str) -> str:
+    """Read any Excel file and return as JSON."""
+    df = pd.read_excel(file_path)
+    return json.dumps(df.to_dict(orient='records'))
+
+# Simple math tools
+@tool
+def add(a: float, b: float) -> float:
+    """Add two numbers."""
+    return a + b
+
+@tool
+def sum_list(numbers: list) -> float:
+    """Sum a list of numbers."""
+    return sum(numbers)
+
+# Simple data tools
+@tool
+def extract_values(data: str, column: str) -> list:
+    """Extract all values from a column in JSON data."""
+    parsed = json.loads(data)
+    values = []
+    for row in parsed:
+        for key, value in row.items():
+            if column.lower() in key.lower():
+                try:
+                    values.append(float(value))
+                except:
+                    pass
+    return values
+
+@tool
+def filter_rows(data: str, exclude_words: list) -> str:
+    """Remove rows containing any of the exclude words."""
+    parsed = json.loads(data)
+    filtered = []
+    for row in parsed:
+        row_text = " ".join(str(v).lower() for v in row.values())
+        if not any(word.lower() in row_text for word in exclude_words):
+            filtered.append(row)
+    return json.dumps(filtered)
+
+def file_agent_tools():
+    tools = [read_excel]
+    return tools
+
+def math_agent_tools():
+    tools = [add, sum_list]
+    return tools
+
+def data_agent_tools():
+    tools = [extract_values, filter_rows]
+    return tools