Commit
·
a605490
1
Parent(s):
43199e3
Updated the README file
Browse files- .env.example +1 -1
- README.md +15 -2
- app.py +7 -20
- nodes/{chunking_node.py → chunking_handler.py} +0 -0
- nodes/nodes.py +10 -8
- tools/math_agent.py +2 -12
.env.example
CHANGED
|
@@ -12,5 +12,5 @@ CHESS_ENGINE_PATH=stock-fish-engine-location-here
|
|
| 12 |
|
| 13 |
# Configure this if you want to enable observability with LangSmith
|
| 14 |
LANGSMITH_API_KEY=your-langsmith-key
|
| 15 |
-
LANGSMITH_TRACING=
|
| 16 |
LANGSMITH_PROJECT=gaia_agent
|
|
|
|
| 12 |
|
| 13 |
# Configure this if you want to enable observability with LangSmith
|
| 14 |
LANGSMITH_API_KEY=your-langsmith-key
|
| 15 |
+
LANGSMITH_TRACING=false
|
| 16 |
LANGSMITH_PROJECT=gaia_agent
|
README.md
CHANGED
|
@@ -39,7 +39,8 @@ GAIA introductory paper [”GAIA: A Benchmark for General AI Assistants”](http
|
|
| 39 |
|
| 40 |
**Nodes**
|
| 41 |
|
| 42 |
-
- **Pre-processor**:
|
|
|
|
| 43 |
- **Assistant**: The brain of the agent. Decides which tool to call. Uses `gpt-4.1`.
|
| 44 |
- **Tools**: The invocation of a tool.
|
| 45 |
- **Optimize memory**: This step summarizes and the removes all the messages except the last 2. If the last message is the response of a web
|
|
@@ -106,12 +107,24 @@ TODO: analyze YouTube videos and answer questions about objects in the video.
|
|
| 106 |
## Future work and improvements 🔜
|
| 107 |
- **Evaluation**: Evaluate the agent against other questions from the GAIA validation set.
|
| 108 |
- **Large Web Extracts**: Try other chunking strategies.
|
| 109 |
-
- **Audio Analysis**:Use a lesser expensive model to get the transcripts (like whisper) and if this is not enough to answer the question and more sophiticated processing is needed
|
| 110 |
for other sounds like music, barks or other type of sounds then indeed use a better model.
|
| 111 |
- **Python File execution**:Improve safety when executing python code or python files.
|
| 112 |
- **Video Analysis**: Answer questions about objects in the video.
|
| 113 |
- **Chessboard Images Analysis**: Detect correctly all pieces on a chess board image.
|
| 114 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
## References 📚
|
| 116 |
The math tool implementation was inspired from this repo https://github.com/langchain-ai/open_deep_research
|
| 117 |
|
|
|
|
| 39 |
|
| 40 |
**Nodes**
|
| 41 |
|
| 42 |
+
- **Pre-processor**: Initialization and preparation of the state. Input handling: attached pictures are directly sent to the assistant node and to the model.
|
| 43 |
+
Other type of attachments are loaded only in the tools.
|
| 44 |
- **Assistant**: The brain of the agent. Decides which tool to call. Uses `gpt-4.1`.
|
| 45 |
- **Tools**: The invocation of a tool.
|
| 46 |
- **Optimize memory**: This step summarizes and the removes all the messages except the last 2. If the last message is the response of a web
|
|
|
|
| 107 |
## Future work and improvements 🔜
|
| 108 |
- **Evaluation**: Evaluate the agent against other questions from the GAIA validation set.
|
| 109 |
- **Large Web Extracts**: Try other chunking strategies.
|
| 110 |
+
- **Audio Analysis**: Use a lesser expensive model to get the transcripts (like `whisper`) and if this is not enough to answer the question and more sophiticated processing is needed
|
| 111 |
for other sounds like music, barks or other type of sounds then indeed use a better model.
|
| 112 |
- **Python File execution**:Improve safety when executing python code or python files.
|
| 113 |
- **Video Analysis**: Answer questions about objects in the video.
|
| 114 |
- **Chessboard Images Analysis**: Detect correctly all pieces on a chess board image.
|
| 115 |
|
| 116 |
+
|
| 117 |
+
## How to use
|
| 118 |
+
Please check the `.env.example` file for the environment variables that need to be configured. If running in a Huggingface Space, you can
|
| 119 |
+
set them as secrets, otherwise rename the `.env.example` to `.env`.
|
| 120 |
+
|
| 121 |
+
The `CHESS_ENGINE_PATH` needs to be configured only if running on a Windows machine and needs to point
|
| 122 |
+
to the `stockfish` executable, otherwise the `stockfish`installation is automatically detected.
|
| 123 |
+
|
| 124 |
+
The `LANGSMITH_*` properties need to be configured only if you want to enable observability with LangSmith.
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
|
| 128 |
## References 📚
|
| 129 |
The math tool implementation was inspired from this repo https://github.com/langchain-ai/open_deep_research
|
| 130 |
|
app.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
| 1 |
import os
|
| 2 |
import time
|
| 3 |
from pathlib import Path
|
| 4 |
-
from typing import Optional
|
| 5 |
|
| 6 |
import gradio as gr
|
| 7 |
import pandas as pd
|
|
@@ -16,23 +15,6 @@ from utils.dependencies_checker import check_dependencies
|
|
| 16 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 17 |
|
| 18 |
|
| 19 |
-
# --- Basic Agent Definition ---
|
| 20 |
-
class BasicAgent:
|
| 21 |
-
agent: GaiaAgent
|
| 22 |
-
|
| 23 |
-
def __init__(self):
|
| 24 |
-
self.agent = GaiaAgent()
|
| 25 |
-
|
| 26 |
-
def __call__(self, question: str, attachment: Optional[Attachment] = None) -> str:
|
| 27 |
-
print(f"Agent received question (first 50 chars): {question[:50]}...")
|
| 28 |
-
if attachment:
|
| 29 |
-
print(f"Agent received an attachment : {attachment.file_path}...")
|
| 30 |
-
|
| 31 |
-
answer = self.agent.__call__(question, attachment)
|
| 32 |
-
print(f"Agent returning fixed answer: {answer}")
|
| 33 |
-
return answer
|
| 34 |
-
|
| 35 |
-
|
| 36 |
def get_question_attached_file(task_id, file_name) -> Attachment:
|
| 37 |
api_url = DEFAULT_API_URL
|
| 38 |
attachment_url = f"{api_url}/files/{task_id}"
|
|
@@ -82,7 +64,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
|
|
| 82 |
|
| 83 |
# 1. Instantiate Agent ( modify this part to create your agent)
|
| 84 |
try:
|
| 85 |
-
agent =
|
| 86 |
except Exception as e:
|
| 87 |
print(f"Error instantiating agent: {e}")
|
| 88 |
return f"Error initializing agent: {e}", None
|
|
@@ -135,8 +117,13 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
|
|
| 135 |
# check if the answer is cached, if not invoke the agent
|
| 136 |
submitted_answer = cache.get(task_id)
|
| 137 |
if submitted_answer is None:
|
|
|
|
|
|
|
|
|
|
| 138 |
submitted_answer = agent(question_text, attachment)
|
| 139 |
-
|
|
|
|
|
|
|
| 140 |
cache.set(task_id, submitted_answer)
|
| 141 |
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
|
| 142 |
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
|
|
|
|
| 1 |
import os
|
| 2 |
import time
|
| 3 |
from pathlib import Path
|
|
|
|
| 4 |
|
| 5 |
import gradio as gr
|
| 6 |
import pandas as pd
|
|
|
|
| 15 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 16 |
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
def get_question_attached_file(task_id, file_name) -> Attachment:
|
| 19 |
api_url = DEFAULT_API_URL
|
| 20 |
attachment_url = f"{api_url}/files/{task_id}"
|
|
|
|
| 64 |
|
| 65 |
# 1. Instantiate Agent ( modify this part to create your agent)
|
| 66 |
try:
|
| 67 |
+
agent = GaiaAgent()
|
| 68 |
except Exception as e:
|
| 69 |
print(f"Error instantiating agent: {e}")
|
| 70 |
return f"Error initializing agent: {e}", None
|
|
|
|
| 117 |
# check if the answer is cached, if not invoke the agent
|
| 118 |
submitted_answer = cache.get(task_id)
|
| 119 |
if submitted_answer is None:
|
| 120 |
+
print(f"Agent received question (first 50 chars): {question_text[:50]}...")
|
| 121 |
+
if attachment:
|
| 122 |
+
print(f"Agent received an attachment : {attachment.file_path}...")
|
| 123 |
submitted_answer = agent(question_text, attachment)
|
| 124 |
+
print(f"Agent returning fixed answer: {submitted_answer}")
|
| 125 |
+
#sleep in
|
| 126 |
+
time.sleep(30)
|
| 127 |
cache.set(task_id, submitted_answer)
|
| 128 |
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
|
| 129 |
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
|
nodes/{chunking_node.py → chunking_handler.py}
RENAMED
|
File without changes
|
nodes/nodes.py
CHANGED
|
@@ -8,7 +8,7 @@ from langchain_openai import ChatOpenAI
|
|
| 8 |
from config.settings import config
|
| 9 |
from core.messages import attachmentHandler
|
| 10 |
from core.state import State
|
| 11 |
-
from nodes.
|
| 12 |
from tools.audio_tool import query_audio
|
| 13 |
from tools.chess_tool import chess_analysis_tool
|
| 14 |
from tools.excel_tool import query_excel_file
|
|
@@ -17,14 +17,16 @@ from tools.python_executor import execute_python_code
|
|
| 17 |
from tools.tavily_tools import web_search_tools
|
| 18 |
from utils.prompt_manager import prompt_mgmt
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
|
|
|
|
|
|
| 25 |
|
| 26 |
model = ChatOpenAI(model=config.model_name)
|
| 27 |
-
model = model.bind_tools(
|
| 28 |
|
| 29 |
response_processing_model = ChatOpenAI(model=config.response_processing_model_name)
|
| 30 |
|
|
@@ -115,7 +117,7 @@ def optimize_memory(state: State):
|
|
| 115 |
# Delete all but the 2 most recent messages and the first one
|
| 116 |
remaining_messages = [RemoveMessage(id=m.id) for m in state["messages"][:-2]]
|
| 117 |
|
| 118 |
-
# If the last message returned from a tool is oversized, chunk it and
|
| 119 |
content_handler = OversizedContentHandler()
|
| 120 |
chunked = content_handler.process_oversized_message(state["messages"][-1], state.get("question"))
|
| 121 |
|
|
|
|
| 8 |
from config.settings import config
|
| 9 |
from core.messages import attachmentHandler
|
| 10 |
from core.state import State
|
| 11 |
+
from nodes.chunking_handler import OversizedContentHandler
|
| 12 |
from tools.audio_tool import query_audio
|
| 13 |
from tools.chess_tool import chess_analysis_tool
|
| 14 |
from tools.excel_tool import query_excel_file
|
|
|
|
| 17 |
from tools.tavily_tools import web_search_tools
|
| 18 |
from utils.prompt_manager import prompt_mgmt
|
| 19 |
|
| 20 |
+
agent_tools = []
|
| 21 |
+
agent_tools.extend(web_search_tools)
|
| 22 |
+
agent_tools.append(query_audio)
|
| 23 |
+
agent_tools.append(query_excel_file)
|
| 24 |
+
agent_tools.append(execute_python_code)
|
| 25 |
+
agent_tools.append(math_tool)
|
| 26 |
+
agent_tools.append(chess_analysis_tool)
|
| 27 |
|
| 28 |
model = ChatOpenAI(model=config.model_name)
|
| 29 |
+
model = model.bind_tools(agent_tools, parallel_tool_calls=False)
|
| 30 |
|
| 31 |
response_processing_model = ChatOpenAI(model=config.response_processing_model_name)
|
| 32 |
|
|
|
|
| 117 |
# Delete all but the 2 most recent messages and the first one
|
| 118 |
remaining_messages = [RemoveMessage(id=m.id) for m in state["messages"][:-2]]
|
| 119 |
|
| 120 |
+
# If the last message returned from a tool is oversized, chunk it and replace it with the relevant chunks
|
| 121 |
content_handler = OversizedContentHandler()
|
| 122 |
chunked = content_handler.process_oversized_message(state["messages"][-1], state.get("question"))
|
| 123 |
|
tools/math_agent.py
CHANGED
|
@@ -1,9 +1,3 @@
|
|
| 1 |
-
"""Research Agent Implementation.
|
| 2 |
-
|
| 3 |
-
This module implements a research agent that can perform iterative web searches
|
| 4 |
-
and synthesis to answer complex research questions.
|
| 5 |
-
"""
|
| 6 |
-
|
| 7 |
from typing import Literal
|
| 8 |
|
| 9 |
from langchain.chat_models import init_chat_model
|
|
@@ -30,8 +24,6 @@ compress_model = init_chat_model(model="openai:gpt-4.1",
|
|
| 30 |
max_tokens=32000)
|
| 31 |
|
| 32 |
|
| 33 |
-
# ===== AGENT NODES =====
|
| 34 |
-
|
| 35 |
def llm_call(state: MathAgentState):
|
| 36 |
"""Analyze current state and decide on next actions.
|
| 37 |
|
|
@@ -100,12 +92,10 @@ def compress_research(state: MathAgentState) -> dict:
|
|
| 100 |
}
|
| 101 |
|
| 102 |
|
| 103 |
-
# ===== ROUTING LOGIC =====
|
| 104 |
-
|
| 105 |
def should_continue(state: MathAgentState) -> Literal["tool_node", "compress_research"]:
|
| 106 |
-
"""Determine whether to continue
|
| 107 |
|
| 108 |
-
Determines whether the agent should continue the
|
| 109 |
a final answer based on whether the LLM made tool calls.
|
| 110 |
|
| 111 |
Returns:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from typing import Literal
|
| 2 |
|
| 3 |
from langchain.chat_models import init_chat_model
|
|
|
|
| 24 |
max_tokens=32000)
|
| 25 |
|
| 26 |
|
|
|
|
|
|
|
| 27 |
def llm_call(state: MathAgentState):
|
| 28 |
"""Analyze current state and decide on next actions.
|
| 29 |
|
|
|
|
| 92 |
}
|
| 93 |
|
| 94 |
|
|
|
|
|
|
|
| 95 |
def should_continue(state: MathAgentState) -> Literal["tool_node", "compress_research"]:
|
| 96 |
+
"""Determine whether to continue the solving of the problem or provide final answer.
|
| 97 |
|
| 98 |
+
Determines whether the agent should continue the solving loop or provide
|
| 99 |
a final answer based on whether the LLM made tool calls.
|
| 100 |
|
| 101 |
Returns:
|