Refactor agent logic and enhance data handling; add tools for file processing and update system prompt format

.gitignore

@@ -0,0 +1,28 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+results/*
+*.log
+.env
+test.png
+graph.png
+questions_data.json
+.vscode/
+cache

app.py

@@ -3,6 +3,11 @@ import gradio as gr
 import requests
 import inspect
 import pandas as pd
+from langchain_core.messages import HumanMessage, SystemMessage
+
+
+from src.my_app import build_graph, get_prompt
+import json
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -13,11 +18,19 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 class BasicAgent:
     def __init__(self):
         print("BasicAgent initialized.")
-    def __call__(self, question: str) -> str:
+        self.graph = build_graph()  # Build the state graph for the agent
+    def __call__(self, question: str, task_id: str) -> str:
         print(f"Agent received question (first 50 chars): {question[:50]}...")
-        fixed_answer = "This is a default answer."
-        print(f"Agent returning fixed answer: {fixed_answer}")
-        return fixed_answer
+        system_prompt = SystemMessage(content=get_prompt())
+        messages = self.graph.invoke({
+            "messages": [
+                system_prompt,
+                {"role": "user", "content": question}
+            ],
+        "task_id": task_id
+    })
+        answer = messages['messages'][-1].content
+        return answer[14:]
 
 def run_and_submit_all( profile: gr.OAuthProfile | None):
     """
@@ -54,6 +67,9 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
         response = requests.get(questions_url, timeout=15)
         response.raise_for_status()
         questions_data = response.json()
+        # Save questions_data to a local file for inspection/debugging
+        with open("questions_data.json", "w", encoding="utf-8") as f:
+            json.dump(questions_data, f, ensure_ascii=False, indent=2)
         if not questions_data:
              print("Fetched questions list is empty.")
              return "Fetched questions list is empty or invalid format.", None
@@ -80,7 +96,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text)
+            submitted_answer = agent(question_text, task_id)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
         except Exception as e:

fetch_all_data.py

@@ -0,0 +1,51 @@
+import os
+import requests
+import json
+
+# Load task IDs from questions_data.json
+questions_file = "/home/pietro/Courses/HF_Agents/Final_Assignment_Template/questions_data.json"
+with open(questions_file, "r", encoding="utf-8") as f:
+    questions_data = json.load(f)
+
+task_ids = [question["task_id"] for question in questions_data]
+
+# Directory to save the results
+output_dir = "results"
+os.makedirs(output_dir, exist_ok=True)
+
+base_url = "https://agents-course-unit4-scoring.hf.space/files/{}"
+headers = {"accept": "application/json"}
+
+for task_id in task_ids:
+    url = base_url.format(task_id)
+    response = requests.get(url, headers=headers)
+    
+    # Extract file name from Content-Disposition header if available
+    content_disposition = response.headers.get("content-disposition", "")
+    if "filename=" in content_disposition:
+        file_name = content_disposition.split("filename=")[-1].strip('"')
+    else:
+        file_name = f"{task_id}.unknown"  # Default file name if not provided
+    
+    # Determine file extension based on Content-Type header
+    content_type = response.headers.get("content-type", "")
+    if "image/png" in content_type:
+        file_extension = ".png"
+    elif "application/json" in content_type:
+        file_extension = ".json"
+    elif "audio/mpeg" in content_type:
+        file_extension = ".mp3"
+    elif "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" in content_type:
+        file_extension = ".xlsx"
+    elif "text/plain" in content_type:
+        file_extension = ".txt"
+    elif "application/x-python-code" in content_type or file_name.endswith(".py"):
+        file_extension = ".py"
+    else:
+        file_extension = ""  # Unknown file type
+    
+    # Save the file with the appropriate name and extension
+    output_path = os.path.join(output_dir, file_name if file_extension else f"{task_id}{file_extension}")
+    with open(output_path, "wb") as f:
+        f.write(response.content)
+    print(f"Saved: {output_path}")

src/my_app.py

@@ -0,0 +1,127 @@
+import os
+from typing import List, Dict, Any, Optional
+from langgraph.graph import StateGraph, START, END
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage
+from IPython.display import Image, display
+
+from langchain_openai import AzureChatOpenAI
+from langgraph.prebuilt import ToolNode, tools_condition
+from langgraph.prebuilt import tools_condition
+
+from .tools import reverse_text, is_question_reversed, route_question, avaiable_tools, CustomToolNode
+from langchain_core.messages import SystemMessage
+from .state import State
+# from langgraph.prebuilt import create_react_agent
+from langchain.agents import create_tool_calling_agent
+from langchain_core.runnables import Runnable
+
+
+# llm_with_tools = llm.bind_tools(avaiable_tools, parallel_tool_calls=False)
+def get_prompt() -> str:
+    with open("system_prompt.txt", "r", encoding="utf-8") as f:
+        return f.read()
+    # return """You are a helpful assistant tasked with answering questions using a set of tools. 
+    #         Now, I will ask you a question. Report your thoughts, show the task_id and finish your answer with the following template: 
+    #         FINAL ANSWER: [YOUR FINAL ANSWER]. 
+    #         YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+    #         Your answer should only start with "FINAL ANSWER: ", then follows with the answer. 
+    # """
+
+
+def build_graph():
+    """Builds the state graph for the React agent."""
+    # Initialize our LLM
+    llm = AzureChatOpenAI(
+        azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),  # Corrected variable name
+        openai_api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
+        deployment_name=os.getenv("AZURE_OPENAI_DEPLOYMENT"),  # Corrected variable name
+        openai_api_key=os.getenv("AZURE_OPENAI_API_KEY"),
+        temperature=0.0,
+    )
+    llm_with_tools = llm.bind_tools(avaiable_tools)
+
+
+    def assistant(state: State):
+        """Assistant node"""
+        response = llm_with_tools.invoke(state["messages"])
+        if response.content == '':
+            messages = [response]  # tool calling message
+        else:
+            final_message = response.content
+            # final_message += f"\n\nTask ID: {state['task_id']}"
+            messages = [final_message]
+        return {"messages": messages, 
+                "task_id": state["task_id"]
+                }
+    
+    # Initialize the state graph
+    graph_builder = StateGraph(State)
+
+    # Add nodes
+    graph_builder.add_node("check_question_reversed", is_question_reversed)
+    graph_builder.add_node("reverse_text", reverse_text)
+    graph_builder.add_node("assistant", assistant)
+    tools_dict = {tool.name: tool for tool in avaiable_tools}
+
+    graph_builder.add_node("tools", CustomToolNode(tools_dict))
+
+    # graph_builder.add_edge(START, "check_question_reversed")
+    # graph_builder.add_conditional_edges(
+    #     "check_question_reversed",
+    #     route_question,
+    #     {
+    #         "question_reversed": "reverse_text",
+    #         "question_not_reversed": "assistant"
+    #     }
+    # )
+    # graph_builder.add_edge("reverse_text", "assistant")
+    graph_builder.add_edge(START, "assistant")
+    graph_builder.add_conditional_edges(
+        "assistant",
+        tools_condition,
+    )
+    graph_builder.add_edge("tools", "assistant")
+    graph_builder.add_edge("assistant", END)
+    return graph_builder.compile()
+
+if __name__ == "__main__":
+    # Build the graph
+    react_graph = build_graph()
+
+    # Display the graph visualization
+    # graph = react_graph.get_graph(xray=True)
+    # display(Image(graph.draw_mermaid_png(output_file_path='graph.png')))
+
+    # Example question to test the agent
+    # question = "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia."
+    # question = ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI"
+    # question = "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?"
+    #question = """Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.\n\nWhat does Teal'c say in response to the question \"Isn't that hot?\""""
+    # question = """Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(\n\nCould you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order."""
+    # question = """The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places."""
+    question = """What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?"""
+    task_id = "7bd855d8-463d-4ed5-93ca-5fe35145f733"
+    system_prompt = SystemMessage(content=get_prompt())
+    messages = react_graph.invoke({
+        "messages": [
+            system_prompt,
+            {"role": "user", "content": question}
+        ],
+        "task_id": task_id
+    })
+    for m in messages["messages"]:
+        m.pretty_print()
+
+    answer = messages['messages'][-1].content
+    print(f"Final Answer: {answer[14:]}")
+    # Stream the response from the agent
+    # events = react_graph.stream(
+    #     {"messages": [("user", question)]},
+    #     config={"configurable": {"return_intermediate_steps": True}},
+    #     stream_mode="values"
+    # )
+    
+    # for event in events:
+    #     print(event)  # Replace `_print_event(event, _printed)` with direct printing
+    #     print("----\n---")

src/state.py

@@ -0,0 +1,14 @@
+from langgraph.graph.message import add_messages
+from typing import Annotated
+from typing_extensions import TypedDict
+
+
+
+class State(TypedDict):
+    # Messages have the type "list". The `add_messages` function
+    # in the annotation defines how this state key should be updated
+    # (in this case, it appends messages to the list, rather than overwriting them)
+    is_question_reversed: bool
+    question: str
+    messages: Annotated[list, add_messages]
+    task_id: str

src/tools.py

@@ -0,0 +1,235 @@
+from langchain_community.tools import WikipediaQueryRun
+from langchain_community.utilities import WikipediaAPIWrapper
+from langchain_community.tools import DuckDuckGoSearchRun
+from langchain_community.tools import ArxivQueryRun
+
+from langchain_core.messages import HumanMessage
+from dotenv import load_dotenv
+from langchain_openai import AzureChatOpenAI
+from langchain_perplexity import ChatPerplexity
+from langchain_core.tools import tool
+from langchain_community.tools.tavily_search.tool import TavilySearchResults
+
+from langchain_huggingface import HuggingFaceEndpoint, ChatHuggingFace
+from youtube_transcript_api import YouTubeTranscriptApi
+
+from pandasai.llm.openai import OpenAI
+from llama_index.readers.pandas_ai import PandasAIReader
+
+import os
+import re
+import whisper
+import pandas as pd
+
+from .state import State
+
+# wikipedia_tool = WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper())
+
+load_dotenv()
+
+# Set your OpenAI API key here
+# os.environ["OPENAI_API_KEY"] = "sk-xxxxx"  # Replace with your actual API key
+
+# Initialize our LLM
+llm = AzureChatOpenAI(
+    azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),  # Corrected variable name
+    openai_api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
+    deployment_name=os.getenv("AZURE_OPENAI_DEPLOYMENT"),  # Corrected variable name
+    openai_api_key=os.getenv("AZURE_OPENAI_API_KEY"),
+    temperature=0.0,
+)
+
+hf_endpoint = HuggingFaceEndpoint(
+    # repo_id="microsoft/Phi-3-mini-4k-instruct",
+    repo_id="Qwen/Qwen3-4B",
+    task="text-generation",
+    max_new_tokens=1000,
+    do_sample=False,
+)
+llm_hf = ChatHuggingFace(llm=hf_endpoint, verbose=True)
+
+@tool
+def get_youtube_transcript(video_id: str) -> list:
+    """Fetches the transcript of a YouTube video."""
+    ytt_api = YouTubeTranscriptApi()
+    fetched_transcript = ytt_api.fetch(video_id)
+    return fetched_transcript.to_raw_data()
+
+@tool
+def transcript_mp3_audio(task_id: str) -> str:
+    """Transcribes an MP3 audio file using a speech-to-text model."""
+    file_path = f"{task_id}.mp3"  # Assuming the file is named with the task_id
+    file_path = os.path.join('results', file_path)
+    model = whisper.load_model("base")  # puoi usare tiny, base, small, medium, large
+    result = model.transcribe(file_path)
+    return result["text"]
+
+@tool
+def load_and_analyze_excel_file(question: str, task_id: str) -> str:
+    """Loads an Excel file and analyzes it using PandasAI."""
+    file_path = f"{task_id}.xlsx"  # Assuming the file is named with the task_id
+    file_path = os.path.join('results', file_path)
+    df = pd.read_excel(file_path)
+    reader = PandasAIReader(pandas_llm=llm)
+    response = reader.run_pandas_ai(
+        df, question, is_conversational_answer=True
+    )
+    return response
+
+
+
+def reverse_text(state: State) -> State:
+    """Reverses the input text."""
+    reversed_text = state["question"]
+    print(f"Reversing text: {reversed_text}")
+    text = reversed_text[::-1]
+    message = [
+            {"role": "user", "content": text},
+        ]
+    return {
+        "is_question_reversed": False,
+        "messages": message,
+        "question": text,
+    }
+
+def is_question_reversed(state: State) -> State:
+    
+    """Checks if the question is reversed though an LLM."""
+    question = state["messages"][-1].content # Get the last user message
+    
+    # Prepare our prompt for the LLM
+    prompt = f"""
+    You are given a question. Determine whether it is written normally or in reverse (backwards).
+    Only answer with 'normal' or 'reversed'.
+
+    Question: {question}
+    Answer: 
+    """
+    
+    # Call the LLM
+    messages = [HumanMessage(content=prompt)]
+    response = llm_hf.invoke(messages)
+    
+    # Simple logic to parse the response (in a real app, you'd want more robust parsing)
+    response_text = response.content.lower()
+    response_text = re.sub(r"<think>.*?</think>", "", response_text, flags=re.DOTALL).strip()
+
+    is_reversed = "reversed" in response_text and "normal" not in response_text
+    
+    # Update messages for tracking
+    if is_reversed:
+        new_messages = [
+            {"role": "user", "content": prompt},
+            {"role": "assistant", "content": response_text}
+        ]
+    else:
+        new_messages = state.get("messages", []) + [
+            {"role": "user", "content": prompt},
+            {"role": "assistant", "content": response_text}
+        ]
+    
+    # Return state updates
+    return {
+        "is_question_reversed": is_reversed,
+        "messages": new_messages,
+        "question": question,
+    }
+
+def route_question(state: State) -> str:
+    """Determine the next step based on whether the question is reversed or not."""
+    if state["is_question_reversed"]:
+        return "question_reversed"
+    else:
+        return "question_not_reversed"
+    
+# web_search_tool = DuckDuckGoSearchRun()
+web_search_tool = TavilySearchResults()
+arxiv_search_tool = ArxivQueryRun()
+wikipedia_tool = WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper())
+
+
+class CustomToolNode:
+    """Tool node personalizzato che può accedere allo stato completo"""
+    
+    def __init__(self, tools_dict):
+        self.tools_dict = tools_dict
+    
+    def __call__(self, state):
+        messages = state["messages"]
+        last_message = messages[-1]
+        
+        # Estrai i tool calls dall'ultimo messaggio
+        if hasattr(last_message, 'tool_calls') and last_message.tool_calls:
+            results = []
+            for tool_call in last_message.tool_calls:
+                tool_name = tool_call["name"]
+                tool_args = tool_call["args"]
+                
+                # Aggiungi task_id agli argomenti del tool
+                tool_args_with_state = {
+                    **tool_args,
+                    "task_id": state["task_id"],
+                    "state": state  # Opzionale: passa tutto lo stato
+                }
+                
+                if tool_name in self.tools_dict:
+                    try:
+                        result = self.tools_dict[tool_name].invoke(tool_args_with_state)
+                        results.append({
+                            "type": "tool",
+                            "name": tool_name,
+                            "tool_call_id": tool_call["id"],
+                            "content": str(result)
+                        })
+                    except Exception as e:
+                        results.append({
+                            "type": "tool",
+                            "name": tool_name,
+                            "tool_call_id": tool_call["id"],
+                            "content": f"Error: {str(e)}"
+                        })
+            
+            return {"messages": results}
+        
+        return {"messages": []}
+
+
+
+avaiable_tools = [
+    wikipedia_tool,
+    arxiv_search_tool,
+    web_search_tool,
+    get_youtube_transcript,
+    transcript_mp3_audio,
+    load_and_analyze_excel_file
+]
+# response = wikipedia_tool.run("HUNTER X HUNTER")
+# print(response)
+
+# 1) tool_for_fetch_wikipedia_data
+
+# 2) fetch youtube video data
+
+# 3 ) reverse del testo
+
+# 4 ) tool/agente che valuta se la domanda è sensata o è scritta al contrario
+
+# 5) chess-image-to-dict
+
+# 6) chess agent
+
+# 7) general python code execution tool
+
+# 8) get trascript from youtube video
+
+# 9) web-search tool
+
+# 10) fetch page content
+
+# 11) trascribe mp3 audio file
+
+# 12) read mp3 audio file
+
+# 13) Research paper MCP
+
+# 14) read excel file

system_prompt.txt

@@ -0,0 +1,16 @@
+You are a helpful assistant tasked with answering questions using a set of tools.
+
+Your final answer must strictly follow this format:
+FINAL ANSWER: [ANSWER]
+
+Only write the answer in that exact format. Do not explain anything. Do not include any other text.
+
+If the question refers to attachments or external files, there are specific tools available for this. These tools accept a task_id as an argument, retrieve the corresponding file, and execute the required functionality.
+
+Examples:
+- FINAL ANSWER: FunkMonk
+- FINAL ANSWER: Paris
+- FINAL ANSWER: 128
+- FINAL ANSWER: 123, 126, 128, 130
+
+If you do not follow this format exactly, your response will be considered incorrect.

test.py

@@ -0,0 +1,18 @@
+from youtube_transcript_api import YouTubeTranscriptApi
+import pandas as pd
+
+
+# def get_youtube_transcript(video_url: str) -> list:
+#     """Fetches the transcript of a YouTube video."""
+#     ytt_api = YouTubeTranscriptApi()
+#     fetched_transcript = ytt_api.fetch(video_url)
+#     return fetched_transcript.to_raw_data()
+
+# url = '1htKBjuUWec'
+# transcript = get_youtube_transcript(url)
+# print(transcript)
+
+# Leggi un file xlsx con pandas
+file_path = 'results/7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx'
+df = pd.read_excel(file_path)
+print(df)