add presentation notebook

.gitignore

@@ -135,3 +135,5 @@ cython_debug/
 
 # configurations for VS Code
 .vscode
+
+data/

notebooks/presentation.ipynb

@@ -0,0 +1,218 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "64b2c237",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from langchain_huggingface import HuggingFaceEndpoint, ChatHuggingFace\n",
+    "\n",
+    "import sys\n",
+    "\n",
+    "sys.path.append(os.path.abspath(\"../src\"))\n",
+    "\n",
+    "from agent import SmartAgent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "8a1ece26",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "HUGGINGFACEHUB_API_TOKEN = os.getenv(\"HUGGINGFACEHUB_API_TOKEN\")\n",
+    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
+    "TAVILY_API_KEY = os.getenv(\"TAVILY_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1d5bd941",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MODEL_ID = \"gpt-4o\"\n",
+    "PROVIDER_TYPE = \"openai\"  # \"openai\" or \"huggingface\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "711e347b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Agent initialized.\n",
+      "Telemetry initialized.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Instantiate Agent\n",
+    "try:\n",
+    "    if PROVIDER_TYPE == \"huggingface\":\n",
+    "        llm = HuggingFaceEndpoint(\n",
+    "            repo_id=MODEL_ID,\n",
+    "            huggingfacehub_api_token=HUGGINGFACEHUB_API_TOKEN,\n",
+    "        )\n",
+    "        chat = ChatHuggingFace(llm=llm, verbose=True)\n",
+    "    elif PROVIDER_TYPE == \"openai\":\n",
+    "        chat = ChatOpenAI(model=MODEL_ID, temperature=0.2)\n",
+    "    else:\n",
+    "        print(f\"Provider {PROVIDER_TYPE} not supported.\")\n",
+    "\n",
+    "    agent = SmartAgent(chat)\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(f\"Error instantiating agent: {e}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "57656b10",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Agent received question: The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places..\n",
+      "Provided file: ../data/sales.xlsx.\n",
+      "Agent returning answer: 29623.00\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run Agent\n",
+    "\n",
+    "question = \"The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.\"\n",
+    "filename = \"../data/sales.xlsx\"\n",
+    "answer = agent(question, filename)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0f82fc20",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Agent received question: Here is a scanned invoice. Please extract the vendor name, invoice number, and total amount..\n",
+      "Provided file: ../data/invoice.png.\n",
+      "Agent returning answer: Adeline Palmerston, 01234, 440\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run Agent\n",
+    "\n",
+    "question = \"Here is a scanned invoice. Please extract the vendor name, invoice number, and total amount.\"\n",
+    "filename = \"../data/invoice.png\"\n",
+    "answer = agent(question, filename)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "833cb42b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Agent received question: Explain in detail what the conclusion of the paper Attention is all you need.\n",
+      "Agent returning answer: The conclusion of the paper \"Attention is All You Need\" is that the Transformer model, which relies entirely on attention mechanisms and does not use recurrence or convolution, achieves state-of-the-art results on translation tasks and is highly efficient in terms of parallelization, making it suitable for training on large datasets. The paper demonstrates that attention mechanisms alone are sufficient for achieving high performance in sequence transduction tasks.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run Agent\n",
+    "\n",
+    "question = (\n",
+    "    \"Explain in detail what the conclusion of the paper Attention is all you need\"\n",
+    ")\n",
+    "answer = agent(question)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "9077c2b0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Agent received question: Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec. What does Teal say in response to the question Isn't that hot?.\n",
+      "                                                                       \r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/giulia/Progetti/Agent_Course_Final_Assignment/.venv/lib/python3.10/site-packages/whisper/transcribe.py:126: UserWarning: FP16 is not supported on CPU; using FP32 instead\n",
+      "  warnings.warn(\"FP16 is not supported on CPU; using FP32 instead\")\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Agent returning answer: Extremely\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run Agent\n",
+    "\n",
+    "question = \"Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec. What does Teal say in response to the question Isn't that hot?\"\n",
+    "answer = agent(question)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ee42f12",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/agent.py

@@ -10,18 +10,38 @@ from langgraph.graph import START, StateGraph
 from langgraph.graph.message import add_messages
 from langgraph.prebuilt import ToolNode, tools_condition
 
-from .tools import (
-    DescribeImage,
-    ExtractTextFromImage,
-    arxiv_search,
-    download_youtube_video,
-    extract_audio_from_video,
-    read_excel,
-    read_python,
-    transcribe_audio,
-    web_search,
-    wiki_search,
-)
+try:
+    from tools import (
+        DescribeImage,
+        ExtractTextFromImage,
+        arxiv_search,
+        download_youtube_video,
+        extract_audio_from_video,
+        read_excel,
+        read_python,
+        transcribe_audio,
+        web_search,
+        wiki_search,
+        add,
+        divide,
+        multiply,
+    )
+except:
+    from .tools import (
+        DescribeImage,
+        ExtractTextFromImage,
+        arxiv_search,
+        download_youtube_video,
+        extract_audio_from_video,
+        read_excel,
+        read_python,
+        transcribe_audio,
+        web_search,
+        wiki_search,
+        add,
+        divide,
+        multiply,
+    )
 
 
 class AgentState(TypedDict):
@@ -53,6 +73,9 @@ class SmartAgent:
             arxiv_search,
             download_youtube_video,
             extract_audio_from_video,
+            add,
+            divide,
+            multiply,
         ]
         self.chat_with_tools = chat.bind_tools(self.tools)
         self._initialize_graph()
@@ -91,7 +114,7 @@ class SmartAgent:
         self.langfuse_handler = CallbackHandler()
         print("Telemetry initialized.")
 
-    def __call__(self, question: str, file_name: str) -> str:
+    def __call__(self, question: str, file_name: str | None = None) -> str:
         """Call the agent, passing system prompt and eventual file name."""
         sys_msg = SystemMessage(
             content="""You are a general AI assistant. You will be asked a factual question. 

src/tools.py

@@ -8,6 +8,11 @@ from langchain.tools import tool
 from langchain.tools.tavily_search import TavilySearchResults
 from langchain_community.document_loaders import ArxivLoader, WikipediaLoader
 from langchain_core.messages import HumanMessage
+from typing import List
+from functools import reduce
+import operator
+import contextlib
+import os
 
 
 @tool
@@ -204,13 +209,14 @@ def download_youtube_video(youtube_url: str, output_path: str) -> str:
         Path to the saved video file.
     """
     ydl_opts = {
-        "format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
+        "format": "bestvideo+bestaudio/best",
         "outtmpl": output_path,
         "merge_output_format": "mp4",
         "quiet": True,
     }
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        ydl.download([youtube_url])
+    with contextlib.redirect_stderr(open(os.devnull, "w")):
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            ydl.download([youtube_url])
     return output_path
 
 
@@ -277,10 +283,11 @@ def web_search(query: str) -> str:
 
 @tool
 def arxiv_search(query: str) -> str:
-    """Search Arxiv for a query and return maximum 2 result.
+    """Search Arxiv for a paper.
 
     Args:
-        query: The search query.
+        query: The search query to retrieve a specific paper, consisting
+        of title and/or authors name and/or year of publication.
     """
     search_docs = ArxivLoader(query=query, load_max_docs=2).load()
     formatted_search_docs = "\n\n---\n\n".join(
@@ -297,3 +304,40 @@ def arxiv_search(query: str) -> str:
         ]
     )
     return {"arvix_results": formatted_search_docs}
+
+
+@tool
+def add(numbers: List[float]) -> float:
+    """Calculates the sum of a list of numbers.
+
+    Args:
+        numbers: A list of numeric values to be summed.
+
+    Returns:
+        The sum of all numbers in the list.
+    """
+    return sum(numbers)
+
+
+@tool
+def multiply(numbers: List[float]) -> float:
+    """Calculates the product of a list of numbers.
+
+    Args:
+        numbers: A list of numeric values to be multiplied.
+
+    Returns:
+        The product of all numbers in the list.
+    """
+    return reduce(operator.mul, numbers, 1.0)
+
+
+@tool
+def divide(a: int, b: int) -> float:
+    """Divide a and b.
+
+    Args:
+        a: first number
+        b: second number
+    """
+    return a / b