added audio transcriber

main_agent.py

@@ -4,7 +4,7 @@ from langgraph.graph.message import add_messages
 from langchain_core.messages import AnyMessage, SystemMessage
 from langchain_openai import AzureChatOpenAI
 from langgraph.graph import START, StateGraph
-from tools import duckduck_tool, wiki_RAG_tool, image_analyser_tool
+from tools import duckduck_tool, wiki_RAG_tool, image_analyser_tool, audio_transcriber_tool
 from langgraph.prebuilt import ToolNode, tools_condition
 
 import prompts_lib as my_prompts
@@ -30,7 +30,7 @@ llm = AzureChatOpenAI(
     )
 
 # bild tools
-tools = [duckduck_tool, wiki_RAG_tool, image_analyser_tool]
+tools = [duckduck_tool, wiki_RAG_tool, image_analyser_tool, audio_transcriber_tool]
 chat_w_tools = llm.bind_tools(tools)
 
 # load system prompt

test_app2.ipynb

@@ -28,7 +28,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 32,
    "id": "64e382be",
    "metadata": {},
    "outputs": [],
@@ -45,7 +45,7 @@
     "\n",
     "for i, item in enumerate(questions_data):\n",
     "\n",
-    "    process = [3]\n",
+    "    process = [13]\n",
     "    if i not in process:\n",
     "        continue\n",
     "\n",
@@ -55,17 +55,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 33,
    "id": "5076d2ac",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "\"Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.\""
+       "\"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(\\n\\nCould you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.\""
       ]
      },
-     "execution_count": 9,
+     "execution_count": 33,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -76,7 +76,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 36,
    "id": "c3d1d2be",
    "metadata": {},
    "outputs": [],
@@ -130,7 +130,7 @@
     "    print(f\"Running agent on {len(questions_data)} questions...\")\n",
     "    for i, item in enumerate(questions_data):\n",
     "\n",
-    "        process = [3]\n",
+    "        process = [13]\n",
     "        if i not in process:\n",
     "            continue\n",
     "\n",
@@ -203,7 +203,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 37,
    "id": "2d2800d9",
    "metadata": {},
    "outputs": [
@@ -215,24 +215,24 @@
       "Fetching questions from: https://agents-course-unit4-scoring.hf.space/questions\n",
       "Fetched 20 questions.\n",
       "Running agent on 20 questions...\n",
-      "Downloaded file for task cca530fc-4052-43b2-b130-b30968d8aa44 to C:\\Users\\ANDREA~1.GIO\\AppData\\Local\\Temp\\tmpvysw5uzj\\cca530fc-4052-43b2-b130-b30968d8aa44.png\n"
+      "Downloaded file for task 1f975693-876d-457b-a649-393859e79bf3 to C:\\Users\\ANDREA~1.GIO\\AppData\\Local\\Temp\\tmpn7a3k53q\\1f975693-876d-457b-a649-393859e79bf3.mp3\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "[{'Task ID': 'cca530fc-4052-43b2-b130-b30968d8aa44',\n",
-       "  'Question': \"Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.\",\n",
-       "  'Submitted Answer': {'messages': [HumanMessage(content=\"Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.\", additional_kwargs={}, response_metadata={}, id='06b6aeb0-6ee4-431b-b612-6656344186a3'),\n",
-       "    SystemMessage(content='File path provided: C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpvysw5uzj\\\\cca530fc-4052-43b2-b130-b30968d8aa44.png', additional_kwargs={}, response_metadata={}, id='090a87ef-a41b-4cb7-b0b8-bbd9b67a985c'),\n",
-       "    AIMessage(content='', additional_kwargs={'tool_calls': [{'id': 'call_9k5K33EtplnDY0qTud6PeFki', 'function': {'arguments': '{\"image_path\":\"C:\\\\\\\\Users\\\\\\\\ANDREA~1.GIO\\\\\\\\AppData\\\\\\\\Local\\\\\\\\Temp\\\\\\\\tmpvysw5uzj\\\\\\\\cca530fc-4052-43b2-b130-b30968d8aa44.png\"}', 'name': 'image-analyser'}, 'type': 'function'}], 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 62, 'prompt_tokens': 381, 'total_tokens': 443, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-11-20', 'system_fingerprint': 'fp_ee1d74bde0', 'id': 'chatcmpl-CGGh1yEgXBhEreyd0aKAmRczBe5G4', 'service_tier': None, 'prompt_filter_results': [{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}], 'finish_reason': 'tool_calls', 'logprobs': None, 'content_filter_results': {}}, id='run--1bdae0df-3aec-4584-aef8-118df8cea3d3-0', tool_calls=[{'name': 'image-analyser', 'args': {'image_path': 'C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpvysw5uzj\\\\cca530fc-4052-43b2-b130-b30968d8aa44.png'}, 'id': 'call_9k5K33EtplnDY0qTud6PeFki', 'type': 'tool_call'}], usage_metadata={'input_tokens': 381, 'output_tokens': 62, 'total_tokens': 443, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n",
-       "    ToolMessage(content=\"This image depicts a chess position on a standard 8x8 chessboard. The pieces are arranged as follows:\\n\\n### **White Pieces:**\\n- **King**: g1\\n- **Queen**: h5\\n- **Rooks**: e3\\n- **Bishops**: d3, c3\\n- **Pawns**: f2, g2, h3, a3\\n\\n### **Black Pieces:**\\n- **King**: g8\\n- **Queen**: b3\\n- **Rook**: d8\\n- **Bishop**: e6\\n- **Knight**: d4\\n- **Pawns**: f7, g7, h7, a7, b7, h6\\n\\nThe position appears to be in the middle or late stage of the game, with both sides having developed their pieces. Black's queen is aggressively placed on b3, and the knight on d4 is centralized. White's queen on h5 is threatening, and White has strong control over the center with the bishops and rook.\", name='image-analyser', id='b123287f-3d9b-4605-b95d-9f2ce541ed74', tool_call_id='call_9k5K33EtplnDY0qTud6PeFki'),\n",
-       "    SystemMessage(content='File path provided: C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpvysw5uzj\\\\cca530fc-4052-43b2-b130-b30968d8aa44.png', additional_kwargs={}, response_metadata={}, id='8afd3ce6-3fb6-486b-a8cf-68471f0acf96'),\n",
-       "    AIMessage(content='Qb1+', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 6, 'prompt_tokens': 726, 'total_tokens': 732, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-11-20', 'system_fingerprint': 'fp_ee1d74bde0', 'id': 'chatcmpl-CGGh79wNX9bhXB2uUKwyKStFSCDzG', 'service_tier': None, 'prompt_filter_results': [{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}], 'finish_reason': 'stop', 'logprobs': None, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}, id='run--217b1dea-81ed-435c-bc1f-1457c90f9e99-0', usage_metadata={'input_tokens': 726, 'output_tokens': 6, 'total_tokens': 732, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}})],\n",
-       "   'file_path': 'C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpvysw5uzj\\\\cca530fc-4052-43b2-b130-b30968d8aa44.png'}}]"
+       "[{'Task ID': '1f975693-876d-457b-a649-393859e79bf3',\n",
+       "  'Question': \"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(\\n\\nCould you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.\",\n",
+       "  'Submitted Answer': {'messages': [HumanMessage(content=\"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(\\n\\nCould you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.\", additional_kwargs={}, response_metadata={}, id='d3783293-cd5d-4c09-a3cf-e75a966f02a9'),\n",
+       "    SystemMessage(content='File path provided: C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpn7a3k53q\\\\1f975693-876d-457b-a649-393859e79bf3.mp3', additional_kwargs={}, response_metadata={}, id='ddb142ff-f32b-4097-a378-6169b5515ffc'),\n",
+       "    AIMessage(content='', additional_kwargs={'tool_calls': [{'id': 'call_XGiU0i4Rz0MuAO9wEtZeRu5j', 'function': {'arguments': '{\"audio_path\":\"C:\\\\\\\\Users\\\\\\\\ANDREA~1.GIO\\\\\\\\AppData\\\\\\\\Local\\\\\\\\Temp\\\\\\\\tmpn7a3k53q\\\\\\\\1f975693-876d-457b-a649-393859e79bf3.mp3\"}', 'name': 'audio-transcriber'}, 'type': 'function'}], 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 64, 'prompt_tokens': 500, 'total_tokens': 564, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-11-20', 'system_fingerprint': 'fp_ee1d74bde0', 'id': 'chatcmpl-CGISCs9DiCf5UJmEh2doQtepBaZdg', 'service_tier': None, 'prompt_filter_results': [{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}], 'finish_reason': 'tool_calls', 'logprobs': None, 'content_filter_results': {}}, id='run--e5ebc307-99f5-47b8-8024-80e115513513-0', tool_calls=[{'name': 'audio-transcriber', 'args': {'audio_path': 'C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpn7a3k53q\\\\1f975693-876d-457b-a649-393859e79bf3.mp3'}, 'id': 'call_XGiU0i4Rz0MuAO9wEtZeRu5j', 'type': 'tool_call'}], usage_metadata={'input_tokens': 500, 'output_tokens': 64, 'total_tokens': 564, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n",
+       "    ToolMessage(content='Transcription(text=\"Before you all go, I want to remind you that the midterm is next week. Here\\'s a little hint. You should be familiar with the differential equations on page 245. Problems that are very similar to problems 32, 33 and 44 from that page might be on the test, and also some of you might want to brush up on the last page in the integration section, page 197. I know some of you struggled on last week\\'s quiz. I foresee problem 22 from page 197 being on your midterm. Oh, and don\\'t forget to brush up on the section on related rates on pages 132, 133 and 134.\", logprobs=None, usage=None)', name='audio-transcriber', id='9c51ff74-7b74-4737-abda-76e4392761e0', tool_call_id='call_XGiU0i4Rz0MuAO9wEtZeRu5j'),\n",
+       "    SystemMessage(content='File path provided: C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpn7a3k53q\\\\1f975693-876d-457b-a649-393859e79bf3.mp3', additional_kwargs={}, response_metadata={}, id='9b898c81-a735-460d-ae95-eafb149b846f'),\n",
+       "    AIMessage(content='132,133,134,197,245', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 11, 'prompt_tokens': 773, 'total_tokens': 784, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-11-20', 'system_fingerprint': 'fp_ee1d74bde0', 'id': 'chatcmpl-CGISIEPpGfou4mkqp4KlYvhwsm2me', 'service_tier': None, 'prompt_filter_results': [{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}], 'finish_reason': 'stop', 'logprobs': None, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}, id='run--691d3a38-23be-4f38-9c6d-a5ee182f3c63-0', usage_metadata={'input_tokens': 773, 'output_tokens': 11, 'total_tokens': 784, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}})],\n",
+       "   'file_path': 'C:\\\\Users\\\\ANDREA~1.GIO\\\\AppData\\\\Local\\\\Temp\\\\tmpn7a3k53q\\\\1f975693-876d-457b-a649-393859e79bf3.mp3'}}]"
       ]
      },
-     "execution_count": 11,
+     "execution_count": 37,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -265,13 +265,47 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "id": "ee202a8a",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\andrea.giorni\\Desktop\\code\\Hugging face Agent course\\Final_Assignment_Template\\.venv310\\lib\\site-packages\\gradio\\oauth.py:162: UserWarning: Gradio does not support OAuth features outside of a Space environment. To help you debug your app locally, the login and logout buttons are mocked with your profile. To make it work, your machine must be logged in to Huggingface.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "Your machine must be logged in to HF to debug a Gradio app locally. Please run `huggingface-cli login` or set `HF_TOKEN` as environment variable with one of your access token. You can generate a new token in your settings page (https://huggingface.co/settings/tokens).",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[13], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mapp\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\Users\\andrea.giorni\\Desktop\\code\\Hugging face Agent course\\Final_Assignment_Template\\app.py:203\u001b[0m\n\u001b[0;32m    199\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m status_message, results_df\n\u001b[0;32m    202\u001b[0m \u001b[38;5;66;03m# --- Build Gradio Interface using Blocks ---\u001b[39;00m\n\u001b[1;32m--> 203\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m gr\u001b[38;5;241m.\u001b[39mBlocks() \u001b[38;5;28;01mas\u001b[39;00m demo:\n\u001b[0;32m    204\u001b[0m     gr\u001b[38;5;241m.\u001b[39mMarkdown(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# Basic Agent Evaluation Runner\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    205\u001b[0m     gr\u001b[38;5;241m.\u001b[39mMarkdown(\n\u001b[0;32m    206\u001b[0m \u001b[38;5;250m        \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    207\u001b[0m \u001b[38;5;124;03m        **Instructions:**\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    215\u001b[0m \u001b[38;5;124;03m        \"\"\"\u001b[39;00m\n\u001b[0;32m    216\u001b[0m     )\n",
+      "File \u001b[1;32mc:\\Users\\andrea.giorni\\Desktop\\code\\Hugging face Agent course\\Final_Assignment_Template\\.venv310\\lib\\site-packages\\gradio\\blocks.py:2318\u001b[0m, in \u001b[0;36mBlocks.__exit__\u001b[1;34m(self, exc_type, *args)\u001b[0m\n\u001b[0;32m   2316\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mparent\u001b[38;5;241m.\u001b[39mchildren\u001b[38;5;241m.\u001b[39mextend(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchildren)\n\u001b[0;32m   2317\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_config_file()\n\u001b[1;32m-> 2318\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapp \u001b[38;5;241m=\u001b[39m \u001b[43mApp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_app\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmcp_server\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m   2319\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprogress_tracking \u001b[38;5;241m=\u001b[39m \u001b[38;5;28many\u001b[39m(\n\u001b[0;32m   2320\u001b[0m     block_fn\u001b[38;5;241m.\u001b[39mtracks_progress \u001b[38;5;28;01mfor\u001b[39;00m block_fn \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfns\u001b[38;5;241m.\u001b[39mvalues()\n\u001b[0;32m   2321\u001b[0m )\n\u001b[0;32m   2322\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpage \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
+      "File \u001b[1;32mc:\\Users\\andrea.giorni\\Desktop\\code\\Hugging face Agent course\\Final_Assignment_Template\\.venv310\\lib\\site-packages\\gradio\\routes.py:568\u001b[0m, in \u001b[0;36mApp.create_app\u001b[1;34m(blocks, app_kwargs, auth_dependency, strict_cors, ssr_mode, mcp_server)\u001b[0m\n\u001b[0;32m    560\u001b[0m \u001b[38;5;66;03m###############\u001b[39;00m\n\u001b[0;32m    561\u001b[0m \u001b[38;5;66;03m# OAuth Routes\u001b[39;00m\n\u001b[0;32m    562\u001b[0m \u001b[38;5;66;03m###############\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    565\u001b[0m \u001b[38;5;66;03m# It allows users to \"Sign in with HuggingFace\". Otherwise, add the default\u001b[39;00m\n\u001b[0;32m    566\u001b[0m \u001b[38;5;66;03m# logout route.\u001b[39;00m\n\u001b[0;32m    567\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m app\u001b[38;5;241m.\u001b[39mblocks \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m app\u001b[38;5;241m.\u001b[39mblocks\u001b[38;5;241m.\u001b[39mexpects_oauth:\n\u001b[1;32m--> 568\u001b[0m     \u001b[43mattach_oauth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mapp\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    569\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    571\u001b[0m     \u001b[38;5;129m@app\u001b[39m\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/logout\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    572\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mlogout\u001b[39m(request: fastapi\u001b[38;5;241m.\u001b[39mRequest, user: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m=\u001b[39m Depends(get_current_user)):\n",
+      "File \u001b[1;32mc:\\Users\\andrea.giorni\\Desktop\\code\\Hugging face Agent course\\Final_Assignment_Template\\.venv310\\lib\\site-packages\\gradio\\oauth.py:41\u001b[0m, in \u001b[0;36mattach_oauth\u001b[1;34m(app)\u001b[0m\n\u001b[0;32m     39\u001b[0m     _add_oauth_routes(app)\n\u001b[0;32m     40\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 41\u001b[0m     \u001b[43m_add_mocked_oauth_routes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mapp\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     43\u001b[0m \u001b[38;5;66;03m# Session Middleware requires a secret key to sign the cookies. Let's use a hash\u001b[39;00m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;66;03m# of the OAuth secret key to make it unique to the Space + updated in case OAuth\u001b[39;00m\n\u001b[0;32m     45\u001b[0m \u001b[38;5;66;03m# config gets updated.\u001b[39;00m\n\u001b[0;32m     46\u001b[0m session_secret \u001b[38;5;241m=\u001b[39m (OAUTH_CLIENT_SECRET \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m-v4\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
+      "File \u001b[1;32mc:\\Users\\andrea.giorni\\Desktop\\code\\Hugging face Agent course\\Final_Assignment_Template\\.venv310\\lib\\site-packages\\gradio\\oauth.py:167\u001b[0m, in \u001b[0;36m_add_mocked_oauth_routes\u001b[1;34m(app)\u001b[0m\n\u001b[0;32m    157\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Add fake oauth routes if Gradio is run locally and OAuth is enabled.\u001b[39;00m\n\u001b[0;32m    158\u001b[0m \n\u001b[0;32m    159\u001b[0m \u001b[38;5;124;03mClicking on a gr.LoginButton will have the same behavior as in a Space (i.e. gets redirected in a new tab) but\u001b[39;00m\n\u001b[0;32m    160\u001b[0m \u001b[38;5;124;03minstead of authenticating with HF, a mocked user profile is added to the session.\u001b[39;00m\n\u001b[0;32m    161\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    162\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m    163\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGradio does not support OAuth features outside of a Space environment. To help\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    164\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m you debug your app locally, the login and logout buttons are mocked with your\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    165\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m profile. To make it work, your machine must be logged in to Huggingface.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    166\u001b[0m )\n\u001b[1;32m--> 167\u001b[0m mocked_oauth_info \u001b[38;5;241m=\u001b[39m \u001b[43m_get_mocked_oauth_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    169\u001b[0m \u001b[38;5;66;03m# Define OAuth routes\u001b[39;00m\n\u001b[0;32m    170\u001b[0m \u001b[38;5;129m@app\u001b[39m\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/login/huggingface\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    171\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21moauth_login\u001b[39m(request: fastapi\u001b[38;5;241m.\u001b[39mRequest):  \u001b[38;5;66;03m# noqa: ARG001\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\Users\\andrea.giorni\\Desktop\\code\\Hugging face Agent course\\Final_Assignment_Template\\.venv310\\lib\\site-packages\\gradio\\oauth.py:310\u001b[0m, in \u001b[0;36m_get_mocked_oauth_info\u001b[1;34m()\u001b[0m\n\u001b[0;32m    308\u001b[0m token \u001b[38;5;241m=\u001b[39m get_token()\n\u001b[0;32m    309\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m token \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m--> 310\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    311\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYour machine must be logged in to HF to debug a Gradio app locally. Please\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    312\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m run `huggingface-cli login` or set `HF_TOKEN` as environment variable \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    313\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwith one of your access token. You can generate a new token in your \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    314\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msettings page (https://huggingface.co/settings/tokens).\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    315\u001b[0m     )\n\u001b[0;32m    317\u001b[0m user \u001b[38;5;241m=\u001b[39m whoami()\n\u001b[0;32m    318\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m user[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muser\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
+      "\u001b[1;31mValueError\u001b[0m: Your machine must be logged in to HF to debug a Gradio app locally. Please run `huggingface-cli login` or set `HF_TOKEN` as environment variable with one of your access token. You can generate a new token in your settings page (https://huggingface.co/settings/tokens)."
+     ]
+    }
+   ],
    "source": [
-    "submitted_answer = agent.builder.invoke(question_text, file_path)"
+    "import app"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5def4105",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

tools.py

@@ -9,6 +9,7 @@ from langchain_core.tools import tool
 import base64
 import os
 from pydantic import BaseModel, Field
+from openai import AzureOpenAI
 
 
 from dotenv import load_dotenv  
@@ -72,10 +73,6 @@ llm_img = AzureChatOpenAI(
     temperature=0
     )
 
-
-
-
-
 class ImageAnalyserInput(BaseModel):
     image_path: str = Field(description="path to file")
 
@@ -106,11 +103,26 @@ def image_analyser_tool(image_path: str) -> str:
     response = llm_img.invoke([message])
     return response.content
 
-# image_analyser_tool = Tool(
-#     name="image analyser",
-#     func=image_analyser,
-#     description="Analises an image and returns a text description"
-# )
 
 
+# create wishper interface
+client = AzureOpenAI(
+    api_key=os.getenv("AZURE_OPENAI_API_KEY"),  
+    api_version="2024-02-01",
+    azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+    )
+
+class AudioTranscriberInput(BaseModel):
+    audio_path: str = Field(description="path to audio file")
+
+@tool("audio-transcriber", args_schema=AudioTranscriberInput)
+def audio_transcriber_tool(audio_path: str) -> str:
+    """Receives path to audio file and returns text transcription of the audio recording."""
+
+    result = client.audio.transcriptions.create(
+        file=open(audio_path, "rb"),            
+        model='whisper'
+    )
+    return result
+