50% tuned prompt and other stuff

agent.py

@@ -16,6 +16,11 @@ from langchain_community.tools.tavily_search import TavilySearchResults
 # Python loader
 from langchain_community.document_loaders import PythonLoader
 
+# Whisper
+from langchain_community.document_loaders.parsers.audio import AzureOpenAIWhisperParser
+from langchain_core.documents.base import Blob
+
+
 load_dotenv()
 
 
@@ -33,6 +38,8 @@ def meaning_of_life(a: int, b: int) -> int:
 
 # https://www.restack.io/docs/langchain-knowledge-wikipedia-loader-cat-ai
 # https://api.python.langchain.com/en/latest/community/document_loaders/langchain_community.document_loaders.wikipedia.WikipediaLoader.html#
+# ¤ I ended up not using this tool since I could not get it to return the table data in the Markov question. The Taveli search tool also find wiki content
+# Better approach could be to combine this tool (to get URL) +  a webreader to get content
 def wikipedia_search(query: str) -> str:
     """Searches Wikipedia for a given query and fetches full document
 
@@ -49,7 +56,7 @@ def wikipedia_search(query: str) -> str:
     formatted_search_docs = "\n\n---\n\n"
 
     for next_doc in documents:
-        formatted_doc = f'<Document source="{next_doc.metadata["source"]}" title="{next_doc.metadata.get("title", "")}"/>\n{next_doc.page_content}\n</Document>'
+        formatted_doc = f'<Document source="{next_doc.metadata["source"]}" title="{next_doc.metadata.get("title", "")}"\n{next_doc.page_content}\n</Document>'
         formatted_search_docs = formatted_search_docs + formatted_doc
 
     result = f"{{wiki_results: {formatted_search_docs}}}"
@@ -79,7 +86,7 @@ def web_search(query: str) -> str:
         title = next_doc["title"]
         content = next_doc["content"]
         formatted_doc = (
-            f'<Document source="{url}" title="{title}"/>\n{content}\n</Document>'
+            f'<Document source="{url}" title="{title}"\n{content}\n</Document>'
         )
         formatted_search_docs = formatted_search_docs + formatted_doc
 
@@ -102,7 +109,7 @@ def python_file_reader(file_name: str) -> str:
 
     for next_doc in documents:
         formatted_doc = (
-            f'<Document source="{file_name}"/>\n{next_doc.page_content}\n</Document>'
+            f'<Document source="{file_name}"\n{next_doc.page_content}\n</Document>'
         )
         formatted_search_docs = formatted_search_docs + formatted_doc
 
@@ -111,7 +118,49 @@ def python_file_reader(file_name: str) -> str:
     return result
 
 
-tools = [meaning_of_life, web_search, python_file_reader]
+# https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.parsers.audio.AzureOpenAIWhisperParser.html
+def audio_to_text(audio_file_name: str) -> str:
+    """Listen to audio and extract text from speech
+
+    Args:
+        audio_file_name: the audio filename to read
+    """
+    file_path = os.path.join(os.path.dirname(__file__), "files", audio_file_name)
+
+    deployment_name = os.environ.get("AZURE_WHISPER_DEPLOYMENT")
+    api_version = os.environ.get("AZURE_WHISPER_API_VERSION")
+    api_key = os.environ.get("AZURE_WHISPER_API_KEY")
+    azure_endpoint = os.environ.get("AZURE_WHISPER_ENDPOINT")
+
+    whisper_parser = AzureOpenAIWhisperParser(
+        deployment_name=deployment_name,
+        api_version=api_version,
+        api_key=api_key,
+        azure_endpoint=azure_endpoint,
+        # other params...
+    )
+
+    audio_blob = Blob(path=file_path)
+    response = whisper_parser.parse(audio_blob)
+
+    formatted_search_docs = "\n\n---\n\n"
+
+    for next_doc in response:
+        formatted_doc = f'<Document source="{audio_file_name}"\n{next_doc.page_content}\n</Document>'
+        formatted_search_docs = formatted_search_docs + formatted_doc
+
+    result = f"{{transscribed_audio: {formatted_search_docs}}}"
+
+    return result
+
+
+tools = [
+    meaning_of_life,
+    web_search,
+    python_file_reader,
+    audio_to_text,
+    wikipedia_search,
+]
 
 
 # --- GRAPH ---
@@ -133,7 +182,7 @@ def create_graph():
     llm = AzureChatOpenAI(
         azure_deployment=deployment,
         api_version=api_version,
-        temperature=0,
+        temperature=0.01,
         max_tokens=None,
         timeout=None,
         max_retries=2,
@@ -144,7 +193,7 @@ def create_graph():
 
     # System message
     original_system_prompt_txt = "You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string."
-    system_prompt_txt = "You are a general AI assistant that uses tools to answer questions. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string."
+    system_prompt_txt = "You are a general AI assistant that uses tools to answer questions. YOUR FINAL ANSWER should be a number represented as digits OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number or how many, only reply with a number represented as digits nothing else, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for an abbreviation or a code only reply with that. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string."
 
     sys_msg = SystemMessage(system_prompt_txt)
 
@@ -211,7 +260,7 @@ if __name__ == "__main__":
     messages = graph.invoke({"messages": messages})
     for m in messages["messages"]:
         m.pretty_print()
-"""
+
 
     print("******** PYTHON LOAD TOOL ********")
     question = "what does this python code do? filename is f918266a-b3e0-4914-865d-4faa564f1aef.py"
@@ -219,3 +268,18 @@ if __name__ == "__main__":
     messages = graph.invoke({"messages": messages})
     for m in messages["messages"]:
         m.pretty_print()
+
+
+    print("******** TRANSSCRIBE AUDIO TOOL ********")
+    question = "Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :( Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order. File to use is 1f975693-876d-457b-a649-393859e79bf3.mp3"
+    messages = [HumanMessage(content=question)]
+    messages = graph.invoke({"messages": messages})
+    for m in messages["messages"]:
+        m.pretty_print()
+"""
+
+    question = "What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?"
+    messages = [HumanMessage(content=question)]
+    messages = graph.invoke({"messages": messages})
+    for m in messages["messages"]:
+        m.pretty_print()