rebuild

.gitignore

@@ -1,2 +1,6 @@
 .env
-*.json
+*.json
+TODO.md
+.venv
+test.py
+__pycache__

agent.py

@@ -1,132 +1,13 @@
 import os
-from smolagents import CodeAgent, tool, DuckDuckGoSearchTool, OpenAIServerModel, VisitWebpageTool, PythonInterpreterTool, SpeechToTextTool
-from langchain_community.document_loaders import WikipediaLoader, ArxivLoader
-from tools.csv_reader import CsvReaderTool
-from tools.excel_reader import ExcelReaderTool
-from tools.file_downloader import FileDownloaderTool
-from tools.img_txt_extractor import ImageTextExtractorTool
-from tools.pdf_reader import PdfReaderTool
-
-@tool
-def add(a:int, b:int) -> int:
-  """
-  This tool returns the sum of two numbers.
-
-  Args:
-    a: first number
-    b: second number
-  """
-
-  return a+b
-
-
-@tool
-def subtract(a:int, b:int) -> int:
-  """
-  This tool returns the difference between two numbers.
-
-  Args:
-    a: first number
-    b: second number
-  """
-
-  return a-b
-
-
-@tool
-def multiply(a:int, b:int) -> int:
-  """
-  This tool multiplies two numbers.
-
-  Args:
-    a: first number
-    b: second number
-  """
-
-  return a*b
-
-
-@tool
-def divide(a:int, b:int) -> float:
-  """
-  This tool divides two numbers.
-
-  Args:
-    a: first number
-    b: second number
-  """
-
-  if b==0: raise ValueError('Cannot divide by zero')
-  return a/b
-
-
-@tool
-def modulus(a:int, b:int) -> int:
-  """
-  This tool returns the modulus of two numbers.
-
-  Args:
-    a: first number
-    b: second number
-  """
-
-  return a%b
-
-
-@tool
-def rounder(a:float, n:int) -> float:
-  """
-  This tool return a number rounded to a certain number of decimals.
-
-  Args:
-    a: number to be rounded
-    n: number of decimals to use when rounding the number
-  """
-
-  return round(a,n)
-
-@tool
-def wiki_search(query: str) -> str:
-    """Search Wikipedia for a query and return maximum 2 results.
-    
-    Args:
-        query: The search query."""
-    search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"wiki_results": formatted_search_docs}
-
-
-@tool
-def arvix_search(query: str) -> str:
-    """Search Arxiv for a query and return maximum 3 result.
-    
-    Args:
-        query: The search query."""
-    search_docs = ArxivLoader(query=query, load_max_docs=3).load()
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"arvix_results": formatted_search_docs}
-
+from smolagents import CodeAgent, DuckDuckGoSearchTool, OpenAIServerModel, VisitWebpageTool
+from tools.attached_files import csv_reader, download_file_from_url, excel_reader
+from tools.basic_math import *
+from tools.browser import arvix_search, wiki_search
+from tools.media import transcribe_audio
 
 def get_agent() -> CodeAgent:
   search_tool = DuckDuckGoSearchTool(max_results=5)
   web_page_tool = VisitWebpageTool(max_output_length=1_000_000)
-  py_interpreter = PythonInterpreterTool()
-
-  csv_reader = CsvReaderTool()
-  excel_reader = ExcelReaderTool()
-  file_downloader = FileDownloaderTool()
-  img_txt_extractor = ImageTextExtractorTool()
-  pdf_reader = PdfReaderTool()
-
-  speech_txt = SpeechToTextTool()
 
   api_key = os.getenv('CODESTRAL_API_KEY')
 
@@ -135,4 +16,4 @@ def get_agent() -> CodeAgent:
     api_base="https://codestral.mistral.ai/v1/",
     api_key=api_key)
 
-  return CodeAgent(tools=[add, subtract, multiply, divide, modulus, rounder, search_tool, web_page_tool, py_interpreter, wiki_search, arvix_search, csv_reader, excel_reader, file_downloader, img_txt_extractor, pdf_reader, speech_txt], model=model)
+  return CodeAgent(tools=[search_tool, web_page_tool, add, subtract, multiply, divide, modulus, rounder, power, square_root, download_file_from_url, csv_reader, excel_reader, transcribe_audio, wiki_search, arvix_search], model=model, planning_interval=None, additional_authorized_imports=['random', 'time'], verbosity_level=1, max_steps=5, add_base_tools=True)

app.py

@@ -72,7 +72,7 @@ def run_random_question(profile: gr.OAuthProfile | None):
         return "Random question is missing task_id or question", None
     try:
         with open('system_prompt.txt') as f:
-            submitted_answer = agent(f.readline()+"\n\n"+question_text)
+            submitted_answer = agent(f.read()+"\n\n"+question_text)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
     except Exception as e:

requirements.txt

@@ -2,15 +2,14 @@ gradio
 requests
 pandas
 smolagents
+smolagents[toolkit]
 smolagents[openai]
 smolagents[transformers]
-smolagents[audio]
 langchain_community
 wikipedia
 arxiv
 pymupdf
 duckduckgo_search
-requests
 markdownify
 openpyxl
 easyocr

system_prompt.txt

@@ -1 +1,5 @@
-You are a helpful assistant tasked with answering questions using a set of tools. Now, I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string. Your answer should only start with "FINAL ANSWER: ", then follows with the answer.
+You are a helpful assistant tasked with answering questions using a set of tools. 
+Now, I will ask you a question. Report your thoughts, and finish your answer with the following template: 
+FINAL ANSWER: [YOUR FINAL ANSWER]. 
+YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, Apply the rules above for each element (number or string), ensure there is exactly one space after each comma.
+Your answer should only start with "FINAL ANSWER: ", then follows with the answer.

tools/attached_files.py

@@ -0,0 +1,50 @@
+from smolagents import tool
+import urllib
+import pandas as pd
+
+@tool
+def download_file_from_url(url: str) -> str:
+  """
+  Download a file from a URL and save it to a temporary location.
+  Args:
+    url: the URL of the file to download.
+  """
+  file_path = None
+
+  try:
+    result = urllib.request.urlretrieve(url)
+    file_path = result[0]
+  except Exception as e:
+    return f"Error downloading file: {str(e)}"
+
+  return file_path
+
+@tool
+def csv_reader(file_path: str) -> str:
+  """
+  Extract CSV file content and return it in a json format. Supported file extensions: .csv
+  Args:
+    file_path: the path to the CSV file.
+  """
+
+  try:
+    df = pd.read_csv(file_path)
+    return df.to_json()
+
+  except Exception as e:
+    return f"Error analyzing CSV file: {str(e)}"
+
+@tool
+def excel_reader(file_path: str) -> str:
+  """
+  Extract Excel file content and return it in a json format. Supported file extensions: .xls, .xlsx, .xlsb, .xlsm, .odf, .ods, .odt
+  Args:
+    file_path: the path to the Excel file.
+  """
+
+  try:
+    df = pd.read_excel(file_path)
+    return df.to_json()
+
+  except Exception as e:
+    return f"Error analyzing Excel file: {str(e)}"

tools/basic_math.py

@@ -0,0 +1,105 @@
+from smolagents import tool
+import cmath
+
+@tool
+def add(a:int, b:int) -> int:
+  """
+  This tool returns the sum of two numbers.
+
+  Args:
+    a: first number
+    b: second number
+  """
+
+  return a+b
+
+
+@tool
+def subtract(a:int, b:int) -> int:
+  """
+  This tool returns the difference between two numbers.
+
+  Args:
+    a: first number
+    b: second number
+  """
+
+  return a-b
+
+
+@tool
+def multiply(a:int, b:int) -> int:
+  """
+  This tool multiplies two numbers.
+
+  Args:
+    a: first number
+    b: second number
+  """
+
+  return a*b
+
+
+@tool
+def divide(a:int, b:int) -> float:
+  """
+  This tool divides two numbers.
+
+  Args:
+    a: first number
+    b: second number
+  """
+
+  if b==0: raise ValueError('Cannot divide by zero')
+  return a/b
+
+
+@tool
+def modulus(a:int, b:int) -> int:
+  """
+  This tool returns the modulus of two numbers.
+
+  Args:
+    a: first number
+    b: second number
+  """
+
+  return a%b
+
+
+@tool
+def rounder(a:float, n:int) -> float:
+  """
+  This tool return a number rounded to a certain number of decimals.
+
+  Args:
+    a: number to be rounded
+    n: number of decimals to use when rounding the number
+  """
+
+  return round(a,n)
+
+
+@tool
+def power(a: float, b: float) -> float:
+  """
+  Get the power of two numbers.
+  Args:
+    a: the first number
+    b: the second number
+  """
+
+  return a**b
+
+
+@tool
+def square_root(a: float) -> float | complex:
+  """
+  Get the square root of a number.
+  Args:
+    a: the number to get the square root of
+  """
+
+  if a >= 0:
+    return a**0.5
+  return cmath.sqrt(a)

tools/browser.py

@@ -0,0 +1,31 @@
+from smolagents import tool
+from langchain_community.document_loaders import WikipediaLoader, ArxivLoader
+
+@tool
+def wiki_search(query: str) -> str:
+    """Search Wikipedia for a query and return maximum 2 results.
+    
+    Args:
+        query: The search query."""
+    search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
+    formatted_search_docs = "\n\n---\n\n".join(
+        [
+            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
+            for doc in search_docs
+        ])
+    return {"wiki_results": formatted_search_docs}
+
+
+@tool
+def arvix_search(query: str) -> str:
+    """Search Arxiv for a query and return maximum 3 result.
+    
+    Args:
+        query: The search query."""
+    search_docs = ArxivLoader(query=query, load_max_docs=3).load()
+    formatted_search_docs = "\n\n---\n\n".join(
+        [
+            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
+            for doc in search_docs
+        ])
+    return {"arvix_results": formatted_search_docs}

tools/csv_reader.py

@@ -1,21 +0,0 @@
-import pandas as pd
-from smolagents import Tool
-
-class CsvReaderTool(Tool):
-    name = "csv_reader"
-    description = "Extract CSV file content. Supported file extensions: .csv"
-    inputs = {
-        "file_path": {
-            "type": "string",
-            "description": "Path to the CSV file",
-        }
-    }
-    output_type = "string"
-
-    def forward(self, file_path) -> str:
-        try:
-            df = pd.read_csv(file_path)
-            print(f"Describe CSV file:\n {df.describe()}")
-            return df.to_json()
-        except Exception as e:
-            return f"Error processing CSV file: {str(e)}"

tools/excel_reader.py

@@ -1,21 +0,0 @@
-import pandas as pd
-from smolagents import Tool
-
-class ExcelReaderTool(Tool):
-  name = "excel_reader"
-  description = "Extract Excel file content. Supported file extensions: .xls, .xlsx, .xlsb, .xlsm, .odf, .ods, .odt"
-  inputs = {
-        "file_path": {
-            "type": "string",
-            "description": "Path to the Excel file",
-        }
-    }
-  output_type = "string"
-
-  def forward(self, file_path) -> str:
-      try:
-          df = pd.read_excel(file_path)
-          print(f"Describe Excel file:\n {df.describe()}")
-          return df.to_json()
-      except Exception as e:
-          return f"Error processing Excel file: {str(e)}"

tools/file_downloader.py

@@ -1,24 +0,0 @@
-import urllib.request
-from smolagents import Tool
-
-class FileDownloaderTool(Tool):
-    name = "file_downloader"
-    description = "Download a file from Internet by URL provided, save it into temp dir and return file path"
-    inputs = {
-        "url": {
-            "type": "string",
-            "description": "URL to download from",
-        }
-    }
-    output_type = "string"
-
-    def forward(self, url: str) -> str:
-        file_path = None
-
-        try:
-            result = urllib.request.urlretrieve(url)
-            file_path = result[0]
-        except Exception as e:
-            print(f"Error downloading file: {str(e)}")
-
-        return file_path

tools/img_txt_extractor.py

@@ -1,24 +0,0 @@
-from smolagents import Tool
-import easyocr
-
-class ImageTextExtractorTool(Tool):
-    name = "img_txt_extractor"
-    description = """
-        Multilingual OCR tool to extract key information or presented text from any image.
-        Supported image extensions: .png, .jpg, .jpeg, .bmp, .svg.
-    """
-    inputs = {
-        "image_path": {
-            "type": "string",
-            "description": "The path to the image file",
-        }
-    }
-    output_type = "array"
-
-    def __init__(self):
-        super().__init__()
-        self.reader = easyocr.Reader(['ch_sim', 'en'])
-
-    def forward(self, image_path: str) -> list[str]:
-        result = self.reader.readtext(image_path, detail=False)
-        return result

tools/media.py

@@ -0,0 +1,60 @@
+from smolagents import tool
+import whisper
+import requests
+from board_to_fen.predict import get_fen_from_image_path
+
+STOCKFISH_API_URL = "https://stockfish.online/api/s/v2.php"
+MODEL = whisper.load_model("tiny")
+
+@tool
+def transcribe_audio(file_path: str) -> str:
+  """
+  Extract MP3 file content and return it as text. Supported file extensions: .mp3
+  Args:
+    file_path: the path to the mp3 file.
+  """
+
+  result = None
+
+  try:
+    result = MODEL.transcribe(file_path)
+  except Exception as e:
+    return f"Error transcribing file: {str(e)}"
+
+  return result['text']
+
+#@tool
+def get_fen(file_path:str) -> str:
+  """
+  Extract Chess Board Image file content and return a string representing the board in FEN notation. Supported file extensions: .png
+  Args:
+    file_path: the path to the chess board image file.
+  """
+  fen = None
+
+  try:
+    fen = get_fen_from_image_path(file_path)
+  except Exception as e:
+    return f"Error decoding image file: {str(e)}"
+
+  return fen
+
+#@tool
+def get_best_chess_move(fen: str) -> str:
+  """
+  Return the best chess move provided the FEN notation of the board.
+  Args:
+    fen: FEN string to analyze.
+  """
+
+  data = None
+
+  try:
+    response = requests.get(STOCKFISH_API_URL, {"fen":fen, "depth":8})
+    response.raise_for_status()
+    data = response.json()
+
+  except Exception as e:
+    return f"Error fetching best move: {str(e)}"
+  
+  return data.get('bestmove').split(' ')[1].strip()

tools/pdf_reader.py

@@ -1,19 +0,0 @@
-from pdfminer.high_level import extract_text
-from smolagents import Tool
-
-class PdfReaderTool(Tool):
-    name = "pdf_reader"
-    description = "Extract PDF content. Supported file extensions: .pdf"
-    inputs = {
-        "file_path": {
-            "type": "string",
-            "description": "Path to the PDF file",
-        }
-    }
-    output_type = "string"
-
-    def forward(self, file_path) -> str:
-        try:
-            return extract_text(file_path)
-        except Exception as e:
-            return f"Error processing PDF file: {str(e)}"