feat: tools complete (1st version)

requirements.txt

@@ -5,4 +5,8 @@ smolagents
 wikipedia
 openpyxl
 pydub
-mistralai
+mistralai
+opencv-python
+chess
+SpeechRecognition
+pypdf2

src/inference.py

@@ -10,6 +10,13 @@ from src.tools import (
     web_search,
     visit_webpage,
     load_file,
+    reverse_text,
+    analyze_chess,
+    analyze_document,
+    classify_foods,
+    transcribe_audio,
+    execute_code,
+    analyze_excel,
 )
 
 load_dotenv()
@@ -24,13 +31,32 @@ class Agent:
             "web_search": web_search,
             "visit_webpage": visit_webpage,
             "load_file": load_file,
+            "reverse_text": reverse_text,
+            "analyze_chess": analyze_chess,
+            "analyze_document": analyze_document,
+            "classify_foods": classify_foods,
+            "transcribe_audio": transcribe_audio,
+            "execute_code": execute_code,
+            "analyze_excel": analyze_excel,
         }
         self.conversation_log = []
 
     def get_tools(self):
         """Generate the tools.json file with the tools to be used by the agent."""
         return generate_tools_json(
-            [web_search, visit_webpage, load_file]).get('tools')
+            [
+                web_search,
+                visit_webpage,
+                load_file,
+                reverse_text,
+                analyze_chess,
+                analyze_document,
+                classify_foods,
+                transcribe_audio,
+                execute_code,
+                analyze_excel,
+            ]
+        ).get('tools')
 
     def make_initial_request(self, input):
         """Make the initial request to the agent with the given input."""
@@ -115,7 +141,7 @@ class Agent:
                 file, ensure_ascii=False, indent=4
             )
 
-    def run(self, input, task_id, truth, max_steps=20):
+    def run(self, input, task_id, truth, max_steps=100):
         """Run the agent with the given input and process the response."""
         print("\n... Asking the agent ...\n")
         response, messages = self.make_initial_request(input)
@@ -125,7 +151,7 @@ class Agent:
             steps += 1
             thought_result = self.thought(response)
 
-            final_answer_match = re.search(r'Final Answer\n(.*)', thought_result, re.DOTALL)
+            final_answer_match = re.search(r'FINAL ANSWER:(.*)', thought_result, re.DOTALL)
             if final_answer_match:
                 self.save_conversation(task_id, truth, final_answer_match.group(1).strip())
                 return final_answer_match.group(1).strip()

src/tools/__init__.py

@@ -1,3 +1,10 @@
 from .web_search import web_search
 from .visit_webpage import visit_webpage
 from .load_file import load_file
+from .reverse_text import reverse_text
+from .analyze_chess import analyze_chess
+from .analyze_document import analyze_document
+from .classify_food import classify_foods
+from .transcript_audio import transcribe_audio
+from .execute_code import execute_code
+from .analyze_excel import analyze_excel

src/tools/analyze_chess.py

@@ -0,0 +1,52 @@
+from src.utils.tooling import tool
+
+@tool
+def analyze_chess(image_path: str) -> str:
+    """
+    Analyzes a chess position from an image and determines the best next move.
+    Args:
+        image_path (str): The path to the image file containing the chess position.
+    Returns:
+        str: The recommended move in algebraic notation.
+    """
+    try:
+        import chess.engine
+        from PIL import Image
+        import cv2
+        import numpy as np
+    except ImportError as e:
+        raise ImportError(
+            "You must install packages `python-chess`, `Pillow`, and `opencv-python` to run this tool."
+            "For instance, run `pip install chess pillow opencv-python`."
+        ) from e
+
+    def preprocess_image(image_path):
+        image = Image.open(image_path)
+        image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+        return image
+
+    def detect_board_position(image):
+        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)      # For example, use contour detection to find the chessboard
+        edges = cv2.Canny(gray, 50, 150, apertureSize=3)
+        contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        board_contour = max(contours, key=cv2.contourArea)      # Assume the largest contour is the chessboard
+        return board_contour
+
+    def extract_fen_from_image(image):
+        fen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"        # Placeholder FEN string
+        return fen
+
+    def get_best_move(fen):
+
+        engine = chess.engine.SimpleEngine.popen_uci("/opt/homebrew/bin/stockfish")         # Initialize the chess engine
+        board = chess.Board(fen)                                                            # Create a board from the FEN string
+        result = engine.play(board, chess.engine.Limit(time=2.0))                           # Get the best move
+        engine.quit()
+        return result.move.uci()
+
+    image = preprocess_image(image_path)            # Preprocess the image
+    board_contour = detect_board_position(image)    # Detect the board position
+    fen = extract_fen_from_image(image)             # Extract the FEN string from the image
+    best_move = get_best_move(fen)                  # Get the best move
+
+    return best_move

src/tools/analyze_document.py

@@ -0,0 +1,72 @@
+from src.utils.tooling import tool
+import PyPDF2
+import re
+
+@tool
+def analyze_document(file_path: str, keywords: list) -> str:
+    """
+    Extracts specific information from a PDF or text document based on given keywords.
+    Args:
+        file_path (str): The path to the PDF or text document to analyze.
+        keywords (list): A list of keywords to search for in the document.
+    Returns:
+        str: The extracted information as text.
+    """
+    def extract_text_from_pdf(file_path: str) -> str:
+        """
+        Extracts text from a PDF file.
+        Args:
+            file_path (str): The path to the PDF file.
+        Returns:
+            str: The extracted text from the PDF.
+        """
+        try:
+            with open(file_path, 'rb') as file:
+                reader = PyPDF2.PdfFileReader(file)
+                text = ''
+                for page_num in range(reader.numPages):
+                    page = reader.getPage(page_num)
+                    text += page.extract_text()
+                return text
+        except Exception as e:
+            raise Exception(f"Error reading PDF file: {e}")
+
+    def extract_text_from_txt(file_path: str) -> str:
+        """
+        Extracts text from a text file.
+        Args:
+            file_path (str): The path to the text file.
+        Returns:
+            str: The extracted text from the text file.
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                return file.read()
+        except Exception as e:
+            raise Exception(f"Error reading text file: {e}")
+
+    def extract_information(text: str, keywords: list) -> str:
+        """
+        Extracts information based on keywords from the text.
+        Args:
+            text (str): The text to analyze.
+            keywords (list): A list of keywords to search for in the text.
+        Returns:
+            str: The extracted information as text.
+        """
+        extracted_info = []
+        for keyword in keywords:
+            pattern = re.compile(r'\b{}\b'.format(re.escape(keyword)), re.IGNORECASE)
+            matches = pattern.findall(text)
+            if matches:
+                extracted_info.append(f"Keyword '{keyword}': {', '.join(matches)}")
+        return "\n".join(extracted_info)
+
+    if file_path.lower().endswith('.pdf'):
+        text = extract_text_from_pdf(file_path)
+    elif file_path.lower().endswith('.txt'):
+        text = extract_text_from_txt(file_path)
+    else:
+        raise ValueError("Unsupported file format. Please provide a PDF or text file.")
+
+    return extract_information(text, keywords)

src/tools/analyze_excel.py

@@ -0,0 +1,35 @@
+from src.utils.tooling import tool
+import pandas as pd
+
+@tool
+def analyze_excel(file_path: str, sheet_name: str = None, specific_columns: list = None) -> str:
+    """
+    Analyzes data from an Excel file to extract specific information.
+    Args:
+        file_path (str): The path to the Excel file to analyze.
+        sheet_name (str, optional): The name of the sheet to read. If None, the first sheet is used.
+        specific_columns (list, optional): A list of column names to extract. If None, all columns are extracted.
+    Returns:
+        str: Extracted information in text or structured data format.
+    """
+    try:
+        if sheet_name:
+            df = pd.read_excel(file_path, sheet_name=sheet_name)
+        else:
+            df = pd.read_excel(file_path)
+
+        if specific_columns:
+            df = df[specific_columns]
+
+        analysis_result = df.describe(include='all').to_string()
+
+        return f"## Excel Analysis Results\n\n{analysis_result}"
+
+    except FileNotFoundError:
+        return "File not found. Please check the file path."
+    except pd.errors.EmptyDataError:
+        return "The Excel file is empty."
+    except pd.errors.ParserError:
+        return "Error parsing the Excel file."
+    except Exception as e:
+        return f"An error occurred: {str(e)}"

src/tools/classify_food.py

@@ -0,0 +1,56 @@
+from src.utils.tooling import tool
+
+@tool
+def classify_foods(food_list: list) -> dict:
+    """
+    Classifies a list of foods into specific botanical categories.
+    Args:
+        food_list (list): A list of foods to classify.
+    Returns:
+        dict: A dictionary with categories as keys and lists of foods as values.
+    """
+    categories = {
+        "fruits": [
+            "apple", "banana", "orange", "grape", "strawberry", "plum", "peach", "pear",
+            "cherry", "blueberry", "raspberry", "pineapple", "mango", "kiwi", "lemon",
+            "lime", "watermelon", "cantaloupe", "avocado", "tomato", "cucumber", "bell pepper",
+            "eggplant", "okra", "zucchini", "pumpkin", "olive"
+        ],
+        "vegetables": [
+            "carrot", "broccoli", "spinach", "lettuce", "celery", "fresh basil", "sweet potato",
+            "potato", "onion", "garlic", "cabbage", "kale", "cauliflower", "asparagus", "radish",
+            "turnip", "beet", "artichoke", "brussels sprouts", "green beans", "peas", "mushroom"
+        ],
+        "grains": [
+            "rice", "wheat", "oats", "barley", "quinoa", "corn", "rye", "millet", "sorghum",
+            "buckwheat", "flour"
+        ],
+        "nuts": [
+            "almond", "walnut", "cashew", "peanut", "hazelnut", "pecan", "pistachio", "macadamia",
+            "brazil nut", "chestnut", "acorn"
+        ],
+        "legumes": [
+            "lentil", "chickpea", "bean", "pea", "soybean", "black bean", "kidney bean", "pinto bean",
+            "navy bean", "lima bean", "green bean"
+        ],
+        "other": [
+            "milk", "eggs", "coffee", "Oreos", "allspice", "sugar", "salt", "honey", "maple syrup",
+            "vinegar", "oil", "butter", "cheese", "yogurt", "cream", "meat", "fish", "poultry"
+        ]
+    }
+    classified_foods = {category: [] for category in categories}
+
+    for food in food_list:
+        food_lower = food.lower()
+        classified = False
+        for category, examples in categories.items():
+            if food_lower in examples:
+                classified_foods[category].append(food)
+                classified = True
+                break
+        if not classified:
+            classified_foods.setdefault("unknown", []).append(food)
+
+    classified_foods["vegetables"].sort()
+
+    return classified_foods

src/tools/execute_code.py

@@ -0,0 +1,32 @@
+from src.utils.tooling import tool
+import subprocess
+import tempfile
+
+@tool
+def execute_code(file_path: str) -> str:
+    """
+    Executes Python code from a file and returns the final result.
+    Args:
+        file_path (str): The path to the file containing the Python code to execute.
+    Returns:
+        str: The result of the code execution.
+    """
+    try:
+        with open(file_path, 'r') as file:
+            code = file.read()
+
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".py") as temp_file:
+            temp_file.write(code.encode('utf-8'))
+            temp_file_path = temp_file.name
+
+        result = subprocess.run(['python', temp_file_path], capture_output=True, text=True)
+
+        if result.returncode != 0:
+            raise Exception(f"Error executing code: {result.stderr}")
+
+        return result.stdout
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f"The file at {file_path} does not exist.")
+    except Exception as e:
+        raise Exception(f"An error occurred: {str(e)}")

src/tools/reverse_text.py

@@ -0,0 +1,19 @@
+from src.utils.tooling import tool
+
+@tool
+def reverse_text(input_text: str) -> str:
+    """
+    Reverses an input string to make it readable.
+    Args:
+        input_text (str): The reversed text string to process.
+    """
+    try:
+        corrected_text = input_text[::-1]
+
+        if not corrected_text:
+            raise ValueError("The input text is empty! Please provide a valid reversed text string.")
+
+        return corrected_text
+
+    except Exception as e:
+        raise Exception(f"An error occurred while processing the text: {e}")

src/tools/transcript_audio.py

@@ -0,0 +1,30 @@
+from src.utils.tooling import tool
+
+@tool
+def transcribe_audio(file_path: str, language: str = 'en') -> str:
+    """
+    Transcribes the content of an audio file into text.
+    Args:
+        file_path (str): The path to the audio file to transcribe.
+        language (str, optional): The language of the audio content. Defaults to 'en' (English).
+    Returns:
+        str: The transcribed text from the audio file.
+    """
+    try:
+        import speech_recognition as sr
+    except ImportError as e:
+        raise ImportError(
+            "You must install the package `SpeechRecognition` to run this tool. For instance, run `pip install SpeechRecognition`."
+        ) from e
+
+    recognizer = sr.Recognizer()
+
+    try:
+        with sr.AudioFile(file_path) as source:
+            audio_data = recognizer.record(source)
+            text = recognizer.recognize_google(audio_data, language=language)
+            return text
+    except sr.UnknownValueError:
+        raise Exception("Speech Recognition could not understand the audio.")
+    except sr.RequestError as e:
+        raise Exception(f"Could not request results from Speech Recognition service; {e}")

src/tools/web_search.py

@@ -1,7 +1,7 @@
 from src.utils.tooling import tool
 
 @tool
-def web_search(query: str, max_results: int = 3, timeout: int = 10) -> str:
+def web_search(query: str, max_results: int = 3, timeout: int = 1) -> str:
     """
     Performs a web search based on the query and returns the top search results.
     Args:

src/utils/__init__.py

@@ -1,2 +1 @@
-from .api import fetch_questions, submit_answers, get_file
-from .prompt import load_prompt
+from .api import fetch_questions, submit_answers, get_file

src/workflow.py

@@ -16,7 +16,6 @@ from src.inference import Agent
 
 def run_and_submit_all(profile: gr.OAuthProfile | None):
     console = Console()
-    agent = Agent()
     space_id = os.getenv("SPACE_ID")
 
     if profile:
@@ -38,6 +37,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     answers_payload = []
 
     for item in questions_data:
+
         task_id = item.get("task_id")
         question_text = item.get("question")
         file_name = item.get("file_name")
@@ -60,8 +60,8 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
                     item = json.loads(line)
                     if item.get('task_id') == task_id:
                         final_answer = item.get('Final answer')
-                        console.print(Panel(f"The correct final answer is: [bold]{final_answer}[/bold]"))
 
+            agent = Agent()
             submitted_answer = agent.run(
                 input=question_text + file_context,
                 task_id=task_id,
@@ -69,18 +69,11 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             )
 
             console.print(Panel(f"[bold green]Submitted Answer[/bold green]\n{submitted_answer}", expand=False))
+            console.print(Panel(f"The correct final answer is: [bold]{final_answer}[/bold]"))
 
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
 
-            # Vérification des métadonnées
-            with open('./metadata.jsonl', 'r') as file:
-                for line in file:
-                    item = json.loads(line)
-                    if item.get('task_id') == task_id:
-                        final_answer = item.get('Final answer')
-                        console.print(Panel(f"The correct final answer is: [bold]{final_answer}[/bold]"))
-
         except Exception as e:
             console.print(f"Error: {e}", style="bold red")
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})

tools.json

@@ -63,5 +63,155 @@
                 ]
             }
         }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "reverse_text",
+            "description": "Reverses an input string to make it readable.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "input_text": {
+                        "type": "string",
+                        "description": "The reversed text string to process."
+                    }
+                },
+                "required": [
+                    "input_text"
+                ]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "analyze_chess",
+            "description": "Analyzes a chess position from an image and determines the best next move.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "image_path": {
+                        "type": "string",
+                        "description": "The path to the image file containing the chess position."
+                    }
+                },
+                "required": [
+                    "image_path"
+                ]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "analyze_document",
+            "description": "Extracts specific information from a PDF or text document based on given keywords.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "The path to the PDF or text document to analyze."
+                    },
+                    "keywords": {
+                        "type": "array",
+                        "description": "A list of keywords to search for in the document."
+                    }
+                },
+                "required": [
+                    "file_path",
+                    "keywords"
+                ]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "classify_foods",
+            "description": "Classifies a list of foods into specific botanical categories.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "food_list": {
+                        "type": "array",
+                        "description": "A list of foods to classify."
+                    }
+                },
+                "required": [
+                    "food_list"
+                ]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "transcribe_audio",
+            "description": "Transcribes the content of an audio file into text.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "The path to the audio file to transcribe."
+                    },
+                    "language": {
+                        "type": "string",
+                        "description": "The language of the audio content. Defaults to 'en' (English)."
+                    }
+                },
+                "required": [
+                    "file_path"
+                ]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "execute_code",
+            "description": "Executes Python code from a file and returns the final result.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "The path to the file containing the Python code to execute."
+                    }
+                },
+                "required": [
+                    "file_path"
+                ]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "analyze_excel",
+            "description": "Analyzes data from an Excel file to extract specific information.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "The path to the Excel file to analyze."
+                    },
+                    "sheet_name": {
+                        "type": "string",
+                        "description": "The name of the sheet to read. If None, the first sheet is used."
+                    },
+                    "specific_columns": {
+                        "type": "array",
+                        "description": "A list of column names to extract. If None, all columns are extracted."
+                    }
+                },
+                "required": [
+                    "file_path"
+                ]
+            }
+        }
     }
 ]