Add new tools and functionalities for audio transcription, code execution, document handling, image processing, and mathematical operations

- Updated requirements.txt to include new dependencies for langchain and various tools.
- Created system_prompt.txt to define assistant behavior and response format.
- Implemented audiotools for audio transcription using OpenAI Whisper.
- Developed codetools for executing code in multiple programming languages with safety measures.
- Added documenttools for file handling, including reading, writing, and downloading files.
- Introduced imagetools for image analysis, transformation, and drawing functionalities.
- Created mathtools for basic arithmetic operations.
- Implemented searchtools for querying Wikipedia, Arxiv, and YouTube transcripts.

.env.example

@@ -0,0 +1,11 @@
+# LangChain and Agent API Keys - Copy this file to .env and fill in your keys
+
+# Google API Key (if using Google-based models like Gemini directly)
+GOOGLE_API_KEY="YOUR_GOOGLE_API_KEY"
+
+# Tavily API Key (for Tavily search tool)
+TAVILY_API_KEY="YOUR_TAVILY_API_KEY"
+
+# OpenAI API Key (often used for various models or services, e.g., embeddings, other LLMs)
+# OPENAI_API_KEY="YOUR_OPENAI_API_KEY"
+

.gitignore

@@ -0,0 +1,172 @@
+### Example user template template
+### Example user template
+
+# IntelliJ project files
+.idea
+*.iml
+out
+gen
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+

README.md

@@ -12,4 +12,4 @@ hf_oauth: true
 hf_oauth_expiration_minutes: 480
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at <https://huggingface.co/docs/hub/spaces-config-reference>

agents/agent.py

@@ -0,0 +1,89 @@
+from dotenv import load_dotenv
+from langgraph.graph import START, StateGraph, MessagesState
+from langgraph.prebuilt import tools_condition
+from langgraph.prebuilt import ToolNode
+from langchain_core.messages import SystemMessage
+from tools.searchtools import wiki_search, web_search, arxiv_search, get_youtube_transcript
+from tools.mathtools import multiply, add, subtract, divide, modulus, power, square_root
+from tools.codetools import execute_code_multilang
+from tools.documenttools import create_file_with_content, read_file_content, download_file_from_url, extract_text_from_image, analyze_csv_file, analyze_excel_file
+from tools.imagetools import analyze_image, transform_image, draw_on_image, generate_simple_image, combine_images
+from tools.audiotools import transcribe_audio
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
+
+load_dotenv()
+
+# load the system prompt from the file
+with open("system_prompt.txt", "r", encoding="utf-8") as f:
+    system_prompt = f.read()
+
+# System message
+sys_msg = SystemMessage(content=system_prompt)
+
+
+tools = [
+    web_search,
+    wiki_search,
+    arxiv_search,
+    get_youtube_transcript,
+    multiply,
+    add,
+    subtract,
+    divide,
+    modulus,
+    power,
+    square_root,
+    create_file_with_content,
+    read_file_content,
+    download_file_from_url,
+    extract_text_from_image,
+    analyze_csv_file,
+    analyze_excel_file,
+    execute_code_multilang,
+    analyze_image,
+    transform_image,
+    draw_on_image,
+    generate_simple_image,
+    combine_images,
+    transcribe_audio,
+]
+
+
+# Build graph function
+def build_graph():
+    """Build the graph"""
+    # Load environment variables from .env file
+    llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0)
+
+    # Bind tools to LLM
+    llm_with_tools = llm.bind_tools(tools)
+
+    # Node
+    def assistant(state: MessagesState):
+        """Assistant node"""
+        # Prepend system message to the current messages
+        # Ensure sys_msg is only added if not already present or if it's the first turn
+        current_messages = state["messages"]
+        if not current_messages or current_messages[0].type != "system":
+            # Or, if you want to ensure it's always the first message for each LLM call in this node:
+            # updated_messages = [sys_msg] + [m for m in current_messages if m.type != "system"]
+            # For simplicity, let's assume we add it if it's not the very first message overall.
+            # A more robust check might be needed depending on multi-turn conversation flow.
+            updated_messages = [sys_msg] + current_messages
+        else:
+            updated_messages = current_messages
+        return {"messages": [llm_with_tools.invoke(updated_messages)]}
+
+    builder = StateGraph(MessagesState)
+    builder.add_node("assistant", assistant)
+    builder.add_node("tools", ToolNode(tools))
+    builder.add_edge(START, "assistant")
+    builder.add_conditional_edges(
+        "assistant",
+        tools_condition,
+    )
+    builder.add_edge("tools", "assistant")
+
+    # Compile graph
+    return builder.compile()

api_integration.py

@@ -0,0 +1,38 @@
+import requests
+from typing import List, Dict, Any
+
+class GAIAApiClient:
+    def __init__(self, api_url="https://agents-course-unit4-scoring.hf.space"):
+        self.api_url = api_url
+        self.questions_url = f"{api_url}/questions"
+        self.submit_url = f"{api_url}/submit"
+        self.files_url = f"{api_url}/files"
+        
+    def get_questions(self) -> List[Dict[str, Any]]:
+        """Fetch all evaluation questions"""
+        response = requests.get(self.questions_url)
+        response.raise_for_status()
+        return response.json()
+    
+    def get_random_question(self) -> Dict[str, Any]:
+        """Fetch a single random question"""
+        response = requests.get(f"{self.api_url}/random-question")
+        response.raise_for_status()
+        return response.json()
+    
+    def get_file(self, task_id: str) -> bytes:
+        """Download a file for a specific task"""
+        response = requests.get(f"{self.files_url}/{task_id}")
+        response.raise_for_status()
+        return response.content
+    
+    def submit_answers(self, username: str, agent_code: str, answers: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Submit agent answers and get score"""
+        data = {
+            "username": username,
+            "agent_code": agent_code,
+            "answers": answers
+        }
+        response = requests.post(self.submit_url, json=data)
+        response.raise_for_status()
+        return response.json()

app.py

@@ -1,37 +1,63 @@
+""" Basic Agent Evaluation Runner"""
 import os
+import inspect
 import gradio as gr
 import requests
-import inspect
 import pandas as pd
-from smolagents import CodeAgent, DuckDuckGoSearchTool, OpenAIServerModel
+from langchain_core.messages import HumanMessage
+from agents.agent import build_graph
+from api_integration import GAIAApiClient
+import tempfile
+import mimetypes # Added for MIME type detection
+import base64 # Added for base64 encoding images
+
+
 # (Keep Constants as is)
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
-
 # --- Basic Agent Definition ---
 # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
+
+
 class BasicAgent:
-    def __init__(self, openai_key):
-        self.openai_key = openai_key
+    """A langgraph agent."""
+    def __init__(self):
         print("BasicAgent initialized.")
-        # Initialize the model
-        #model = HfApiModel()
-        model = OpenAIServerModel(model_id="gpt-4.1", api_key=self.openai_key)
-        # Initialize the search tool
-        search_tool = DuckDuckGoSearchTool()
-        # Initialize Agent
-        self.agent = CodeAgent(
-            model = model,
-            tools=[search_tool]
-        )
-    def __call__(self, question: str) -> str:
-        print(f"Agent received question (first 50 chars): {question[:50]}...")
-        fixed_answer =self.agent.run(question)
-        print(f"Agent returning fixed answer: {fixed_answer}")
-        return fixed_answer
+        self.graph = build_graph()
+
+    def __call__(self, messages: list) -> str: # Modified to accept a list of messages
+        print(f"Agent received messages: {messages}")
+        # Ensure messages are in the correct format for the graph
+        processed_messages = self.graph.invoke({"messages": messages})
+        # The final answer should be in the 'content' of the last message
+        raw_answer = processed_messages['messages'][-1].content
+        
+        # Attempt to find "FINAL ANSWER:" and extract text after it
+        final_answer_marker = "FINAL ANSWER:"
+        marker_index = raw_answer.rfind(final_answer_marker) # Use rfind to get the last occurrence
+        
+        if marker_index != -1:
+            # Extract the text after "FINAL ANSWER: "
+            extracted_answer = raw_answer[marker_index + len(final_answer_marker):].strip()
+            # If there's a newline after the extracted answer, take only the first line
+            # This handles cases where the LLM might add extra explanations after the marker on a new line
+            first_line_of_extracted_answer = extracted_answer.split('\\n')[0].strip()
+            if first_line_of_extracted_answer: # Ensure it's not empty after stripping
+                print(f"Extracted answer: {first_line_of_extracted_answer}")
+                return first_line_of_extracted_answer
+            else: # If the first line is empty, it might be that the answer is just the marker itself (unlikely but handle)
+                print(f"Warning: Extracted answer after '{final_answer_marker}' is empty. Returning raw answer part after marker if any, or full raw answer.")
+                # Fallback to extracted_answer if first_line was empty but extracted_answer was not
+                return extracted_answer if extracted_answer else raw_answer
+
+
+        # Fallback if "FINAL ANSWER:" is not found or extraction results in empty string
+        print(f"Warning: '{final_answer_marker}' not found in agent's output or extraction failed. Returning raw answer: {raw_answer}")
+        return raw_answer
 
-def run_and_submit_all(profile: gr.OAuthProfile | None, openai_key: str):
+
+def run_and_submit_all( profile: gr.OAuthProfile | None):
     """
     Fetches all questions, runs the BasicAgent on them, submits all answers,
     and displays the results.
@@ -47,12 +73,13 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, openai_key: str):
         return "Please Login to Hugging Face with the button.", None
 
     api_url = DEFAULT_API_URL
-    questions_url = f"{api_url}/questions"
     submit_url = f"{api_url}/submit"
 
+    gaia_client = GAIAApiClient(api_url=api_url)
+
     # 1. Instantiate Agent ( modify this part to create your agent)
     try:
-        agent = BasicAgent(openai_key)
+        agent = BasicAgent()
     except Exception as e:
         print(f"Error instantiating agent: {e}")
         return f"Error initializing agent: {e}", None
@@ -61,26 +88,21 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, openai_key: str):
     print(agent_code)
 
     # 2. Fetch Questions
-    print(f"Fetching questions from: {questions_url}")
+    print(f"Fetching questions using GAIAApiClient from: {api_url}")
     try:
-        response = requests.get(questions_url, timeout=15)
-        response.raise_for_status()
-        questions_data = response.json()
+        questions_data = gaia_client.get_questions()
         if not questions_data:
-             print("Fetched questions list is empty.")
-             return "Fetched questions list is empty or invalid format.", None
+            print("Fetched questions list is empty.")
+            return "Fetched questions list is empty or invalid format.", None
         print(f"Fetched {len(questions_data)} questions.")
     except requests.exceptions.RequestException as e:
-        print(f"Error fetching questions: {e}")
+        print(f"Error fetching questions via GAIAApiClient: {e}")
         return f"Error fetching questions: {e}", None
-    except requests.exceptions.JSONDecodeError as e:
-         print(f"Error decoding JSON response from questions endpoint: {e}")
-         print(f"Response text: {response.text[:500]}")
-         return f"Error decoding server response for questions: {e}", None
     except Exception as e:
-        print(f"An unexpected error occurred fetching questions: {e}")
+        print(f"An unexpected error occurred fetching questions via GAIAApiClient: {e}")
         return f"An unexpected error occurred fetching questions: {e}", None
 
+
     # 3. Run your Agent
     results_log = []
     answers_payload = []
@@ -88,16 +110,79 @@ def run_and_submit_all(profile: gr.OAuthProfile | None, openai_key: str):
     for item in questions_data:
         task_id = item.get("task_id")
         question_text = item.get("question")
+        original_file_name = item.get("file_name")
+        
+        content_parts = [{"type": "text", "text": question_text}]
+        downloaded_file_path_for_log = None # For logging purposes
+
+        if task_id and original_file_name:
+            print(f"Question {task_id} has an associated file: {original_file_name}. Attempting to download.")
+            try:
+                file_bytes = gaia_client.get_file(task_id)
+                if file_bytes:
+                    temp_dir = tempfile.gettempdir()
+                    safe_original_filename = "".join(c if c.isalnum() or c in ('.', '_', '-') else '_' for c in original_file_name)
+                    temp_file_name = f"task_{task_id}_{safe_original_filename}"
+                    downloaded_file_path = os.path.join(temp_dir, temp_file_name)
+                    downloaded_file_path_for_log = downloaded_file_path
+
+                    with open(downloaded_file_path, "wb") as f_out:
+                        f_out.write(file_bytes)
+                    print(f"File for task {task_id} downloaded to: {downloaded_file_path}")
+
+                    # Determine MIME type and construct message part
+                    mime_type, _ = mimetypes.guess_type(downloaded_file_path)
+                    if mime_type and mime_type.startswith("image/"):
+                        base64_image = base64.b64encode(file_bytes).decode('utf-8')
+                        content_parts.append({
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:{mime_type};base64,{base64_image}"
+                            }
+                        })
+                        current_question_for_log = f"{question_text}\n\n[System Note: Image file {original_file_name} ({mime_type}) was processed and included directly in the message.]"
+                    # elif mime_type and mime_type.startswith("audio/"):
+                    #     # For audio, tools might expect a path or raw bytes.
+                    #     # For now, let's add a note with the path, assuming tools can handle it.
+                    #     # This part might need adjustment based on specific audio tool capabilities.
+                    #     content_parts.append({
+                    #         "type": "text", # Or a custom type if LangGraph/tools support it
+                    #         "text": f"[System Note: An audio file '{original_file_name}' is available at: {downloaded_file_path}]"
+                    #     })
+                    #     current_question_for_log = f"{question_text}\n\n[System Note: Audio file {original_file_name} available at {downloaded_file_path}]"
+                    else: # For other file types (text, csv, py, etc.)
+                        # Add a system note with the file path. Tools will need to be able
+                        # to read the file from this path.
+                        content_parts.append({
+                            "type": "text",
+                            "text": f"[System Note: An associated file '{original_file_name}' ({mime_type if mime_type else 'unknown type'}) has been downloaded. It is available at: {downloaded_file_path}]"
+                        })
+                        current_question_for_log = f"{question_text}\n\n[System Note: File {original_file_name} ({mime_type if mime_type else 'unknown type'}) available at {downloaded_file_path}]"
+
+                else:
+                    print(f"Warning: File indicated for task {task_id} ('{original_file_name}'), but download returned no content.")
+                    content_parts.append({"type": "text", "text": f"[System Note: A file ('{original_file_name}') was indicated for this question, but the download attempt returned no content.]"})
+                    current_question_for_log = f"{question_text}\n\n[System Note: File {original_file_name} download returned no content.]"
+            except Exception as e_file:
+                print(f"Error downloading or processing file '{original_file_name}' for task {task_id}: {e_file}")
+                content_parts.append({"type": "text", "text": f"[System Note: An error occurred while trying to download/process the associated file ('{original_file_name}') for this question: {e_file}]"})
+                current_question_for_log = f"{question_text}\n\n[System Note: Error with file {original_file_name}: {e_file}]"
+        else:
+            current_question_for_log = question_text # No file associated
+
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
+        
         try:
-            submitted_answer = agent(question_text)
+            # The agent now expects a list of content parts
+            human_message = HumanMessage(content=content_parts)
+            submitted_answer = agent([human_message]) # Pass as a list of messages
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
-            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
+            results_log.append({"Task ID": task_id, "Question": current_question_for_log, "File Path": downloaded_file_path_for_log if downloaded_file_path_for_log else "N/A", "Submitted Answer": submitted_answer})
         except Exception as e:
              print(f"Error running agent on task {task_id}: {e}")
-             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
+             results_log.append({"Task ID": task_id, "Question": current_question_for_log, "File Path": downloaded_file_path_for_log if downloaded_file_path_for_log else "N/A", "Submitted Answer": f"AGENT ERROR: {e}"})
 
     if not answers_payload:
         print("Agent did not produce any answers to submit.")
@@ -160,27 +245,24 @@ with gr.Blocks() as demo:
         **Instructions:**
         1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
         2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
-        3.  Enter your OpenAI key below (if required by your agent).
-        4.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
+        3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
         ---
         **Disclaimers:**
-        Once clicking on the "submit" button, it can take quite some time (this is the time for the agent to go through all the questions).
-        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance, for the delay process of the submit button, a solution could be to cache the answers and submit in a separate action or even to answer the questions in async.
+        Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
+        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
         """
     )
 
     gr.LoginButton()
 
-    openai_key_box = gr.Textbox(label="OpenAI API Key", type="password", placeholder="sk-...", lines=1)
-
     run_button = gr.Button("Run Evaluation & Submit All Answers")
 
     status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
+    # Removed max_rows=10 from DataFrame constructor
     results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
 
     run_button.click(
         fn=run_and_submit_all,
-        inputs=[openai_key_box],
         outputs=[status_output, results_table]
     )
 

requirements.txt

@@ -1,4 +1,24 @@
 gradio
+gradio[oauth]
 requests
-smolagents
-smolagents[openai]
+langchain
+langchain-community
+langchain-core
+langchain-google-genai
+langchain-huggingface
+langchain-groq
+langchain-tavily
+langchain-chroma
+langgraph
+huggingface_hub
+arxiv
+pymupdf
+wikipedia
+pgvector
+python-dotenv
+pytesseract
+matplotlib
+openai-whisper
+openpyxl
+youtube-transcript-api
+pytube

system_prompt.txt

@@ -0,0 +1,5 @@
+You are a helpful assistant tasked with answering questions using a set of tools. 
+Now, I will ask you a question. Report your thoughts, and finish your answer with the following template: 
+FINAL ANSWER: [YOUR FINAL ANSWER]. 
+YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, Apply the rules above for each element (number or string), ensure there is exactly one space after each comma.
+Your answer should only start with "FINAL ANSWER: ", then follows with the answer.

tools/audiotools.py

@@ -0,0 +1,33 @@
+from langchain_core.tools import tool
+import os
+import whisper
+
+@tool
+def transcribe_audio(file_path: str) -> str:
+    """
+    Transcribes an audio file using OpenAI Whisper and returns the transcribed text.
+
+    Args:
+        file_path (str): Path to the audio file.
+    """
+    if not os.path.exists(file_path):
+        return f"Error: Audio file {file_path} not found."
+
+    try:
+        # Attempt transcription with Whisper
+        # Using "base" model for a balance of speed and accuracy.
+        # Other models: "tiny", "small", "medium", "large", "large-v2", "large-v3"
+        # Consider making the model choice configurable if needed.
+        model = whisper.load_model("base") 
+        result = model.transcribe(file_path, fp16=False) # fp16=False can improve compatibility/stability on some systems
+        transcription = result["text"]
+
+        if transcription.strip(): # Check if transcription is not empty or just whitespace
+            return f"Audio transcription: {transcription}"
+        else:
+            return "Audio transcribed, but no text was detected."
+            
+    except Exception as e_whisper:
+        # Catching a general exception, but more specific ones can be added
+        # (e.g., for model loading errors, unsupported file formats by Whisper)
+        return f"Error during audio transcription: {str(e_whisper)}"

tools/codetools.py

@@ -0,0 +1,336 @@
+from langchain_core.tools import tool
+import os
+import io
+import sys
+import uuid
+import base64
+import traceback
+import contextlib
+import tempfile
+import subprocess
+import sqlite3
+from typing import Dict, List, Any, Optional, Union
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from PIL import Image
+
+class CodeInterpreter:
+    def __init__(self, allowed_modules=None, max_execution_time=30, working_directory=None):
+        """Initialize the code interpreter with safety measures."""
+        self.allowed_modules = allowed_modules or [
+            "numpy", "pandas", "matplotlib", "scipy", "sklearn", 
+            "math", "random", "statistics", "datetime", "collections",
+            "itertools", "functools", "operator", "re", "json",
+            "sympy", "networkx", "nltk", "PIL", "pytesseract", 
+            "cmath", "uuid", "tempfile", "requests", "urllib", "os", "io", "sys", "base64", "traceback", "contextlib", "sqlite3"
+        ]
+        self.max_execution_time = max_execution_time
+        self.working_directory = working_directory or os.path.join(os.getcwd()) 
+        if not os.path.exists(self.working_directory):
+            os.makedirs(self.working_directory)
+        
+        self.globals = {
+            "__builtins__": __builtins__,
+            "np": np,
+            "pd": pd,
+            "plt": plt,
+            "Image": Image,
+        }
+        self.temp_sqlite_db = os.path.join(tempfile.gettempdir(), "code_exec.db")
+
+    def execute_code(self, code: str, language: str = "python", file_path: Optional[str] = None) -> Dict[str, Any]:
+        """Execute the provided code or code from a file in the selected programming language."""
+        language = language.lower()
+        execution_id = str(uuid.uuid4())
+        
+        result = {
+            "execution_id": execution_id,
+            "status": "error",
+            "stdout": "",
+            "stderr": "",
+            "result": None,
+            "plots": [],
+            "dataframes": []
+        }
+
+        current_code = code
+        if file_path:
+            if not os.path.exists(file_path):
+                result["stderr"] = f"Error: File not found at {file_path}"
+                return result
+            if not os.path.isfile(file_path):
+                result["stderr"] = f"Error: Path {file_path} is not a file."
+                return result
+            try:
+                with open(file_path, "r", encoding='utf-8') as f:
+                    current_code = f.read()
+                if not current_code.strip() and code.strip(): # If file is empty but code arg has content
+                    # This case might be ambiguous. Prioritize file content if path is given.
+                    # If file is truly empty, and code arg was also meant to be empty, it will proceed.
+                    # If code arg had content and file was empty, it implies user might want to run content of code arg.
+                    # For now, if file_path is provided, its content (even if empty) takes precedence.
+                    # If the intention is to run `code` when `file_path` is empty, the caller should not provide `file_path`.
+                    pass # current_code is already empty string from file
+                elif not current_code.strip() and not code.strip():
+                    result["stderr"] = "Error: Both provided code string and file content are empty."
+                    return result
+
+            except Exception as e:
+                result["stderr"] = f"Error reading file {file_path}: {str(e)}"
+                return result
+        elif not code.strip(): # No file_path and code string is empty
+            result["stderr"] = "Error: No code provided either as a string or a file path."
+            return result
+        
+        try:
+            if language == "python":
+                return self._execute_python(current_code, execution_id)
+            elif language == "bash":
+                return self._execute_bash(current_code, execution_id)
+            elif language == "sql":
+                return self._execute_sql(current_code, execution_id)
+            elif language == "c":
+                return self._execute_c(current_code, execution_id)
+            elif language == "java":
+                return self._execute_java(current_code, execution_id)
+            else:
+                result["stderr"] = f"Unsupported language: {language}"
+        except Exception as e:
+            result["stderr"] = str(e)
+        
+        return result
+
+    def _execute_python(self, code: str, execution_id: str) -> dict:
+        output_buffer = io.StringIO()
+        error_buffer = io.StringIO()
+        result = {
+            "execution_id": execution_id,
+            "status": "error",
+            "stdout": "",
+            "stderr": "",
+            "result": None,
+            "plots": [],
+            "dataframes": []
+        }
+        
+        try:
+            exec_dir = os.path.join(self.working_directory, execution_id)
+            os.makedirs(exec_dir, exist_ok=True)
+            plt.switch_backend('Agg')
+            
+            with contextlib.redirect_stdout(output_buffer), contextlib.redirect_stderr(error_buffer):
+                exec_result = exec(code, self.globals)
+
+                if plt.get_fignums():
+                    for i, fig_num in enumerate(plt.get_fignums()):
+                        fig = plt.figure(fig_num)
+                        img_path = os.path.join(exec_dir, f"plot_{i}.png")
+                        fig.savefig(img_path)
+                        with open(img_path, "rb") as img_file:
+                            img_data = base64.b64encode(img_file.read()).decode('utf-8')
+                            result["plots"].append({
+                                "figure_number": fig_num,
+                                "data": img_data
+                            })
+
+                for var_name, var_value in self.globals.items():
+                    if isinstance(var_value, pd.DataFrame) and len(var_value) > 0:
+                        result["dataframes"].append({
+                            "name": var_name,
+                            "head": var_value.head().to_dict(),
+                            "shape": var_value.shape,
+                            "dtypes": str(var_value.dtypes)
+                        })
+                
+            result["status"] = "success"
+            result["stdout"] = output_buffer.getvalue()
+            result["result"] = exec_result
+            
+        except Exception as e:
+            result["status"] = "error"
+            result["stderr"] = f"{error_buffer.getvalue()}\n{traceback.format_exc()}"
+        
+        return result
+
+    def _execute_bash(self, code: str, execution_id: str) -> dict:
+        try:
+            completed = subprocess.run(
+                code, shell=True, capture_output=True, text=True, timeout=self.max_execution_time
+            )
+            return {
+                "execution_id": execution_id,
+                "status": "success" if completed.returncode == 0 else "error",
+                "stdout": completed.stdout,
+                "stderr": completed.stderr,
+                "result": None,
+                "plots": [],
+                "dataframes": []
+            }
+        except subprocess.TimeoutExpired:
+            return {
+                "execution_id": execution_id,
+                "status": "error",
+                "stdout": "",
+                "stderr": "Execution timed out.",
+                "result": None,
+                "plots": [],
+                "dataframes": []
+            }
+
+    def _execute_sql(self, code: str, execution_id: str) -> dict:
+        result = {
+            "execution_id": execution_id,
+            "status": "error",
+            "stdout": "",
+            "stderr": "",
+            "result": None,
+            "plots": [],
+            "dataframes": []
+        }
+        try:
+            conn = sqlite3.connect(self.temp_sqlite_db)
+            cur = conn.cursor()
+            cur.execute(code)
+            if code.strip().lower().startswith("select"):
+                columns = [description[0] for description in cur.description]
+                rows = cur.fetchall()
+                df = pd.DataFrame(rows, columns=columns)
+                result["dataframes"].append({
+                    "name": "query_result",
+                    "head": df.head().to_dict(),
+                    "shape": df.shape,
+                    "dtypes": str(df.dtypes)
+                })
+            else:
+                conn.commit()
+
+            result["status"] = "success"
+            result["stdout"] = "Query executed successfully."
+
+        except Exception as e:
+            result["stderr"] = str(e)
+        finally:
+            conn.close()
+
+        return result
+
+    def _execute_c(self, code: str, execution_id: str) -> dict:
+        temp_dir = tempfile.mkdtemp()
+        source_path = os.path.join(temp_dir, "program.c")
+        binary_path = os.path.join(temp_dir, "program")
+
+        try:
+            with open(source_path, "w") as f:
+                f.write(code)
+
+            compile_proc = subprocess.run(
+                ["gcc", source_path, "-o", binary_path],
+                capture_output=True, text=True, timeout=self.max_execution_time
+            )
+            if compile_proc.returncode != 0:
+                return {
+                    "execution_id": execution_id,
+                    "status": "error",
+                    "stdout": compile_proc.stdout,
+                    "stderr": compile_proc.stderr,
+                    "result": None,
+                    "plots": [],
+                    "dataframes": []
+                }
+
+            run_proc = subprocess.run(
+                [binary_path],
+                capture_output=True, text=True, timeout=self.max_execution_time
+            )
+            return {
+                "execution_id": execution_id,
+                "status": "success" if run_proc.returncode == 0 else "error",
+                "stdout": run_proc.stdout,
+                "stderr": run_proc.stderr,
+                "result": None,
+                "plots": [],
+                "dataframes": []
+            }
+        except Exception as e:
+            return {
+                "execution_id": execution_id,
+                "status": "error",
+                "stdout": "",
+                "stderr": str(e),
+                "result": None,
+                "plots": [],
+                "dataframes": []
+            }
+
+    def _execute_java(self, code: str, execution_id: str) -> dict:
+        temp_dir = tempfile.mkdtemp()
+        source_path = os.path.join(temp_dir, "Main.java")
+
+        try:
+            with open(source_path, "w") as f:
+                f.write(code)
+
+            compile_proc = subprocess.run(
+                ["javac", source_path],
+                capture_output=True, text=True, timeout=self.max_execution_time
+            )
+            if compile_proc.returncode != 0:
+                return {
+                    "execution_id": execution_id,
+                    "status": "error",
+                    "stdout": compile_proc.stdout,
+                    "stderr": compile_proc.stderr,
+                    "result": None,
+                    "plots": [],
+                    "dataframes": []
+                }
+
+            run_proc = subprocess.run(
+                ["java", "-cp", temp_dir, "Main"],
+                capture_output=True, text=True, timeout=self.max_execution_time
+            )
+            return {
+                "execution_id": execution_id,
+                "status": "success" if run_proc.returncode == 0 else "error",
+                "stdout": run_proc.stdout,
+                "stderr": run_proc.stderr,
+                "result": None,
+                "plots": [],
+                "dataframes": []
+            }
+        except Exception as e:
+            return {
+                "execution_id": execution_id,
+                "status": "error",
+                "stdout": "",
+                "stderr": str(e),
+                "result": None,
+                "plots": [],
+                "dataframes": []
+            }
+
+
+interpreter_instance = CodeInterpreter()
+
+@tool
+def execute_code_multilang(code: str, language: str = "python", file_path: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Executes code in various languages (Python, Bash, SQL, C, Java) using a sandboxed interpreter.
+    Can execute code provided as a string or from a specified file path.
+    If file_path is provided, the content of the file will be executed.
+    If both code string and file_path are provided, the content of the file at file_path takes precedence.
+
+    Args:
+        code (str): The code string to execute. Ignored if file_path is provided and valid.
+        language (str, optional): The programming language. Defaults to "python". 
+                                  Supported: "python", "bash", "sql", "c", "java".
+        file_path (Optional[str], optional): Absolute path to a file containing the code to execute. 
+                                            If provided, its content overrides the 'code' argument.
+
+    Returns:
+        Dict[str, Any]: A dictionary containing execution results, including status, stdout, stderr, 
+                        plots (for Python), and dataframes (for Python and SQL).
+    """
+    interpreter = CodeInterpreter()
+    return interpreter.execute_code(code=code, language=language, file_path=file_path)

tools/documenttools.py

@@ -0,0 +1,233 @@
+from langchain_core.tools import tool
+from typing import List, Dict, Any, Optional
+import tempfile
+from urllib.parse import urlparse
+import os
+import uuid
+import requests
+from PIL import Image
+import pytesseract
+import pandas as pd
+
+@tool
+def create_file_with_content(content: str, filename: Optional[str] = None) -> str:
+    """
+    Save content to a new file in a temporary directory and return the absolute file path.
+    Args:
+        content (str): The content to save to the file.
+        filename (str, optional): The desired name of the file. If not provided, a random unique name will be generated.
+    """
+    temp_dir = tempfile.gettempdir()
+    if filename is None:
+        # Generate a unique filename to avoid collisions if no name is provided
+        filename = f"file_{uuid.uuid4().hex[:8]}.txt" # Default to .txt if no extension in name
+    
+    filepath = os.path.join(temp_dir, filename)
+
+    try:
+        with open(filepath, "w", encoding='utf-8') as f:
+            f.write(content)
+        return filepath
+    except Exception as e:
+        return f"Error creating file {filepath}: {str(e)}"
+
+
+@tool
+def read_file_content(file_path: str) -> str:
+    """
+    Read the content of a specified file and return it as a string.
+    Args:
+        file_path (str): The absolute path to the file to be read.
+    """
+    if not os.path.exists(file_path):
+        return f"Error: File not found at {file_path}"
+    if not os.path.isfile(file_path):
+        return f"Error: Path {file_path} is not a file."
+    
+    try:
+        with open(file_path, "r", encoding='utf-8') as f:
+            content = f.read()
+        return content
+    except Exception as e:
+        return f"Error reading file {file_path}: {str(e)}"
+
+
+@tool
+def download_file_from_url(url: str, filename: Optional[str] = None) -> str:
+    """
+    Download a file from a URL and save it to a temporary location.
+    Args:
+        url (str): the URL of the file to download.
+        filename (str, optional): the name of the file. If not provided, a random name file will be created.
+    """
+    try:
+        print(f"Attempting to download file from {url}")
+        
+        # Parse URL to get filename if not provided
+        if not filename:
+            path = urlparse(url).path
+            filename = os.path.basename(path)
+            if not filename:
+                filename = f"downloaded_{uuid.uuid4().hex[:8]}"
+        
+        print(f"Will save as {filename}")
+
+        # Create temporary file
+        temp_dir = tempfile.gettempdir()
+        filepath = os.path.join(temp_dir, filename)
+
+        # Download the file with timeout and proper headers
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+        }
+        
+        response = requests.get(url, stream=True, headers=headers, timeout=30)
+        status_code = response.status_code
+        print(f"Download request status code: {status_code}")
+        
+        response.raise_for_status()
+
+        # Get content type for debugging
+        content_type = response.headers.get('Content-Type', 'unknown')
+        content_length = response.headers.get('Content-Length', 'unknown')
+        print(f"Content type: {content_type}, Content length: {content_length}")
+        
+        # Save the file
+        with open(filepath, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:  # filter out keep-alive new chunks
+                    f.write(chunk)
+
+        # Verify file was downloaded successfully
+        if os.path.exists(filepath) and os.path.getsize(filepath) > 0:
+            print(f"File successfully downloaded to {filepath} ({os.path.getsize(filepath)} bytes)")
+            return filepath
+        else:
+            print(f"File download may have failed. File size: {os.path.getsize(filepath) if os.path.exists(filepath) else 'file does not exist'}")
+            return ""
+
+    except requests.exceptions.Timeout:
+        print(f"Timeout error downloading file from {url}")
+        return ""
+    except requests.exceptions.HTTPError as e:
+        print(f"HTTP error downloading file: {e}")
+        return ""
+    except requests.exceptions.RequestException as e:
+        print(f"Request error downloading file: {e}")
+        return ""
+    except Exception as e:
+        print(f"Unexpected error downloading file: {str(e)}")
+        return ""
+
+
+@tool
+def extract_text_from_image(image_path: str) -> str:
+    """
+    Extract text from an image using OCR library pytesseract (if available).
+    Args:
+        image_path (str): the path to the image file.
+    """
+    try:
+        # Open the image
+        image = Image.open(image_path)
+
+        # Extract text from the image
+        text = pytesseract.image_to_string(image)
+
+        return f"Extracted text from image:\n\n{text}"
+    except Exception as e:
+        return f"Error extracting text from image: {str(e)}"
+
+
+@tool
+def analyze_csv_file(file_path: str, query: str) -> str:
+    """
+    Reads a CSV file using pandas and returns a summary of its structure and content.
+    The summary includes column names, data types, the first 5 rows, and descriptive statistics.
+    Use this information to understand the data.
+    For specific calculations or data manipulations based on the 'query' (e.g., summing columns, filtering rows, complex aggregations),
+    you should use the 'execute_code_multilang' tool with Python pandas code that operates on the file_path.
+    The 'query' argument here is for context and will be included in the summary.
+    Args:
+        file_path (str): The absolute path to the CSV file.
+        query (str): The user's question about the data; use this to plan subsequent steps.
+    """
+    try:
+        # Read the CSV file
+        df = pd.read_csv(file_path)
+
+        result = f"CSV File Analysis for: {os.path.basename(file_path)}\n"
+        result += f"Query: {query}\n\n"
+        result += f"File loaded with {len(df)} rows and {len(df.columns)} columns.\n"
+        result += f"Columns: {', '.join(df.columns)}\n\n"
+        
+        result += "First 5 rows:\n"
+        result += df.head().to_string() + "\n\n"
+
+        result += "Data types:\n"
+        result += df.dtypes.to_string() + "\n\n"
+
+        result += "Summary statistics (for numerical columns):\n"
+        result += df.describe(include='number').to_string() + "\n\n"
+        
+        result += "Summary statistics (for object/categorical columns):\n"
+        result += df.describe(include='object').to_string() + "\n"
+
+        return result
+
+    except Exception as e:
+        return f"Error analyzing CSV file {file_path}: {str(e)}"
+
+
+@tool
+def analyze_excel_file(file_path: str, query: str) -> str:
+    """
+    Reads an Excel file using pandas and returns a summary of its structure and content.
+    The summary includes sheet names, column names, data types, the first 5 rows (of the first sheet), and descriptive statistics.
+    It defaults to analyzing the first sheet.
+    Use this information to understand the data.
+    For specific calculations or data manipulations based on the 'query' (e.g., summing columns, filtering rows, complex aggregations),
+    you should use the 'execute_code_multilang' tool with Python pandas code that operates on the file_path (and specifies a sheet if not the first).
+    The 'query' argument here is for context and will be included in the summary.
+    Args:
+        file_path (str): The absolute path to the Excel file.
+        query (str): The user's question about the data; use this to plan subsequent steps.
+    """
+    try:
+        # Read the Excel file
+        # To handle multiple sheets, pandas reads the first sheet by default.
+        # For more specific sheet analysis, the tool would need a sheet_name parameter.
+        xls = pd.ExcelFile(file_path)
+        sheet_names = xls.sheet_names
+        
+        result = f"Excel File Analysis for: {os.path.basename(file_path)}\n"
+        result += f"Query: {query}\n"
+        result += f"Available sheets: {', '.join(sheet_names)}\n\n"
+
+        if not sheet_names:
+            return f"Error: No sheets found in Excel file {file_path}"
+
+        # Analyze the first sheet by default
+        sheet_to_analyze = sheet_names[0]
+        df = pd.read_excel(file_path, sheet_name=sheet_to_analyze)
+        
+        result += f"Analyzing sheet: '{sheet_to_analyze}'\n"
+        result += f"Sheet loaded with {len(df)} rows and {len(df.columns)} columns.\n"
+        result += f"Columns: {', '.join(df.columns)}\n\n"
+
+        result += "First 5 rows:\n"
+        result += df.head().to_string() + "\n\n"
+
+        result += "Data types:\n"
+        result += df.dtypes.to_string() + "\n\n"
+
+        result += "Summary statistics (for numerical columns):\n"
+        result += df.describe(include='number').to_string() + "\n\n"
+        
+        result += "Summary statistics (for object/categorical columns):\n"
+        result += df.describe(include='object').to_string() + "\n"
+
+        return result
+
+    except Exception as e:
+        return f"Error analyzing Excel file {file_path}: {str(e)}"

tools/imagetools.py

@@ -0,0 +1,371 @@
+from langchain_core.tools import tool
+import os
+import io
+import base64
+import uuid
+from PIL import Image
+from typing import List, Dict, Any, Optional
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont, ImageEnhance, ImageFilter
+
+# Helper functions for image processing
+def encode_image(image_path: str) -> str:
+    """Convert an image file to base64 string."""
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
+
+
+def decode_image(base64_string: str) -> Image.Image:
+    """Convert a base64 string to a PIL Image."""
+    image_data = base64.b64decode(base64_string)
+    return Image.open(io.BytesIO(image_data))
+
+
+def save_image(image: Image.Image, directory: str = "image_outputs") -> str:
+    """Save a PIL Image to disk and return the path."""
+    os.makedirs(directory, exist_ok=True)
+    image_id = str(uuid.uuid4())
+    image_path = os.path.join(directory, f"{image_id}.png")
+    image.save(image_path)
+    return image_path
+
+
+@tool
+def analyze_image(image_input: str) -> str:
+    """
+    Analyze an image and provide a detailed description.
+    
+    Args:
+        image_input (str): Either a file path to an image or a base64 encoded image string
+    
+    Returns:
+        A string description of the image
+    """
+    try:
+        # Check if input is a file path
+        if os.path.exists(image_input):
+            print(f"Processing image from file path: {image_input}")
+            img = Image.open(image_input)
+        else:
+            # Try to decode as base64
+            try:
+                print("Input not a file path, trying base64 decoding")
+                # Add padding if necessary
+                missing_padding = len(image_input) % 4
+                if missing_padding != 0:
+                    image_input += '=' * (4 - missing_padding)
+                image_data = base64.b64decode(image_input)
+                img = Image.open(io.BytesIO(image_data))
+            except Exception as base64_error:
+                return f"Error: Could not process image. Not a valid file path or base64 string: {str(base64_error)}"
+        
+        # Get basic image properties
+        width, height = img.size
+        mode = img.mode
+        format = getattr(img, 'format', 'Unknown')
+        
+        # Basic image analysis
+        description = "Image analysis:\n"
+        description += f"- Dimensions: {width}x{height} pixels\n"
+        description += f"- Color mode: {mode}\n"
+        description += f"- Format: {format}\n"
+        
+        # More advanced analysis based on image content
+        if mode in ("RGB", "RGBA"):
+            # Sample colors from different regions
+            regions = [
+                ("top-left", (width//4, height//4)),
+                ("top-right", (width*3//4, height//4)),
+                ("center", (width//2, height//2)),
+                ("bottom-left", (width//4, height*3//4)),
+                ("bottom-right", (width*3//4, height*3//4))
+            ]
+            
+            description += "\nColor sampling:\n"
+            for region_name, (x, y) in regions:
+                pixel = img.getpixel((x, y))
+                if len(pixel) >= 3:
+                    r, g, b = pixel[:3]
+                    description += f"- {region_name}: RGB({r},{g},{b})\n"
+        
+        # Analyze overall brightness
+        try:
+            if mode in ("RGB", "RGBA", "L"):
+                # Convert to numpy array for faster processing
+                arr = np.array(img)
+                if mode == "L":
+                    brightness = arr.mean()
+                    description += f"\nOverall brightness: {brightness:.1f}/255 "
+                    if brightness < 85:
+                        description += "(quite dark)"
+                    elif brightness < 170:
+                        description += "(medium brightness)"
+                    else:
+                        description += "(quite bright)"
+                else:
+                    # For RGB/RGBA
+                    if arr.shape[2] >= 3:
+                        avg_colors = arr[:,:,:3].mean(axis=(0, 1))
+                        brightness = avg_colors.mean()
+                        description += f"\nOverall brightness: {brightness:.1f}/255 "
+                        if brightness < 85:
+                            description += "(quite dark)"
+                        elif brightness < 170:
+                            description += "(medium brightness)"
+                        else:
+                            description += "(quite bright)"
+                        
+                        # Determine dominant color
+                        r, g, b = avg_colors
+                        if max(avg_colors) == r:
+                            description += "\nDominant color channel: Red"
+                        elif max(avg_colors) == g:
+                            description += "\nDominant color channel: Green"
+                        else:
+                            description += "\nDominant color channel: Blue"
+        except Exception as analysis_error:
+            description += f"\nError during color analysis: {str(analysis_error)}"
+        
+        return description
+        
+    except Exception as e:
+        return f"Error analyzing image: {str(e)}"
+
+
+@tool
+def transform_image(
+    image_base64: str, operation: str, params: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    """
+    Apply transformations: resize, rotate, crop, flip, brightness, contrast, blur, sharpen, grayscale.
+    Args:
+        image_base64 (str): Base64 encoded input image
+        operation (str): Transformation operation
+        params (Dict[str, Any], optional): Parameters for the operation
+    Returns:
+        Dictionary with transformed image (base64)
+    """
+    try:
+        img = decode_image(image_base64)
+        params = params or {}
+
+        if operation == "resize":
+            img = img.resize(
+                (
+                    params.get("width", img.width // 2),
+                    params.get("height", img.height // 2),
+                )
+            )
+        elif operation == "rotate":
+            img = img.rotate(params.get("angle", 90), expand=True)
+        elif operation == "crop":
+            img = img.crop(
+                (
+                    params.get("left", 0),
+                    params.get("top", 0),
+                    params.get("right", img.width),
+                    params.get("bottom", img.height),
+                )
+            )
+        elif operation == "flip":
+            if params.get("direction", "horizontal") == "horizontal":
+                img = img.transpose(Image.FLIP_LEFT_RIGHT)
+            else:
+                img = img.transpose(Image.FLIP_TOP_BOTTOM)
+        elif operation == "adjust_brightness":
+            img = ImageEnhance.Brightness(img).enhance(params.get("factor", 1.5))
+        elif operation == "adjust_contrast":
+            img = ImageEnhance.Contrast(img).enhance(params.get("factor", 1.5))
+        elif operation == "blur":
+            img = img.filter(ImageFilter.GaussianBlur(params.get("radius", 2)))
+        elif operation == "sharpen":
+            img = img.filter(ImageFilter.SHARPEN)
+        elif operation == "grayscale":
+            img = img.convert("L")
+        else:
+            return {"error": f"Unknown operation: {operation}"}
+
+        result_path = save_image(img)
+        result_base64 = encode_image(result_path)
+        return {"transformed_image": result_base64}
+
+    except Exception as e:
+        return {"error": str(e)}
+
+
+@tool
+def draw_on_image(
+    image_base64: str, drawing_type: str, params: Dict[str, Any]
+) -> Dict[str, Any]:
+    """
+    Draw shapes (rectangle, circle, line) or text onto an image.
+    Args:
+        image_base64 (str): Base64 encoded input image
+        drawing_type (str): Drawing type
+        params (Dict[str, Any]): Drawing parameters
+    Returns:
+        Dictionary with result image (base64)
+    """
+    try:
+        img = decode_image(image_base64)
+        draw = ImageDraw.Draw(img)
+        color = params.get("color", "red")
+
+        if drawing_type == "rectangle":
+            draw.rectangle(
+                [params["left"], params["top"], params["right"], params["bottom"]],
+                outline=color,
+                width=params.get("width", 2),
+            )
+        elif drawing_type == "circle":
+            x, y, r = params["x"], params["y"], params["radius"]
+            draw.ellipse(
+                (x - r, y - r, x + r, y + r),
+                outline=color,
+                width=params.get("width", 2),
+            )
+        elif drawing_type == "line":
+            draw.line(
+                (
+                    params["start_x"],
+                    params["start_y"],
+                    params["end_x"],
+                    params["end_y"],
+                ),
+                fill=color,
+                width=params.get("width", 2),
+            )
+        elif drawing_type == "text":
+            font_size = params.get("font_size", 20)
+            try:
+                font = ImageFont.truetype("arial.ttf", font_size)
+            except IOError:
+                font = ImageFont.load_default()
+            draw.text(
+                (params["x"], params["y"]),
+                params.get("text", "Text"),
+                fill=color,
+                font=font,
+            )
+        else:
+            return {"error": f"Unknown drawing type: {drawing_type}"}
+
+        result_path = save_image(img)
+        result_base64 = encode_image(result_path)
+        return {"result_image": result_base64}
+
+    except Exception as e:
+        return {"error": str(e)}
+
+
+@tool
+def generate_simple_image(
+    image_type: str,
+    width: int = 500,
+    height: int = 500,
+    params: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    Generate a simple image (gradient, noise, pattern, chart).
+    Args:
+        image_type (str): Type of image
+        width (int), height (int)
+        params (Dict[str, Any], optional): Specific parameters
+    Returns:
+        Dictionary with generated image (base64)
+    """
+    try:
+        params = params or {}
+
+        if image_type == "gradient":
+            direction = params.get("direction", "horizontal")
+            start_color = params.get("start_color", (255, 0, 0))
+            end_color = params.get("end_color", (0, 0, 255))
+
+            img = Image.new("RGB", (width, height))
+            draw = ImageDraw.Draw(img)
+
+            if direction == "horizontal":
+                for x in range(width):
+                    r = int(
+                        start_color[0] + (end_color[0] - start_color[0]) * x / width
+                    )
+                    g = int(
+                        start_color[1] + (end_color[1] - start_color[1]) * x / width
+                    )
+                    b = int(
+                        start_color[2] + (end_color[2] - start_color[2]) * x / width
+                    )
+                    draw.line([(x, 0), (x, height)], fill=(r, g, b))
+            else:
+                for y in range(height):
+                    r = int(
+                        start_color[0] + (end_color[0] - start_color[0]) * y / height
+                    )
+                    g = int(
+                        start_color[1] + (end_color[1] - start_color[1]) * y / height
+                    )
+                    b = int(
+                        start_color[2] + (end_color[2] - start_color[2]) * y / height
+                    )
+                    draw.line([(0, y), (width, y)], fill=(r, g, b))
+
+        elif image_type == "noise":
+            noise_array = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8)
+            img = Image.fromarray(noise_array, "RGB")
+
+        else:
+            return {"error": f"Unsupported image_type {image_type}"}
+
+        result_path = save_image(img)
+        result_base64 = encode_image(result_path)
+        return {"generated_image": result_base64}
+
+    except Exception as e:
+        return {"error": str(e)}
+
+
+@tool
+def combine_images(
+    images_base64: List[str], operation: str, params: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    """
+    Combine multiple images (collage, stack, blend).
+    Args:
+        images_base64 (List[str]): List of base64 images
+        operation (str): Combination type
+        params (Dict[str, Any], optional)
+    Returns:
+        Dictionary with combined image (base64)
+    """
+    try:
+        images = [decode_image(b64) for b64 in images_base64]
+        params = params or {}
+
+        if operation == "stack":
+            direction = params.get("direction", "horizontal")
+            if direction == "horizontal":
+                total_width = sum(img.width for img in images)
+                max_height = max(img.height for img in images)
+                new_img = Image.new("RGB", (total_width, max_height))
+                x = 0
+                for img in images:
+                    new_img.paste(img, (x, 0))
+                    x += img.width
+            else:
+                max_width = max(img.width for img in images)
+                total_height = sum(img.height for img in images)
+                new_img = Image.new("RGB", (max_width, total_height))
+                y = 0
+                for img in images:
+                    new_img.paste(img, (0, y))
+                    y += img.height
+        else:
+            return {"error": f"Unsupported combination operation {operation}"}
+
+        result_path = save_image(new_img)
+        result_base64 = encode_image(result_path)
+        return {"combined_image": result_base64}
+
+    except Exception as e:
+        return {"error": str(e)}

tools/mathtools.py

@@ -0,0 +1,82 @@
+import cmath
+from langchain_core.tools import tool
+
+
+@tool
+def multiply(a: float, b: float) -> float:
+    """
+    Multiplies two numbers.
+    Args:
+        a (float): the first number
+        b (float): the second number
+    """
+    return a * b
+
+
+@tool
+def add(a: float, b: float) -> float:
+    """
+    Adds two numbers.
+    Args:
+        a (float): the first number
+        b (float): the second number
+    """
+    return a + b
+
+
+@tool
+def subtract(a: float, b: float) -> int:
+    """
+    Subtracts two numbers.
+    Args:
+        a (float): the first number
+        b (float): the second number
+    """
+    return a - b
+
+
+@tool
+def divide(a: float, b: float) -> float:
+    """
+    Divides two numbers.
+    Args:
+        a (float): the first float number
+        b (float): the second float number
+    """
+    if b == 0:
+        raise ValueError("Cannot divided by zero.")
+    return a / b
+
+
+@tool
+def modulus(a: int, b: int) -> int:
+    """
+    Get the modulus of two numbers.
+    Args:
+        a (int): the first number
+        b (int): the second number
+    """
+    return a % b
+
+
+@tool
+def power(a: float, b: float) -> float:
+    """
+    Get the power of two numbers.
+    Args:
+        a (float): the first number
+        b (float): the second number
+    """
+    return a**b
+
+
+@tool
+def square_root(a: float) -> float | complex:
+    """
+    Get the square root of a number.
+    Args:
+        a (float): the number to get the square root of
+    """
+    if a >= 0:
+        return a**0.5
+    return cmath.sqrt(a)

tools/searchtools.py

@@ -0,0 +1,108 @@
+from langchain_core.tools import tool
+from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_community.document_loaders import WikipediaLoader
+from langchain_community.document_loaders import ArxivLoader
+from youtube_transcript_api import YouTubeTranscriptApi, TranscriptsDisabled, NoTranscriptFound # Added
+import os
+
+@tool
+def wiki_search(query: str) -> str:
+    """Search Wikipedia for a query and return maximum 2 results.
+    Args:
+        query: The search query."""
+    search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
+    formatted_search_docs = "\n\n---\n\n".join(
+        [
+            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
+            for doc in search_docs
+        ]
+    )
+    return {"wiki_results": formatted_search_docs}
+
+
+@tool
+def web_search(query: str) -> str:
+    """Search Tavily for a query and return maximum 3 results.
+    Args:
+        query: The search query."""
+    search_docs = TavilySearchResults(max_results=3).invoke({"query": query})
+    formatted_search_docs = "\n\n---\n\n".join(
+        [
+            f'<Document source="{doc.get("url", "")}">\n{doc.get("content", doc.get("snippet", ""))}\n</Document>'
+            for doc in search_docs
+        ]
+    )
+    return {"web_results": formatted_search_docs}
+
+
+@tool
+def arxiv_search(query: str) -> str:
+    """Search Arxiv for a query and return maximum 3 result.
+    Args:
+        query: The search query."""
+    search_docs = ArxivLoader(query=query, load_max_docs=3).load()
+    formatted_search_docs = "\n\n---\n\n".join(
+        [
+            f'<Document source="{doc.metadata.get("source", "N/A")}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
+            for doc in search_docs
+        ]
+    )
+    return {"arxiv_results": formatted_search_docs}
+
+
+@tool
+def get_youtube_transcript(youtube_url: str) -> str:
+    """Fetches the transcript for a given YouTube video URL using youtube-transcript-api directly.
+       If the video has no transcript, it will return an error message. Then use web_search to find the transcript.
+    Args:
+        youtube_url: The URL of the YouTube video."""
+    try:
+        video_id = None
+        if "watch?v=" in youtube_url:
+            video_id = youtube_url.split("watch?v=")[1].split("&")[0]
+        elif "youtu.be/" in youtube_url:
+            video_id = youtube_url.split("youtu.be/")[1].split("?")[0]
+        
+        if not video_id:
+            return "Error: Could not parse YouTube video ID from URL."
+
+        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+        
+        transcript = None
+        try:
+            # Try fetching English first if available, then any manual, then any generated
+            transcript = transcript_list.find_manually_created_transcript(['en'])
+        except NoTranscriptFound:
+            try:
+                transcript = transcript_list.find_generated_transcript(['en'])
+            except NoTranscriptFound:
+                # If English not found, try any manual transcript
+                try:
+                    transcript = transcript_list.find_manually_created_transcript(transcript_list.languages)
+                except NoTranscriptFound:
+                    # Finally, try any generated transcript
+                    try:
+                        transcript = transcript_list.find_generated_transcript(transcript_list.languages)
+                    except NoTranscriptFound:
+                        return "Error: No manual or auto-generated transcripts found for this video in any language."
+
+        fetched_transcript = transcript.fetch()
+        
+        if not fetched_transcript:
+            return "Could not retrieve transcript for the video. The video might not have transcripts available."
+        
+        # Changed item['text'] to item.text to handle cases where items are objects
+        full_transcript = " ".join([item.text for item in fetched_transcript])
+        
+        # Returning the transcript text directly, wrapped in a dictionary similar to other tools
+        return {"youtube_transcript": full_transcript}
+
+    except TranscriptsDisabled:
+        return "Error: Transcripts are disabled for this video."
+    except NoTranscriptFound: 
+        return "Error: No transcripts found for this video (this should have been caught earlier, but good fallback)."
+    except Exception as e:
+        # Catching potential network errors or other API issues specifically
+        if "HTTP Error 403" in str(e) or "Too Many Requests" in str(e):
+            return f"Error: YouTube API request failed, possibly due to rate limiting or access restrictions: {str(e)}"
+        return f"Error fetching YouTube transcript using youtube-transcript-api: {str(e)}"