add more tools

app.py

@@ -3,8 +3,8 @@ import gradio as gr
 import requests
 import inspect
 import pandas as pd
-from tools import ReverseTextTool, RunPythonFileTool, download_server, wiki_tool
-from llama_index.llms.gemini import Gemini
+from tools import ReverseTextTool, RunPythonFileTool, download_server, wiki_tool,YoutubeTranscript
+from llama_index.llms.google_genai import GoogleGenAI
 import os
 from dotenv import load_dotenv
 from llama_index.llms.huggingface_api import HuggingFaceInferenceAPI
@@ -41,8 +41,9 @@ Tool Use Guidelines:
 5. For logic or word puzzles, solve them directly unless they are reversed — in which case, decode first using **ReverseTextTool**.
 6. When dealing with Excel files, prioritize using the **excel** tool over writing code in **terminal-controller**.
 7. If you need to download a file, always use the **download_server** tool and save it to the correct path.
-8. Even for complex tasks, assume a solution exists. If one method fails, try another approach using different tools.
-9. Due to context length limits, keep browser-based tasks (e.g., searches) as short and efficient as possible.
+8. To find information about what was said in a video use **YoutubeTranscript** tool to get the transcript and find the correct answer to the asked question
+9. Even for complex tasks, assume a solution exists. If one method fails, try another approach using different tools.
+10. Due to context length limits, keep browser-based tasks (e.g., searches) as short and efficient as possible.
 """
 
 
@@ -53,12 +54,12 @@ class BasicAgent:
         gemini_api_key = os.getenv("GEMINI_API_KEY")
         if not gemini_api_key:
             raise ValueError("GEMINI_API_KEY not set in environment variables.")
-        llm = Gemini(
-            model="models/gemini-2.0-flash",
+        llm = GoogleGenAI(
+            model="gemini-2.0-flash",
             api_key=gemini_api_key,
         )
         agent = FunctionAgent(
-            tools=LoadAndSearchToolSpec.from_defaults([ReverseTextTool,RunPythonFileTool,download_server,wiki_tool]).to_tool_list(),
+            tools=[ReverseTextTool,RunPythonFileTool,download_server,wiki_tool,YoutubeTranscript],
             llm=llm,
             system_prompt=SYSTEM_PROMPT,
         )

requirements.txt

@@ -18,6 +18,10 @@ llama-index-llms-gemini
 chessimg2pos
 llama-index-llms-huggingface-api
 llama-index-embeddings-huggingface
+llama-index-llms-google-genai
+llama-hub-youtube-transcript
+llama-index-readers-youtube-transcript
+llama-index-readers-file
 
 # smol developer agent tools
 smolagents[litellm]

tools.py

@@ -6,6 +6,68 @@ from llama_index.core.tools.tool_spec.load_and_search import (
 )
 from llama_index.tools.wikipedia import WikipediaToolSpec
 #from recognizer import predict_fen
+from llama_index.readers.youtube_transcript import YoutubeTranscriptReader
+from llama_index.readers.file import (
+    DocxReader,
+    HWPReader,
+    PDFReader,
+    EpubReader,
+    FlatReader,
+    HTMLTagReader,
+    ImageCaptionReader,
+    ImageReader,
+    ImageVisionLLMReader,
+    IPYNBReader,
+    MarkdownReader,
+    MboxReader,
+    PptxReader,
+    PandasCSVReader,
+    VideoAudioReader,
+    UnstructuredReader,
+    PyMuPDFReader,
+    ImageTabularChartReader,
+    XMLReader,
+    PagedCSVReader,
+    CSVReader,
+    RTFReader,
+)
+
+def ImageReaderTool(text:str) -> str:
+    """Image reader takes image files and parse it for llm to use"""
+    parser = ImageReader()
+    file_extractor = {
+        ".jpg": parser,
+        ".jpeg": parser,
+        ".png": parser,
+    }  # Add other image formats as needed
+    documents = SimpleDirectoryReader(
+        "./data", file_extractor=file_extractor
+    ).load_data()
+    return documents
+
+def CSVReaderTool(path: str) -> str:
+    """CSV reader takes csv file and parse it for llm to use"""
+    parser = CSVReader()
+    file_extractor = {".csv": parser}  # Add other CSV formats as needed
+    documents = SimpleDirectoryReader(
+        "./data", file_extractor=file_extractor
+    ).load_data()
+    return documents
+
+def YoutubeTranscript(link: str) -> str:
+    """
+    Get video transcript from youtube link
+    Args:
+        link (str): youtube link
+    Returns:
+        str: transcript of video to llm
+    """
+
+    loader = YoutubeTranscriptReader()
+    documents = loader.load_data(
+        ytlinks=[link]
+    )
+    return documents