Add tools for file reading and audio transcription; update .gitignore and requirements

.gitignore

@@ -1 +1,2 @@
 .DS_STORE
+*.pyc

app.py

@@ -10,6 +10,7 @@ from llama_index.tools.wikipedia import WikipediaToolSpec
 from llama_index.core.tools import FunctionTool
 from llama_index.tools.duckduckgo import DuckDuckGoSearchToolSpec
 
+from tools import read_python_file_tool, audio_to_text_tool, read_xlsx_file_tool, read_png_file_tool
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -34,9 +35,8 @@ Here is the question:
 llm = HuggingFaceInferenceAPI(model_name='Qwen/Qwen3-32B', num_output=4096, temperature=0.01)
 wikipedia_tools = WikipediaToolSpec().to_tool_list()
 search_tool = FunctionTool.from_defaults(DuckDuckGoSearchToolSpec().duckduckgo_full_search)
-
 agent = AgentWorkflow.from_tools_or_functions(
-    tools_or_functions=[search_tool],
+    tools_or_functions=[search_tool, read_python_file_tool, audio_to_text_tool, read_xlsx_file_tool, read_png_file_tool] + wikipedia_tools,
     llm=llm,
     #system_prompt=system_prompt,
     verbose=False,
@@ -102,7 +102,10 @@ async def run_and_submit_all( profile: gr.OAuthProfile | None):
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            r = await agent.run(system_prompt + question_text)
+            prompt = system_prompt + question_text
+            if item.get("file_name", "") != "":
+                prompt += f'\nHere is the attached file you might need to answer the question: files/{item.get("file_name", "")}'
+            r = await agent.run(prompt)
             submitted_answer = r.response.blocks[0].text
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})

requirements.txt

@@ -8,3 +8,7 @@ llama-index-llms-huggingface-api
 llama-index-tools-duckduckgo
 llama-index-retrievers-bm25
 llama-index-tools-wikipedia
+llama-index-tools-code-interpreter
+openai-whisper
+pandas
+openpyxl

tools.py

@@ -0,0 +1,109 @@
+from llama_index.core.tools import FunctionTool
+import whisper
+import pandas as pd
+from PIL import Image
+
+def read_python_file(file_path: str) -> bytes:
+    """
+    Reads the content of a .py file given its file path.
+
+    Args:
+        file_path (str): Path to the file.
+
+    Returns:
+        bytes: Content of the file as string.
+    """
+    try:
+        with open(file_path, 'r') as file:
+            return file.read()
+    except Exception as e:
+        raise ValueError(f"Error reading file at {file_path}: {e}")
+
+read_python_file_tool = FunctionTool.from_defaults(
+    fn=read_python_file,
+    name="read_file_content",
+    description="Reads the content of a .py file given its file path."
+)
+
+
+def audio_to_text(file_path: str) -> str:
+    """
+    Converts an audio file to text using OpenAI Whisper.
+
+    Args:
+        file_path (str): Path to the audio file.
+
+    Returns:
+        str: Transcribed text from the audio file.
+    """
+    try:
+        # Load the Whisper model
+        model = whisper.load_model("base")
+
+        # Transcribe the audio file
+        result = model.transcribe(file_path)
+
+        # Return the transcribed text
+        return result['text']
+    except Exception as e:
+        raise ValueError(f"Error processing audio file at {file_path}: {e}")
+
+audio_to_text_tool = FunctionTool.from_defaults(
+    fn=audio_to_text,
+    name="audio_to_text",
+    description="Converts an audio file to text using OpenAI Whisper."
+)
+
+
+def read_xlsx_file(file_path: str) -> str:
+    """
+    Reads the content of an .xlsx file and returns it as a string.
+
+    Args:
+        file_path (str): Path to the .xlsx file.
+
+    Returns:
+        str: Content of the .xlsx file as a string.
+    """
+    try:
+        # Read the Excel file into a DataFrame
+        df = pd.read_excel(file_path)
+
+        # Convert the DataFrame to a string
+        return df.to_string(index=False)
+    except Exception as e:
+        raise ValueError(f"Error reading .xlsx file at {file_path}: {e}")
+
+read_xlsx_file_tool = FunctionTool.from_defaults(
+    fn=read_xlsx_file,
+    name="read_xlsx_file",
+    description="Reads the content of an .xlsx file and returns it as a string."
+)
+
+
+def read_png_file(file_path: str) -> list:
+    """
+    Reads the content of a .png file and returns its RGB pixel representation.
+
+    Args:
+        file_path (str): Path to the .png file.
+
+    Returns:
+        list: A 2D list representing the RGB pixel values of the image.
+    """
+    try:
+        # Open the image file
+        image = Image.open(file_path).convert("RGB")
+
+        # Convert the image to a 2D list of RGB tuples
+        rgb_pixels = list(image.getdata())
+        width, height = image.size
+        return [rgb_pixels[i * width:(i + 1) * width] for i in range(height)]
+    except Exception as e:
+        raise ValueError(f"Error reading .png file at {file_path}: {e}")
+
+read_png_file_tool = FunctionTool.from_defaults(
+    fn=read_png_file,
+    name="read_png_file",
+    description="Reads the content of a .png file and returns its RGB pixel representation."
+)