Update app.py

app.py

@@ -5,34 +5,42 @@ import pandas as pd
 import traceback
 import time 
 import mimetypes
+from tempfile import NamedTemporaryFile
 
 # Import smol-agent and tool components
 from smolagents import CodeAgent, LiteLLMModel, tool
 from smolagents import DuckDuckGoSearchTool
 from unstructured.partition.auto import partition
 
+# Imports for advanced file processing
+import speech_recognition as sr
+from moviepy.editor import VideoFileClip
+
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
-# --- Tool Definition (Updated for Multimodality) ---
+# --- Tool Definition (Upgraded for Full Multimodality) ---
 @tool
 def file_reader(file_path: str) -> str:
     """
-    Reads the content of a file and returns its text content.
-    This tool supports various file types, including text (PDF, TXT, CSV)
-    and can perform Optical Character Recognition (OCR) on images (PNG, JPG).
-    It can be used with either a local path or a web URL.
-    For non-text/image formats like audio or video, it will return a message
-    indicating the file type, as it cannot analyze their content directly.
+    Reads and analyzes the content of a file and returns relevant text-based information.
+    Supports:
+      - Text files (PDF, TXT, CSV)
+      - Images (PNG, JPG) with OCR
+      - Audio (MP3, WAV) via speech recognition
+      - Video (MP4, MOV) via speech recognition on audio track
+    Can be used with a local file path or a web URL.
 
     Args:
         file_path (str): The local path or web URL of the file to be read.
+    Returns:
+        str: Extracted or transcribed content as text.
     """
     temp_file_path = None
     try:
-        # Handle web URLs by downloading the file first
+        # Download the file if it's a URL
         if file_path.startswith("http://") or file_path.startswith("https://"):
-            temp_file_path = "temp_downloaded_file"
+            temp_file_path = NamedTemporaryFile(delete=False).name
             response = requests.get(file_path, timeout=20)
             response.raise_for_status()
             with open(temp_file_path, "wb") as f:
@@ -41,44 +49,62 @@ def file_reader(file_path: str) -> str:
         else:
             local_path = file_path
 
-        # Gracefully handle unsupported file types (e.g., audio, video)
         mime_type, _ = mimetypes.guess_type(local_path)
-        if mime_type and not (mime_type.startswith('text/') or mime_type.startswith('image/') or mime_type == 'application/pdf' or mime_type == 'application/zip'):
-            if temp_file_path and os.path.exists(temp_file_path):
-                os.remove(temp_file_path)
-            return f"File is of a non-visual, non-text format ({mime_type}). Content analysis is not supported by this tool."
+        recognizer = sr.Recognizer()
 
-        # Use 'unstructured' which has built-in OCR for images.
-        # This will extract text from images where possible.
-        elements = partition(local_path)
+        if mime_type:
+            # Handle audio files
+            if mime_type.startswith("audio/"):
+                with sr.AudioFile(local_path) as source:
+                    audio = recognizer.record(source)
+                    # Using whisper for robust speech recognition
+                    return recognizer.recognize_whisper(audio)
 
-        # Clean up the temporary file if it was created
-        if temp_file_path and os.path.exists(temp_file_path):
-            os.remove(temp_file_path)
+            # Handle video files by extracting audio
+            elif mime_type.startswith("video/"):
+                # Use a temporary file for the extracted audio
+                with NamedTemporaryFile(suffix=".wav", delete=False) as audio_temp:
+                    audio_temp_path = audio_temp.name
+                
+                clip = VideoFileClip(local_path)
+                clip.audio.write_audiofile(audio_temp_path, codec='pcm_s16le')
+                
+                with sr.AudioFile(audio_temp_path) as source:
+                    audio = recognizer.record(source)
+                
+                # Clean up the temporary audio file
+                os.remove(audio_temp_path)
+
+                # Using whisper for robust speech recognition
+                return recognizer.recognize_whisper(audio)
 
+        # Default to handling text and images with OCR if not audio/video
+        elements = partition(local_path)
         return "\n\n".join([str(el) for el in elements])
+
     except Exception as e:
-        # Ensure cleanup even if an error occurs
+        return f"Error reading or processing file '{file_path}': {e}"
+    finally:
+        # Clean up the downloaded file if it exists
         if temp_file_path and os.path.exists(temp_file_path):
             os.remove(temp_file_path)
-        return f"Error reading or processing file '{file_path}': {e}"
 
 
-# --- Agent Class (Updated with Native Memory Management) ---
+# --- Agent Class (Updated with More Powerful Model and Tools) ---
 class GaiaSmolAgent:
     def __init__(self):
         """
         Initializes the optimized agent.
-        Now uses the agent's native conversation memory capabilities.
+        Now uses a more powerful model and the agent's native conversation memory.
         """
         print("Initializing Optimized GaiaSmolAgent...")
         api_key = os.getenv("GEMINI_API_KEY")
         if not api_key:
             raise ValueError("API key 'GEMINI_API_KEY' not found in environment secrets.")
 
-        # Use a faster, more cost-effective model optimized for speed.
+        # Use a more powerful, "clever" model for better reasoning.
         model = LiteLLMModel(
-            model_id="gemini/gemini-1.5-flash-latest",
+            model_id="gemini/gemini-1.5-pro-latest",
             api_key=api_key,
             temperature=0.0,
             timeout=120.0, # Add a timeout to prevent hanging
@@ -90,7 +116,7 @@ class GaiaSmolAgent:
 
         **Available Tools:**
         - `duck_duck_go_search(query: str) -> str`: Use this to find information, file URLs, or anything on the web.
-        - `file_reader(file_path: str) -> str`: Use this to read the contents of a file from a local path or a web URL. It can read text and extract text from images (OCR).
+        - `file_reader(file_path: str) -> str`: Use this to read the contents of a file from a local path or a web URL. It can read text, extract text from images (OCR), and transcribe audio from audio/video files.
 
         **Your Thought Process:**
         1.  **Deconstruct the Goal:** Carefully analyze the question to understand what information is needed, considering the previous turns in the conversation.
@@ -113,7 +139,7 @@ class GaiaSmolAgent:
             planning_interval=3 # Re-plan every 3 steps, considering memory.
         )
         
-        print("Optimized GaiaSmolAgent initialized successfully with native memory and multimodal capabilities.")
+        print("Optimized GaiaSmolAgent initialized successfully with native memory and full multimodal capabilities.")
 
     def __call__(self, question: str, reset_memory: bool = False) -> str:
         """