Update app.py

app.py

@@ -4,6 +4,12 @@ import requests
 import inspect
 import pandas as pd
 
+import speech_recognition as sr
+import magic 
+from io import BytesIO
+from pydub import AudioSegment
+from PIL import Image
+
 from typing import TypedDict, Annotated
 from langgraph.graph.message import add_messages
 from langchain_core.messages import AnyMessage, HumanMessage, AIMessage, SystemMessage
@@ -55,9 +61,10 @@ class BasicAgent:
         # Compile the agent
         self.agent = builder.compile()
         
-    def __call__(self, question: str) -> str:
+    def __call__(self, question: str, task_id: str) -> str:
         print(f"Agent received question (first 50 chars): {question[:50]}...")
 
+        
         system_prompt = """
             You are a general AI assistant. I will ask you a question. Report your thoughts, and finish
             your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER].
@@ -70,6 +77,12 @@ class BasicAgent:
             If you are asked for a comma separated list, apply the above rules depending of whether the element to be put
             in the list is a number or a string.
         """
+        text_content = download_and_detect_file(
+            f"{DEFAULT_API_URL}/files/{task_id}",
+        )
+        if  text_content:
+          question += f' attached data: {text_content} '
+
         messages = [SystemMessage(content=system_prompt), HumanMessage(content=question)]
         response = self.agent.invoke({"messages": messages})
         
@@ -82,6 +95,79 @@ class BasicAgent:
         print(f"Agent returning fixed answer: {answer}")
         return answer
 
+def download_and_detect_file(url: str):
+    try:
+        # Download the file
+        response = requests.get(url)
+        response.raise_for_status()  # Raise error if download failed
+        
+        # Detect MIME type using python-magic (it will try to guess the type)
+        mime_type = magic.from_buffer(response.content, mime=True)
+        print(f"Detected MIME type: {mime_type}")
+
+        # Handle the file based on MIME type
+        if mime_type == mime_type_xlsx:
+            # Handle Excel file
+            # Handle Excel file
+            excel_data = BytesIO(response.content)
+            excel_df = pd.read_excel(excel_data)
+            
+            # Convert DataFrame to a plain text string (could be tabular or a simple concatenation)
+            text_content = ""
+            
+            # Iterate through the DataFrame and convert it to text
+            for index, row in excel_df.iterrows():
+                row_text = ' | '.join([str(cell) for cell in row])  # Join cell values in each row with "|"
+                text_content += row_text + "\n"  # Add a newline after each row
+            
+            return text_content
+
+        elif mime_type == mime_type_txt:
+            # Handle Text file
+            return response.text  # Treat it as a text file
+
+        elif mime_type == mime_type_mp3:
+            # Handle MP3 file
+            audio_data = BytesIO(response.content)
+
+            # Convert MP3 to WAV using pydub (speech_recognition only works with WAV, AIFF, or FLAC)
+            audio_segment = AudioSegment.from_mp3(audio_data)
+            wav_audio = BytesIO()
+            audio_segment.export(wav_audio, format="wav")
+            wav_audio.seek(0)  # Go to the start of the BytesIO object
+            
+            # Perform speech recognition using speech_recognition library
+            recognizer = sr.Recognizer()
+            with sr.AudioFile(wav_audio) as source:
+                audio = recognizer.record(source)  # Read the audio file
+            try:
+                # Use Google Web Speech API to convert speech to text
+                text = recognizer.recognize_google(audio)
+                return text
+            except sr.UnknownValueError:
+                return "Google Speech Recognition could not understand the audio"
+            except sr.RequestError as e:
+                return f"Could not request results from Google Speech Recognition service; {e}"
+
+        elif mime_type == mime_type_jpg:
+            # Handle JPEG Image file
+            return url
+
+        elif mime_type == mime_type_png:
+            # Handle PNG Image file
+            return url
+
+        else:
+            return f"Unsupported MIME type: {mime_type}"
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error downloading the file: {e}")
+        return None
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return None
+
+
 def run_and_submit_all( profile: gr.OAuthProfile | None):
     """
     Fetches all questions, runs the BasicAgent on them, submits all answers,
@@ -265,7 +351,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text)
+            submitted_answer = agent(question_text, task_id)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
         except Exception as e: