Update app.py

app.py

@@ -5,7 +5,6 @@ import pandas as pd
 from smolagents import (
     CodeAgent,
     DuckDuckGoSearchTool,
-    WebSearchTool,
     VisitWebpageTool,
     InferenceClientModel,
     tool,
@@ -23,7 +22,7 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 def download_file_from_api(task_id: str) -> str:
     """Downloads a file associated with a GAIA task and returns its text content.
     Use this tool when a question mentions an attached file, spreadsheet, image,
-    audio, or any document that you need to read.
+    audio, document, or any file that you need to read or analyze.
 
     Args:
         task_id: The task_id string for the question that has an associated file.
@@ -36,25 +35,29 @@ def download_file_from_api(task_id: str) -> str:
         response.raise_for_status()
 
         content_type = response.headers.get("Content-Type", "")
+        print(f"  [download_file] Content-Type: {content_type}, Size: {len(response.content)} bytes")
 
-        # If it's a text-based file, return the text directly
-        if "text" in content_type or "json" in content_type or "csv" in content_type:
-            return response.text[:10000]  # Limit to avoid context overflow
-
-        # For Excel files
-        if "spreadsheet" in content_type or "excel" in content_type:
-            import openpyxl
-            import io
-            wb = openpyxl.load_workbook(io.BytesIO(response.content))
-            result = []
-            for sheet_name in wb.sheetnames:
-                ws = wb[sheet_name]
-                result.append(f"--- Sheet: {sheet_name} ---")
-                for row in ws.iter_rows(values_only=True):
-                    result.append("\t".join([str(c) if c is not None else "" for c in row]))
-            return "\n".join(result)[:10000]
-
-        # For PDF files
+        # --- TEXT-BASED FILES ---
+        if any(t in content_type for t in ["text", "json", "csv", "xml", "html"]):
+            return response.text[:15000]
+
+        # --- EXCEL FILES ---
+        if any(t in content_type for t in ["spreadsheet", "excel", "openxmlformats-officedocument"]):
+            try:
+                import openpyxl
+                import io
+                wb = openpyxl.load_workbook(io.BytesIO(response.content))
+                result = []
+                for sheet_name in wb.sheetnames:
+                    ws = wb[sheet_name]
+                    result.append(f"--- Sheet: {sheet_name} ---")
+                    for row in ws.iter_rows(values_only=True):
+                        result.append("\t".join([str(c) if c is not None else "" for c in row]))
+                return "\n".join(result)[:15000]
+            except Exception as e:
+                return f"Excel file detected but error reading it: {str(e)}"
+
+        # --- PDF FILES ---
         if "pdf" in content_type:
             try:
                 import PyPDF2
@@ -63,70 +66,204 @@ def download_file_from_api(task_id: str) -> str:
                 text = ""
                 for page in reader.pages:
                     text += page.extract_text() or ""
-                return text[:10000] if text else "PDF found but could not extract text."
-            except ImportError:
-                return "PDF file detected but PyPDF2 not installed."
+                return text[:15000] if text.strip() else "PDF found but could not extract text (may be scanned/image-based)."
+            except Exception as e:
+                return f"PDF file detected but error reading: {str(e)}"
 
-        # For other binary files, save and report
-        suffix = ""
+        # --- IMAGE FILES ---
         if "image" in content_type:
-            suffix = ".png"
-        elif "audio" in content_type:
-            suffix = ".mp3"
-
-        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as f:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as f:
+                f.write(response.content)
+                img_path = f.name
+            return f"IMAGE_FILE_SAVED:{img_path}"
+
+        # --- AUDIO FILES ---
+        if any(t in content_type for t in ["audio", "mpeg", "wav", "mp3", "ogg"]):
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as f:
+                f.write(response.content)
+                audio_path = f.name
+            return f"AUDIO_FILE_SAVED:{audio_path}"
+
+        # --- PYTHON FILES ---
+        if "python" in content_type or "x-python" in content_type:
+            return response.text[:15000]
+
+        # --- WORD DOCUMENTS ---
+        if "wordprocessingml" in content_type or "msword" in content_type:
+            try:
+                import docx
+                import io
+                doc = docx.Document(io.BytesIO(response.content))
+                text = "\n".join([p.text for p in doc.paragraphs])
+                return text[:15000] if text.strip() else "Word doc found but no text extracted."
+            except Exception as e:
+                return f"Word document detected but error reading: {str(e)}"
+
+        # --- FALLBACK ---
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".bin") as f:
             f.write(response.content)
-            return f"File downloaded to {f.name} (type: {content_type}). File size: {len(response.content)} bytes."
+            return f"File downloaded to {f.name} (type: {content_type}). Size: {len(response.content)} bytes. Could not auto-parse."
 
     except Exception as e:
         return f"Error downloading file for task {task_id}: {str(e)}"
 
 
+@tool
+def describe_image(image_path: str) -> str:
+    """Describes the content of an image file using an AI vision model.
+    Use this when you have an image file path (e.g. from IMAGE_FILE_SAVED)
+    and need to understand what the image shows, including any text in it.
+
+    Args:
+        image_path: The local file path to the image to describe.
+    """
+    try:
+        from huggingface_hub import InferenceClient
+
+        token = os.getenv("HF_TOKEN")
+        client = InferenceClient(token=token)
+
+        with open(image_path, "rb") as f:
+            image_bytes = f.read()
+
+        # Use BLIP2 for image captioning
+        result = client.image_to_text(
+            image=image_bytes,
+            model="Salesforce/blip2-opt-2.7b",
+        )
+
+        if isinstance(result, str):
+            description = result
+        elif hasattr(result, "generated_text"):
+            description = result.generated_text
+        else:
+            description = str(result)
+
+        return f"Image description: {description}"
+
+    except Exception as e:
+        return f"Could not describe image at {image_path}. Error: {str(e)}"
+
+
+@tool
+def transcribe_audio(audio_path: str) -> str:
+    """Transcribes an audio file to text using Whisper speech recognition.
+    Use this when you have an audio file path (e.g. from AUDIO_FILE_SAVED)
+    and need to know what is spoken in the recording.
+
+    Args:
+        audio_path: The local file path to the audio file to transcribe.
+    """
+    try:
+        from huggingface_hub import InferenceClient
+
+        token = os.getenv("HF_TOKEN")
+        client = InferenceClient(token=token)
+
+        with open(audio_path, "rb") as f:
+            audio_bytes = f.read()
+
+        result = client.automatic_speech_recognition(
+            audio=audio_bytes,
+            model="openai/whisper-large-v3-turbo",
+        )
+
+        if isinstance(result, str):
+            return f"Audio transcription: {result}"
+        elif hasattr(result, "text"):
+            return f"Audio transcription: {result.text}"
+        elif isinstance(result, dict):
+            return f"Audio transcription: {result.get('text', str(result))}"
+        else:
+            return f"Audio transcription: {str(result)}"
+
+    except Exception as e:
+        return f"Error transcribing audio at {audio_path}: {str(e)}"
+
+
 @tool
 def read_local_file(file_path: str) -> str:
-    """Reads the content of a local file and returns it as a string.
+    """Reads the content of a local text file and returns it as a string.
 
     Args:
         file_path: The path to the file to read.
     """
     try:
         with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
-            return f.read()[:10000]
+            return f.read()[:15000]
     except Exception as e:
         return f"Error reading file: {str(e)}"
 
 
+@tool
+def execute_python_file(file_path: str) -> str:
+    """Executes a Python script file and returns its stdout output.
+    Use this when you receive a .py file that needs to be run to get the answer.
+
+    Args:
+        file_path: The path to the Python file to execute.
+    """
+    import subprocess
+    try:
+        result = subprocess.run(
+            ["python3", file_path],
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        output = ""
+        if result.stdout:
+            output += result.stdout
+        if result.stderr:
+            output += f"\nSTDERR: {result.stderr}"
+        if result.returncode != 0:
+            output += f"\nReturn code: {result.returncode}"
+        return output.strip() if output.strip() else "Script executed but produced no output."
+    except subprocess.TimeoutExpired:
+        return "Script execution timed out after 30 seconds."
+    except Exception as e:
+        return f"Error executing Python file: {str(e)}"
+
+
 # =============================================
 # AGENT CLASS
 # =============================================
 
 class BasicAgent:
-    """An agent that uses smolagents CodeAgent with web search and file tools
-    to answer GAIA benchmark questions. Uses Hugging Face Inference API (no GPU needed)."""
+    """An agent using smolagents CodeAgent with web search, file handling,
+    image description, and audio transcription tools.
+    Uses HF Inference API — no GPU needed."""
 
     def __init__(self):
         print("Initializing SmolAgent for GAIA benchmark...")
 
-        # Use HF Inference API - completely free, no GPU needed
         model = InferenceClientModel(
             model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
             token=os.getenv("HF_TOKEN"),
             max_tokens=2096,
-            temperature=0.1,  # Low temperature for more precise answers
+            temperature=0.1,
         )
 
-        # System prompt tailored for GAIA exact-match scoring
         system_prompt = """You are a precise AI assistant solving GAIA benchmark questions.
 
-CRITICAL RULES:
-1. Your final answer must be ONLY the answer itself - no explanations, no prefixes like "The answer is", no extra text.
-2. If the answer is a number, give just the number (e.g., "42" not "The answer is 42").
-3. If the answer is a name, give just the name (e.g., "Paris" not "The answer is Paris").
-4. If the answer is a list, use comma-separated values as requested.
-5. Be precise and factual. Use tools to verify information.
-6. If a question mentions a file or attachment, use the download_file_from_api tool with the task_id.
-7. Always search the web when you need current or specific factual information.
-8. Think step by step, but your FINAL output must be ONLY the answer.
+CRITICAL RULES FOR ANSWERING:
+1. Your final answer must be ONLY the answer itself — no explanations, no "The answer is", no extra words.
+2. If the answer is a number, give just the number (e.g., "42").
+3. If the answer is a name, give just the name (e.g., "Paris").
+4. If asked for a comma-separated list, give just the list (e.g., "red, blue, green").
+5. Be precise and factual. Use tools to verify information when needed.
+
+TOOL USAGE RULES:
+6. If a question mentions an attached file, image, audio, spreadsheet, or document, FIRST use download_file_from_api with the task_id.
+7. If download returns "IMAGE_FILE_SAVED:/some/path", then call describe_image("/some/path") to see what the image contains.
+8. If download returns "AUDIO_FILE_SAVED:/some/path", then call transcribe_audio("/some/path") to hear what is said.
+9. If the file is a Python script (.py), you can use read_local_file to view it or execute_python_file to run it.
+10. Use DuckDuckGoSearchTool when you need factual information from the internet.
+11. Use visit_webpage to read the full content of a specific URL.
+
+REASONING:
+12. Think step by step but keep your FINAL output as ONLY the answer.
+13. Double-check your answer before giving it.
 """
 
         self.agent = CodeAgent(
@@ -135,26 +272,26 @@ CRITICAL RULES:
                 DuckDuckGoSearchTool(),
                 VisitWebpageTool(),
                 download_file_from_api,
+                describe_image,
+                transcribe_audio,
                 read_local_file,
+                execute_python_file,
             ],
-            max_steps=8,
+            max_steps=10,
             verbosity_level=1,
             additional_authorized_imports=[
                 "json", "re", "math", "datetime", "collections",
-                "csv", "io", "os", "tempfile",
+                "csv", "io", "os", "tempfile", "subprocess",
+                "base64", "hashlib", "unicodedata", "string",
             ],
         )
 
-        # Override system prompt
         self.agent.system_prompt = system_prompt + "\n\n" + self.agent.system_prompt
-
         print("SmolAgent initialized successfully!")
 
     def __call__(self, question: str, task_id: str = None) -> str:
-        print(f"Agent processing question: {question[:80]}...")
+        print(f"Agent processing: {question[:100]}...")
 
-        # Build the prompt with task_id context if available
-        prompt = question
         if task_id:
             prompt = f"""Answer this question. If you need to download an attached file, use download_file_from_api with task_id="{task_id}".
 
@@ -170,34 +307,42 @@ Remember: respond with ONLY the final answer, nothing else."""
 
         try:
             result = self.agent.run(prompt)
-            # Clean up the answer
             answer = str(result).strip()
-            # Remove common prefixes that models add
-            for prefix in [
-                "The answer is ", "The answer is: ", "Answer: ", "FINAL ANSWER: ",
-                "Final answer: ", "The final answer is ", "The final answer is: ",
-            ]:
+
+            # Clean up common LLM prefixes
+            prefixes_to_remove = [
+                "The answer is ", "The answer is: ",
+                "Answer: ", "FINAL ANSWER: ",
+                "Final answer: ", "The final answer is ",
+                "The final answer is: ", "Result: ",
+            ]
+            for prefix in prefixes_to_remove:
                 if answer.lower().startswith(prefix.lower()):
                     answer = answer[len(prefix):].strip()
-            # Remove trailing periods if the answer is not a sentence
+
+            # Remove wrapping quotes
+            if len(answer) > 2 and \
+               ((answer.startswith('"') and answer.endswith('"')) or
+                (answer.startswith("'") and answer.endswith("'"))):
+                answer = answer[1:-1].strip()
+
+            # Remove trailing period for short answers
             if answer.endswith(".") and len(answer.split()) <= 5:
                 answer = answer[:-1].strip()
-            print(f"Agent answer: {answer}")
+
+            print(f"Final answer: {answer}")
             return answer
+
         except Exception as e:
             print(f"Agent error: {e}")
             return "Unable to determine the answer."
 
 
 # =============================================
-# SUBMISSION LOGIC 
+# SUBMISSION LOGIC
 # =============================================
 
 def run_and_submit_all(profile: gr.OAuthProfile | None):
-    """
-    Fetches all questions, runs the agent on them, submits all answers,
-    and displays the results.
-    """
     space_id = os.getenv("SPACE_ID")
 
     if profile:
@@ -211,7 +356,6 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     questions_url = f"{api_url}/questions"
     submit_url = f"{api_url}/submit"
 
-    # 1. Instantiate Agent
     try:
         agent = BasicAgent()
     except Exception as e:
@@ -221,24 +365,19 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
     print(agent_code)
 
-    # 2. Fetch Questions
     print(f"Fetching questions from: {questions_url}")
     try:
         response = requests.get(questions_url, timeout=15)
         response.raise_for_status()
         questions_data = response.json()
         if not questions_data:
-            print("Fetched questions list is empty.")
             return "Fetched questions list is empty or invalid format.", None
         print(f"Fetched {len(questions_data)} questions.")
     except requests.exceptions.RequestException as e:
-        print(f"Error fetching questions: {e}")
         return f"Error fetching questions: {e}", None
     except Exception as e:
-        print(f"An unexpected error occurred fetching questions: {e}")
         return f"An unexpected error occurred fetching questions: {e}", None
 
-    # 3. Run Agent on each question
     results_log = []
     answers_payload = []
     print(f"Running agent on {len(questions_data)} questions...")
@@ -246,11 +385,11 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         task_id = item.get("task_id")
         question_text = item.get("question")
         if not task_id or question_text is None:
-            print(f"Skipping item with missing task_id or question: {item}")
             continue
-        print(f"\n{'='*50}")
-        print(f"Question {i+1}/{len(questions_data)} - Task: {task_id}")
-        print(f"{'='*50}")
+        print(f"\n{'='*60}")
+        print(f"  Question {i+1}/{len(questions_data)} — Task: {task_id}")
+        print(f"  Q: {question_text[:120]}...")
+        print(f"{'='*60}")
         try:
             submitted_answer = agent(question_text, task_id=task_id)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
@@ -268,19 +407,13 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             })
 
     if not answers_payload:
-        print("Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
-    # 4. Prepare Submission
     submission_data = {
         "username": username.strip(),
         "agent_code": agent_code,
         "answers": answers_payload
     }
-    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
-    print(status_update)
-
-    # 5. Submit
     print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
     try:
         response = requests.post(submit_url, json=submission_data, timeout=120)
@@ -294,8 +427,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             f"Message: {result_data.get('message', 'No message received.')}"
         )
         print("Submission successful.")
-        results_df = pd.DataFrame(results_log)
-        return final_status, results_df
+        return final_status, pd.DataFrame(results_log)
     except requests.exceptions.HTTPError as e:
         error_detail = f"Server responded with status {e.response.status_code}."
         try:
@@ -303,35 +435,23 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
         except requests.exceptions.JSONDecodeError:
             error_detail += f" Response: {e.response.text[:500]}"
-        status_message = f"Submission Failed: {error_detail}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
+        return f"Submission Failed: {error_detail}", pd.DataFrame(results_log)
     except requests.exceptions.Timeout:
-        status_message = "Submission Failed: The request timed out."
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
+        return "Submission Failed: The request timed out.", pd.DataFrame(results_log)
     except requests.exceptions.RequestException as e:
-        status_message = f"Submission Failed: Network error - {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
+        return f"Submission Failed: Network error - {e}", pd.DataFrame(results_log)
     except Exception as e:
-        status_message = f"An unexpected error occurred during submission: {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
+        return f"An unexpected error occurred during submission: {e}", pd.DataFrame(results_log)
 
 
-# --- Build Gradio Interface using Blocks ---
+# --- Build Gradio Interface ---
 with gr.Blocks() as demo:
-    gr.Markdown("# 🤖 GAIA Agent - Final Assignment")
+    gr.Markdown("# 🤖 GAIA Agent — Final Assignment")
     gr.Markdown(
         """
         **Agent**: SmolAgent (CodeAgent) with Qwen2.5-Coder-32B via HF Inference API
 
-        **Tools**: Web Search, Webpage Visitor, File Downloader
+        **Tools**: Web Search · Webpage Visitor · File Downloader · Image Describer · Audio Transcriber · Python Executor
 
         **Instructions:**
         1. Log in to your Hugging Face account using the button below.
@@ -357,15 +477,13 @@ if __name__ == "__main__":
 
     if space_host_startup:
         print(f"✅ SPACE_HOST found: {space_host_startup}")
-        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
     else:
-        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
+        print("ℹ️  SPACE_HOST not found (running locally?).")
 
     if space_id_startup:
         print(f"✅ SPACE_ID found: {space_id_startup}")
-        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
     else:
-        print("ℹ️  SPACE_ID environment variable not found (running locally?).")
+        print("ℹ️  SPACE_ID not found (running locally?).")
 
     print("-"*60 + "\n")
     print("Launching Gradio Interface...")