:sparkles: Adding image modality

.gitignore

@@ -1,2 +1,4 @@
 .venv
-.env
+.env
+
+test*

app.py

@@ -1,4 +1,6 @@
 import os
+import io
+import json
 import base64
 import gradio as gr
 import requests
@@ -17,6 +19,7 @@ load_dotenv()
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
+
 # --- Basic Agent Definition ---
 # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
 class BasicAgent:
@@ -35,60 +38,166 @@ class BasicAgent:
             tools=[DuckDuckGoSearchTool()],
             model=OpenAIServerModel(model_id=OPENAI_MODEL_ID, api_key=OPENAI_API_KEY),
             additional_authorized_imports=["BeautifulSoup"],
-            max_steps=20
+            max_steps=10,
         )
-    def __call__(self, question: str) -> str:
+
+    def __call__(self, question: str, file_data: dict = None) -> str:
         print(f"Agent received question (first 50 chars): {question[:50]}...")
         try:
-            # Use the integrated CodeAgent to get the response
-            result = self.agent.run(question)
+            images = None
+            if file_data is not None:
+                file_data = self.handle_file_data(file_data)
+                if file_data is not None:
+                    images = file_data["images"]
+                    if file_data["text"] is not None:
+                        question += "\n\n" + file_data["text"]
+
+            result = self.agent.run(question, images=images)
+
             print(f"Agent returned answer: {result}")
             return result
         except Exception as e:
             print(f"Error during CodeAgent.run: {e}")
             return f"Error from CodeAgent: {e}"
 
+    def handle_file_data(self, file_data: dict) -> dict:
+        if file_data["type"] in ["excel", "csv"]:
+            return {"images": None, "text": file_data["data"]}
+        elif file_data["type"] == "image":
+            return {"images": [file_data["data"]], "text": None}
+        elif file_data["type"] == "text":
+            return {"images": None, "text": file_data["data"]}
+        elif file_data["type"] == "audio":
+            return None
+        else:
+            return None
+
+
+def load_file_from_response(response):
+    """
+    Loads and identifies file content from an HTTP response based on its content-type.
+    Returns a dictionary with 'type' and 'data' keys.
+    """
+    content_type = response.headers.get("content-type", "").lower()
+    content_bytes = response.content
+
+    try:
+        if "application/json" in content_type:
+            if "No file path" in response.json()["detail"]:
+                return None
+            return {"type": "json", "data": response.json()}
+
+        elif "text/csv" in content_type:
+            return {"type": "csv", "data": pd.read_csv(io.StringIO(response.text))}
+
+        elif "text/plain" in content_type or "text/x-python" in content_type:
+            return {"type": "text", "data": response.text}
+
+        elif "image/" in content_type:
+            return {"type": "image", "data": Image.open(io.BytesIO(content_bytes))}
+
+        elif "audio/" in content_type:
+            audio_data, sample_rate = sf.read(io.BytesIO(content_bytes))
+            return {
+                "type": "audio",
+                "data": {"array": audio_data, "sample_rate": sample_rate},
+            }
+
+        elif "application/octet-stream" in content_type:
+            # Try Excel
+            try:
+                excel_data = pd.read_excel(io.BytesIO(content_bytes))
+                return {"type": "excel", "data": excel_data}
+            except Exception:
+                pass
+
+            # Try image
+            try:
+                img = Image.open(io.BytesIO(content_bytes))
+                return {"type": "image", "data": img}
+            except UnidentifiedImageError:
+                pass
+
+            # Try audio
+            try:
+                audio_data, sample_rate = sf.read(io.BytesIO(content_bytes))
+                return {
+                    "type": "audio",
+                    "data": {"array": audio_data, "sample_rate": sample_rate},
+                }
+            except RuntimeError:
+                pass
+
+            # Try UTF-8 text
+            try:
+                text = content_bytes.decode("utf-8")
+                return {"type": "text", "data": text}
+            except UnicodeDecodeError:
+                pass
+
+            return {"type": "binary", "data": content_bytes}
+
+        else:
+            print(f"⚠️ Unhandled content type: {content_type}")
+            return {"type": "unknown", "data": content_bytes}
+
+    except Exception as e:
+        print(f"❌ Failed to process content: {e}")
+        return {"type": "error", "data": str(e)}
+
+
 def load_pil_image(image_path: str) -> Image.Image:
     """Loads an image from disk for processing by GPT-4O."""
     return Image.open(image_path)
 
+
 def load_image(image_path: str) -> str:
     """Loads image and encodes it as base64 string for GPT-4o."""
     with open(image_path, "rb") as f:
         encoded = base64.b64encode(f.read()).decode("utf-8")
     return f"data:image/jpeg;base64,{encoded}"
 
+
 def load_audio(audio_path: str) -> str:
     """Encodes audio as base64 for GPT-4o (if needed)."""
     with open(audio_path, "rb") as f:
         encoded = base64.b64encode(f.read()).decode("utf-8")
     return f"data:audio/wav;base64,{encoded}"
 
+
+def transcribe_audio(audio_path: str) -> str:
+    """Transcribes audio file using OpenAI Whisper model (whisper-1)."""
+    with open(audio_path, "rb") as f:
+        transcript = openai.Audio.transcribe("whisper-1", f)
+    return transcript.get("text", "")
+
+
 def describe_image(image_path: str) -> str:
     """Sends image directly to GPT-4o to describe it."""
     image_base64 = load_image(image_path)
     messages = [
-        {"role": "user", "content": [
-            {"type": "text", "text": "Describe this image."},
-            {"type": "image_url", "image_url": {"url": image_base64}},
-        ]}
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Describe this image."},
+                {"type": "image_url", "image_url": {"url": image_base64}},
+            ],
+        }
     ]
-    response = openai.ChatCompletion.create(
-        model="gpt-4o",
-        messages=messages
-    )
+    response = openai.ChatCompletion.create(model="gpt-4o", messages=messages)
     return response.choices[0].message["content"]
 
-def run_and_submit_all( profile: gr.OAuthProfile | None):
+
+def run_and_submit_all(profile: gr.OAuthProfile | None):
     """
     Fetches all questions, runs the BasicAgent on them, submits all answers,
     and displays the results.
     """
     # --- Determine HF Space Runtime URL and Repo URL ---
-    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
+    space_id = os.getenv("SPACE_ID")  # Get the SPACE_ID for sending link to the code
 
     if profile:
-        username= f"{profile.username}"
+        username = f"{profile.username}"
         print(f"User logged in: {username}")
     else:
         print("User not logged in.")
@@ -97,6 +206,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
     api_url = DEFAULT_API_URL
     questions_url = f"{api_url}/questions"
     submit_url = f"{api_url}/submit"
+    files_url = f"{api_url}/files"  # /files/{task_id}
 
     # 1. Instantiate Agent ( modify this part to create your agent)
     try:
@@ -115,16 +225,16 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
         response.raise_for_status()
         questions_data = response.json()
         if not questions_data:
-             print("Fetched questions list is empty.")
-             return "Fetched questions list is empty or invalid format.", None
+            print("Fetched questions list is empty.")
+            return "Fetched questions list is empty or invalid format.", None
         print(f"Fetched {len(questions_data)} questions.")
     except requests.exceptions.RequestException as e:
         print(f"Error fetching questions: {e}")
         return f"Error fetching questions: {e}", None
     except requests.exceptions.JSONDecodeError as e:
-         print(f"Error decoding JSON response from questions endpoint: {e}")
-         print(f"Response text: {response.text[:500]}")
-         return f"Error decoding server response for questions: {e}", None
+        print(f"Error decoding JSON response from questions endpoint: {e}")
+        print(f"Response text: {response.text[:500]}")
+        return f"Error decoding server response for questions: {e}", None
     except Exception as e:
         print(f"An unexpected error occurred fetching questions: {e}")
         return f"An unexpected error occurred fetching questions: {e}", None
@@ -139,20 +249,43 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
+
+        file_data = load_file_from_response(
+            requests.get(f"{files_url}/{task_id}", timeout=15)
+        )
+
         try:
-            submitted_answer = agent(question_text)
-            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
-            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
+            submitted_answer = agent(question_text, file_data)
+            answers_payload.append(
+                {"task_id": task_id, "submitted_answer": submitted_answer}
+            )
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question_text,
+                    "Submitted Answer": submitted_answer,
+                }
+            )
         except Exception as e:
-             print(f"Error running agent on task {task_id}: {e}")
-             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
+            print(f"Error running agent on task {task_id}: {e}")
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question_text,
+                    "Submitted Answer": f"AGENT ERROR: {e}",
+                }
+            )
 
     if not answers_payload:
         print("Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
-    # 4. Prepare Submission 
-    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
+    # 4. Prepare Submission
+    submission_data = {
+        "username": username.strip(),
+        "agent_code": agent_code,
+        "answers": answers_payload,
+    }
     status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
     print(status_update)
 
@@ -222,20 +355,19 @@ with gr.Blocks() as demo:
 
     run_button = gr.Button("Run Evaluation & Submit All Answers")
 
-    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
+    status_output = gr.Textbox(
+        label="Run Status / Submission Result", lines=5, interactive=False
+    )
     # Removed max_rows=10 from DataFrame constructor
     results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
 
-    run_button.click(
-        fn=run_and_submit_all,
-        outputs=[status_output, results_table]
-    )
+    run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
 
 if __name__ == "__main__":
-    print("\n" + "-"*30 + " App Starting " + "-"*30)
+    print("\n" + "-" * 30 + " App Starting " + "-" * 30)
     # Check for SPACE_HOST and SPACE_ID at startup for information
     space_host_startup = os.getenv("SPACE_HOST")
-    space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
+    space_id_startup = os.getenv("SPACE_ID")  # Get SPACE_ID at startup
 
     if space_host_startup:
         print(f"✅ SPACE_HOST found: {space_host_startup}")
@@ -243,14 +375,18 @@ if __name__ == "__main__":
     else:
         print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
 
-    if space_id_startup: # Print repo URLs if SPACE_ID is found
+    if space_id_startup:  # Print repo URLs if SPACE_ID is found
         print(f"✅ SPACE_ID found: {space_id_startup}")
         print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
-        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
+        print(
+            f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main"
+        )
     else:
-        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
+        print(
+            "ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined."
+        )
 
-    print("-"*(60 + len(" App Starting ")) + "\n")
+    print("-" * (60 + len(" App Starting ")) + "\n")
 
     print("Launching Gradio Interface for Basic Agent Evaluation...")
-    demo.launch(debug=True, share=False)
+    demo.launch(debug=True, share=False)

pyproject.toml

@@ -8,6 +8,7 @@ dependencies = [
     "beautifulsoup4>=4.13.4",
     "dotenv>=0.9.9",
     "gradio[oauth]>=5.26.0",
+    "openpyxl>=3.1.5",
     "requests>=2.32.3",
     "smolagents[openai]>=1.14.0",
     "soundfile>=0.13.1",

uv.lock

@@ -14,6 +14,7 @@ dependencies = [
     { name = "beautifulsoup4" },
     { name = "dotenv" },
     { name = "gradio", extra = ["oauth"] },
+    { name = "openpyxl" },
     { name = "requests" },
     { name = "smolagents", extra = ["openai"] },
     { name = "soundfile" },
@@ -24,6 +25,7 @@ requires-dist = [
     { name = "beautifulsoup4", specifier = ">=4.13.4" },
     { name = "dotenv", specifier = ">=0.9.9" },
     { name = "gradio", extras = ["oauth"], specifier = ">=5.26.0" },
+    { name = "openpyxl", specifier = ">=3.1.5" },
     { name = "requests", specifier = ">=2.32.3" },
     { name = "smolagents", extras = ["openai"], specifier = ">=1.14.0" },
     { name = "soundfile", specifier = ">=0.13.1" },
@@ -293,6 +295,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/83/a2/66adca41164860dee6d2d47b506fef3262c8879aab727b687c798d67313f/duckduckgo_search-8.0.1-py3-none-any.whl", hash = "sha256:87ea18d9abb1cd5dc8f63fc70ac867996acce2cb5e0129d191b9491c202420be", size = 18125 },
 ]
 
+[[package]]
+name = "et-xmlfile"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059 },
+]
+
 [[package]]
 name = "fastapi"
 version = "0.115.12"
@@ -697,6 +708,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/59/aa/84e02ab500ca871eb8f62784426963a1c7c17a72fea3c7f268af4bbaafa5/openai-1.76.0-py3-none-any.whl", hash = "sha256:a712b50e78cf78e6d7b2a8f69c4978243517c2c36999756673e07a14ce37dc0a", size = 661201 },
 ]
 
+[[package]]
+name = "openpyxl"
+version = "3.1.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "et-xmlfile" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910 },
+]
+
 [[package]]
 name = "orjson"
 version = "3.10.16"