Update app.py

app.py

@@ -1,8 +1,15 @@
+import base64
+import inspect
+import mimetypes
 import os
+import tempfile
+
 import gradio as gr
-import requests
-import inspect
 import pandas as pd
+import requests
+from langchain_community.document_loaders import UnstructuredExcelLoader
+
+from agents import supervisor_agent
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -10,14 +17,98 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
 # --- Basic Agent Definition ---
 # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
-class BasicAgent:
+def fetch_file(task_id):
+    if not task_id:
+        return None
+    url = f"https://agents-course-unit4-scoring.hf.space/files/{task_id}"
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.content  # Return raw bytes
+    else:
+        print(f"Failed to fetch file for task_id {task_id}: {response.status_code}")
+        return None
+
+def build_multimodal_message(question, file_bytes=None, file_name=None):
+    """
+    Build a multimodal message with correct content blocks for text, image, audio, or file.
+    For .xlsx files, extract the text and append it to the question, since LLMs do not natively support .xlsx.
+    Follows: https://python.langchain.com/docs/how_to/multimodal_inputs/
+    """
+    content = []
+    # Special handling for .xlsx files
+    if file_bytes and file_name and file_name.lower().endswith('.xlsx'):
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.xlsx') as tmp:
+            tmp.write(file_bytes)
+            tmp_path = tmp.name
+        loader = UnstructuredExcelLoader(tmp_path, mode="elements")
+        docs = loader.load()
+        excel_text = "\n".join(doc.page_content for doc in docs)
+        question = f"{question}\n\n[Excel file content follows:]\n{excel_text}"
+    content.append({"type": "text", "text": question})
+    if file_bytes and file_name and not file_name.lower().endswith('.xlsx'):
+        ext = file_name.lower().split('.')[-1]
+        b64_data = base64.b64encode(file_bytes).decode("utf-8")
+        mime_type, _ = mimetypes.guess_type(file_name)
+        # Handle common audio/image types explicitly
+        if ext in ["png"]:
+            mime_type = "image/png"
+            block_type = "image"
+        elif ext in ["jpg", "jpeg"]:
+            mime_type = "image/jpeg"
+            block_type = "image"
+        elif ext == "mp3":
+            mime_type = "audio/mpeg"
+            block_type = "audio"
+        elif ext == "wav":
+            mime_type = "audio/wav"
+            block_type = "audio"
+        elif ext == "m4a":
+            mime_type = "audio/mp4"
+            block_type = "audio"
+        else:
+            block_type = "file"
+            if not mime_type:
+                mime_type = "application/octet-stream"
+        block = {
+            "type": block_type,
+            "source_type": "base64",
+            "data": b64_data,
+            "mime_type": mime_type,
+            "filename": file_name,
+        }
+        content.append(block)
+    return [{"role": "user", "content": content}]
+
+def filter_supported_content_blocks(messages):
+    allowed_types = {"text", "image_url", "input_audio", "refusal", "audio", "file", "image"}
+    filtered = []
+    for msg in messages:
+        if "content" in msg and isinstance(msg["content"], list):
+            filtered_content = [block for block in msg["content"] if block.get("type") in allowed_types]
+            msg = dict(msg)
+            msg["content"] = filtered_content
+        filtered.append(msg)
+    return filtered
+
+class Agent:
     def __init__(self):
-        print("BasicAgent initialized.")
-    def __call__(self, question: str) -> str:
+        self.main_agent = supervisor_agent
+        print("Agent initialized.")
+    def __call__(self, question: str, file_name: str = "", task_id: str = "") -> str:
         print(f"Agent received question (first 50 chars): {question[:50]}...")
-        fixed_answer = "This is a default answer."
-        print(f"Agent returning fixed answer: {fixed_answer}")
-        return fixed_answer
+        file_bytes = fetch_file(task_id) if file_name else None
+        message = build_multimodal_message(question, file_bytes, file_name)
+        # Filter out unsupported content block types
+        message = filter_supported_content_blocks(message)
+        result = self.main_agent.invoke({"messages": message})
+        answer = result["messages"][-1]
+        content = answer.content
+        if isinstance(content, list) and content and isinstance(content[0], dict) and "text" in content[0]:
+            return content[0]["text"]
+        elif isinstance(content, str):
+            return content
+        else:
+            return str(content)
 
 def run_and_submit_all( profile: gr.OAuthProfile | None):
     """
@@ -40,7 +131,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
 
     # 1. Instantiate Agent ( modify this part to create your agent)
     try:
-        agent = BasicAgent()
+        agent = Agent()
     except Exception as e:
         print(f"Error instantiating agent: {e}")
         return f"Error initializing agent: {e}", None
@@ -76,16 +167,17 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
     for item in questions_data:
         task_id = item.get("task_id")
         question_text = item.get("question")
+        file_name = item.get("file_name", "")
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text)
+            submitted_answer = agent(question_text, file_name, task_id)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
         except Exception as e:
-             print(f"Error running agent on task {task_id}: {e}")
-             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
+            print(f"Error running agent on task {task_id}: {e}")
+            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
 
     if not answers_payload:
         print("Agent did not produce any answers to submit.")
@@ -149,17 +241,38 @@ with gr.Blocks() as demo:
 
         1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
         2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
-        3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
+        3.  Enter your OpenAI and Google API keys below (these are required for the agent to work).
+        4.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
 
         ---
         **Disclaimers:**
-        Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
-        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
+        Once clicking on the "submit" button, it can take quite some time (this is the time for the agent to go through all the questions).
+        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a separate action or even to answer the questions in async.
         """
     )
 
     gr.LoginButton()
 
+    openai_key_box = gr.Textbox(label="OpenAI API Key", type="password", placeholder="sk-...", lines=1)
+    google_key_box = gr.Textbox(label="Google API Key", type="password", placeholder="AIza...", lines=1)
+    set_keys_btn = gr.Button("Set API Keys")
+    status_api_keys = gr.Textbox(label="API Key Status", lines=1, interactive=False)
+
+    def set_api_keys(openai_key, google_key):
+        if openai_key:
+            os.environ["OPENAI_API_KEY"] = openai_key
+        if google_key:
+            os.environ["GOOGLE_API_KEY"] = google_key
+        if openai_key or google_key:
+            return "API keys set for this session."
+        return "No API keys provided."
+
+    set_keys_btn.click(
+        fn=set_api_keys,
+        inputs=[openai_key_box, google_key_box],
+        outputs=status_api_keys
+    )
+
     run_button = gr.Button("Run Evaluation & Submit All Answers")
 
     status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)