khm12342's picture
Update app.py
d438944 verified
Raw History Blame
8.44 kB
"""Hugging Face Agents Course Unit 4 - GAIA evaluation Space.
Based on the public Final_Assignment_Template API contract. It is a starter
agent, not a guaranteed passing solution. Keep your Space public for scoring.
"""
import io
import os
import re
from pathlib import Path
import gradio as gr
import pandas as pd
import requests
from smolagents import (
CodeAgent,
DuckDuckGoSearchTool,
InferenceClientModel,
VisitWebpageTool,
)
API_URL = "https://agents-course-unit4-scoring.hf.space"
MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-Next-80B-A3B-Instruct")
def read_attachment(task_id: str, file_name: str) -> str:
"""Download an official task attachment and extract readable text."""
if not file_name:
return ""
try:
response = requests.get(f"{API_URL}/files/{task_id}", timeout=30)
response.raise_for_status()
if len(response.content) > 15_000_000:
return "Attachment is too large to read with this starter."
suffix = Path(file_name).suffix.lower()
data = io.BytesIO(response.content)
if suffix in {".txt", ".md", ".py", ".json", ".html", ".xml"}:
contents = response.content.decode("utf-8", errors="replace")
elif suffix == ".pdf":
from pypdf import PdfReader
pdf = PdfReader(data)
contents = "\n".join(page.extract_text() or "" for page in pdf.pages[:40])
elif suffix == ".csv":
contents = pd.read_csv(data).head(100).to_string(index=False)
elif suffix in {".xlsx", ".xls"}:
sheets = pd.read_excel(data, sheet_name=None)
contents = "\n".join(
f"SHEET {name}:\n{frame.head(60).to_string(index=False)}"
for name, frame in sheets.items()
)
elif suffix == ".docx":
from docx import Document
doc = Document(data)
contents = "\n".join(p.text for p in doc.paragraphs)
else:
return (
f"Attachment '{file_name}' is present, but this starter cannot "
"interpret its media format. Do not guess its contents."
)
return f"Attachment ({file_name}):\n{contents[:14000]}"
except Exception as exc:
return f"Could not read attachment {file_name}: {type(exc).__name__}: {exc}"
class BasicAgent:
def __init__(self):
token = os.getenv("HF_TOKEN")
if not token:
raise RuntimeError(
"HF_TOKEN Secret is missing. Add it in Space Settings "
"-> Variables and secrets -> New secret."
)
model = InferenceClientModel(
model_id=MODEL_ID,
token=token,
timeout=120,
max_tokens=1800,
)
self.agent = CodeAgent(
tools=[DuckDuckGoSearchTool(max_results=5), VisitWebpageTool()],
model=model,
max_steps=6,
additional_authorized_imports=[
"math", "statistics", "datetime", "re", "json", "itertools"
],
)
def __call__(self, question: str, attachment: str = "") -> str:
prompt = (
"You are solving a GAIA benchmark question. Use web search and "
"calculations if needed. Check evidence and do not invent facts. "
"Your final_answer must contain ONLY the shortest precise answer "
"requested by the question. No explanation, citations, markdown, "
"or the words FINAL ANSWER. Preserve requested units and format.\n\n"
f"QUESTION:\n{question}\n\n"
f"{attachment}"
)
answer = str(self.agent.run(prompt, reset=True)).strip()
answer = re.sub(r"^\s*(?:final answer|answer)\s*:\s*", "", answer, flags=re.I)
answer = re.sub(r"<think>.*?</think>", "", answer, flags=re.I | re.S).strip()
return answer.strip("` \n")
def answer_task(agent: BasicAgent, item: dict) -> str:
file_context = read_attachment(item["task_id"], item.get("file_name") or "")
return agent(item["question"], file_context)
def test_one_question():
"""Try one official sample before running all 20 questions."""
try:
response = requests.get(f"{API_URL}/random-question", timeout=20)
response.raise_for_status()
item = response.json()
if not isinstance(item, dict) or not item.get("task_id"):
return "The sample API returned an unexpected response."
agent = BasicAgent()
answer = answer_task(agent, item)
return f"Question: {item['question']}\n\nAgent answer: {answer}"
except Exception as exc:
return f"Test failed: {type(exc).__name__}: {exc}"
def run_and_submit_all(profile: gr.OAuthProfile | None):
"""Run the official GAIA question set and submit the generated answers."""
if not profile:
return "Please log in with the Hugging Face button first.", None
space_id = os.getenv("SPACE_ID")
if not space_id:
return "SPACE_ID is missing. Run this code inside your own Hugging Face Space.", None
username = profile.username
try:
agent = BasicAgent()
response = requests.get(f"{API_URL}/questions", timeout=30)
response.raise_for_status()
questions = response.json()
if not isinstance(questions, list) or not questions:
return "The question API did not return a question list.", None
except Exception as exc:
return f"Initialization failed: {type(exc).__name__}: {exc}", None
payload = []
rows = []
for index, item in enumerate(questions, 1):
task_id = item.get("task_id")
question = item.get("question")
if not task_id or not question:
continue
try:
answer = answer_task(agent, item)
if not answer:
raise ValueError("The agent returned an empty answer")
payload.append({"task_id": task_id, "submitted_answer": answer})
rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": answer})
print(f"Task {index}/{len(questions)} completed")
except Exception as exc:
error = f"ERROR: {type(exc).__name__}: {exc}"
print(f"Task {index}/{len(questions)} failed: {error}")
rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": error})
table = pd.DataFrame(rows)
if not payload:
return "No answers generated. Check the Space logs and HF_TOKEN / model quota.", table
request_data = {
"username": username,
"agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main",
"answers": payload,
}
try:
response = requests.post(f"{API_URL}/submit", json=request_data, timeout=90)
response.raise_for_status()
result = response.json()
status = (
"Submission Successful!\n"
f"User: {result.get('username', username)}\n"
f"Overall Score: {result.get('score', '?')}% "
f"({result.get('correct_count', '?')}/{result.get('total_attempted', '?')} correct)\n"
f"Submitted answers: {len(payload)}/{len(questions)}\n"
f"Message: {result.get('message', '')}"
)
return status, table
except Exception as exc:
return f"Submission failed: {type(exc).__name__}: {exc}", table
with gr.Blocks() as demo:
gr.Markdown("# Hugging Face Agents Course - Unit 4")
gr.Markdown(
"First configure the **HF_TOKEN** Secret in your Space Settings. "
"Log in below for the leaderboard. Test one question before submitting all 20. "
"This starter handles web search and text-like attachments; image/audio "
"tasks need additional tools. Model calls may consume paid credits."
)
gr.LoginButton()
test_button = gr.Button("1. Test one question")
sample_output = gr.Textbox(label="Sample test result", lines=8)
test_button.click(fn=test_one_question, outputs=sample_output)
run_button = gr.Button("2. Run Evaluation & Submit All Answers", variant="primary")
status_output = gr.Textbox(label="Submission result", lines=6)
results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
if __name__ == "__main__":
demo.launch()