File size: 8,435 Bytes
d438944 10e9b7d d438944 10e9b7d 3c4371f d438944 10e9b7d e80aab9 31243f4 d438944 31243f4 d438944 31243f4 d438944 eccf8e4 d438944 7d65c66 d438944 31243f4 d438944 31243f4 d438944 e80aab9 d438944 e80aab9 d438944 e80aab9 d438944 e80aab9 d438944 0ee0419 d438944 e80aab9 7e4a06b d438944 e80aab9 d438944 7d65c66 d438944 e80aab9 d438944 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 | """Hugging Face Agents Course Unit 4 - GAIA evaluation Space.
Based on the public Final_Assignment_Template API contract. It is a starter
agent, not a guaranteed passing solution. Keep your Space public for scoring.
"""
import io
import os
import re
from pathlib import Path
import gradio as gr
import pandas as pd
import requests
from smolagents import (
CodeAgent,
DuckDuckGoSearchTool,
InferenceClientModel,
VisitWebpageTool,
)
API_URL = "https://agents-course-unit4-scoring.hf.space"
MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-Next-80B-A3B-Instruct")
def read_attachment(task_id: str, file_name: str) -> str:
"""Download an official task attachment and extract readable text."""
if not file_name:
return ""
try:
response = requests.get(f"{API_URL}/files/{task_id}", timeout=30)
response.raise_for_status()
if len(response.content) > 15_000_000:
return "Attachment is too large to read with this starter."
suffix = Path(file_name).suffix.lower()
data = io.BytesIO(response.content)
if suffix in {".txt", ".md", ".py", ".json", ".html", ".xml"}:
contents = response.content.decode("utf-8", errors="replace")
elif suffix == ".pdf":
from pypdf import PdfReader
pdf = PdfReader(data)
contents = "\n".join(page.extract_text() or "" for page in pdf.pages[:40])
elif suffix == ".csv":
contents = pd.read_csv(data).head(100).to_string(index=False)
elif suffix in {".xlsx", ".xls"}:
sheets = pd.read_excel(data, sheet_name=None)
contents = "\n".join(
f"SHEET {name}:\n{frame.head(60).to_string(index=False)}"
for name, frame in sheets.items()
)
elif suffix == ".docx":
from docx import Document
doc = Document(data)
contents = "\n".join(p.text for p in doc.paragraphs)
else:
return (
f"Attachment '{file_name}' is present, but this starter cannot "
"interpret its media format. Do not guess its contents."
)
return f"Attachment ({file_name}):\n{contents[:14000]}"
except Exception as exc:
return f"Could not read attachment {file_name}: {type(exc).__name__}: {exc}"
class BasicAgent:
def __init__(self):
token = os.getenv("HF_TOKEN")
if not token:
raise RuntimeError(
"HF_TOKEN Secret is missing. Add it in Space Settings "
"-> Variables and secrets -> New secret."
)
model = InferenceClientModel(
model_id=MODEL_ID,
token=token,
timeout=120,
max_tokens=1800,
)
self.agent = CodeAgent(
tools=[DuckDuckGoSearchTool(max_results=5), VisitWebpageTool()],
model=model,
max_steps=6,
additional_authorized_imports=[
"math", "statistics", "datetime", "re", "json", "itertools"
],
)
def __call__(self, question: str, attachment: str = "") -> str:
prompt = (
"You are solving a GAIA benchmark question. Use web search and "
"calculations if needed. Check evidence and do not invent facts. "
"Your final_answer must contain ONLY the shortest precise answer "
"requested by the question. No explanation, citations, markdown, "
"or the words FINAL ANSWER. Preserve requested units and format.\n\n"
f"QUESTION:\n{question}\n\n"
f"{attachment}"
)
answer = str(self.agent.run(prompt, reset=True)).strip()
answer = re.sub(r"^\s*(?:final answer|answer)\s*:\s*", "", answer, flags=re.I)
answer = re.sub(r"<think>.*?</think>", "", answer, flags=re.I | re.S).strip()
return answer.strip("` \n")
def answer_task(agent: BasicAgent, item: dict) -> str:
file_context = read_attachment(item["task_id"], item.get("file_name") or "")
return agent(item["question"], file_context)
def test_one_question():
"""Try one official sample before running all 20 questions."""
try:
response = requests.get(f"{API_URL}/random-question", timeout=20)
response.raise_for_status()
item = response.json()
if not isinstance(item, dict) or not item.get("task_id"):
return "The sample API returned an unexpected response."
agent = BasicAgent()
answer = answer_task(agent, item)
return f"Question: {item['question']}\n\nAgent answer: {answer}"
except Exception as exc:
return f"Test failed: {type(exc).__name__}: {exc}"
def run_and_submit_all(profile: gr.OAuthProfile | None):
"""Run the official GAIA question set and submit the generated answers."""
if not profile:
return "Please log in with the Hugging Face button first.", None
space_id = os.getenv("SPACE_ID")
if not space_id:
return "SPACE_ID is missing. Run this code inside your own Hugging Face Space.", None
username = profile.username
try:
agent = BasicAgent()
response = requests.get(f"{API_URL}/questions", timeout=30)
response.raise_for_status()
questions = response.json()
if not isinstance(questions, list) or not questions:
return "The question API did not return a question list.", None
except Exception as exc:
return f"Initialization failed: {type(exc).__name__}: {exc}", None
payload = []
rows = []
for index, item in enumerate(questions, 1):
task_id = item.get("task_id")
question = item.get("question")
if not task_id or not question:
continue
try:
answer = answer_task(agent, item)
if not answer:
raise ValueError("The agent returned an empty answer")
payload.append({"task_id": task_id, "submitted_answer": answer})
rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": answer})
print(f"Task {index}/{len(questions)} completed")
except Exception as exc:
error = f"ERROR: {type(exc).__name__}: {exc}"
print(f"Task {index}/{len(questions)} failed: {error}")
rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": error})
table = pd.DataFrame(rows)
if not payload:
return "No answers generated. Check the Space logs and HF_TOKEN / model quota.", table
request_data = {
"username": username,
"agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main",
"answers": payload,
}
try:
response = requests.post(f"{API_URL}/submit", json=request_data, timeout=90)
response.raise_for_status()
result = response.json()
status = (
"Submission Successful!\n"
f"User: {result.get('username', username)}\n"
f"Overall Score: {result.get('score', '?')}% "
f"({result.get('correct_count', '?')}/{result.get('total_attempted', '?')} correct)\n"
f"Submitted answers: {len(payload)}/{len(questions)}\n"
f"Message: {result.get('message', '')}"
)
return status, table
except Exception as exc:
return f"Submission failed: {type(exc).__name__}: {exc}", table
with gr.Blocks() as demo:
gr.Markdown("# Hugging Face Agents Course - Unit 4")
gr.Markdown(
"First configure the **HF_TOKEN** Secret in your Space Settings. "
"Log in below for the leaderboard. Test one question before submitting all 20. "
"This starter handles web search and text-like attachments; image/audio "
"tasks need additional tools. Model calls may consume paid credits."
)
gr.LoginButton()
test_button = gr.Button("1. Test one question")
sample_output = gr.Textbox(label="Sample test result", lines=8)
test_button.click(fn=test_one_question, outputs=sample_output)
run_button = gr.Button("2. Run Evaluation & Submit All Answers", variant="primary")
status_output = gr.Textbox(label="Submission result", lines=6)
results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
if __name__ == "__main__":
demo.launch()
|