File size: 8,435 Bytes
d438944
 
 
 
 
 
 
10e9b7d
d438944
 
 
10e9b7d
3c4371f
d438944
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10e9b7d
e80aab9
31243f4
 
d438944
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31243f4
d438944
 
 
 
 
31243f4
d438944
 
 
 
 
 
 
 
 
 
 
 
 
 
 
eccf8e4
d438944
 
7d65c66
d438944
 
 
 
 
 
 
 
 
31243f4
d438944
 
31243f4
 
d438944
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e80aab9
d438944
e80aab9
d438944
 
 
 
 
 
 
 
e80aab9
d438944
 
 
 
 
e80aab9
d438944
0ee0419
d438944
 
 
 
e80aab9
7e4a06b
d438944
 
 
e80aab9
d438944
 
7d65c66
d438944
e80aab9
 
d438944
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
"""Hugging Face Agents Course Unit 4 - GAIA evaluation Space.

Based on the public Final_Assignment_Template API contract. It is a starter
agent, not a guaranteed passing solution. Keep your Space public for scoring.
"""

import io
import os
import re
from pathlib import Path

import gradio as gr
import pandas as pd
import requests
from smolagents import (
    CodeAgent,
    DuckDuckGoSearchTool,
    InferenceClientModel,
    VisitWebpageTool,
)

API_URL = "https://agents-course-unit4-scoring.hf.space"
MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-Next-80B-A3B-Instruct")


def read_attachment(task_id: str, file_name: str) -> str:
    """Download an official task attachment and extract readable text."""
    if not file_name:
        return ""
    try:
        response = requests.get(f"{API_URL}/files/{task_id}", timeout=30)
        response.raise_for_status()
        if len(response.content) > 15_000_000:
            return "Attachment is too large to read with this starter."
        suffix = Path(file_name).suffix.lower()
        data = io.BytesIO(response.content)

        if suffix in {".txt", ".md", ".py", ".json", ".html", ".xml"}:
            contents = response.content.decode("utf-8", errors="replace")
        elif suffix == ".pdf":
            from pypdf import PdfReader
            pdf = PdfReader(data)
            contents = "\n".join(page.extract_text() or "" for page in pdf.pages[:40])
        elif suffix == ".csv":
            contents = pd.read_csv(data).head(100).to_string(index=False)
        elif suffix in {".xlsx", ".xls"}:
            sheets = pd.read_excel(data, sheet_name=None)
            contents = "\n".join(
                f"SHEET {name}:\n{frame.head(60).to_string(index=False)}"
                for name, frame in sheets.items()
            )
        elif suffix == ".docx":
            from docx import Document
            doc = Document(data)
            contents = "\n".join(p.text for p in doc.paragraphs)
        else:
            return (
                f"Attachment '{file_name}' is present, but this starter cannot "
                "interpret its media format. Do not guess its contents."
            )
        return f"Attachment ({file_name}):\n{contents[:14000]}"
    except Exception as exc:
        return f"Could not read attachment {file_name}: {type(exc).__name__}: {exc}"


class BasicAgent:
    def __init__(self):
        token = os.getenv("HF_TOKEN")
        if not token:
            raise RuntimeError(
                "HF_TOKEN Secret is missing. Add it in Space Settings "
                "-> Variables and secrets -> New secret."
            )
        model = InferenceClientModel(
            model_id=MODEL_ID,
            token=token,
            timeout=120,
            max_tokens=1800,
        )
        self.agent = CodeAgent(
            tools=[DuckDuckGoSearchTool(max_results=5), VisitWebpageTool()],
            model=model,
            max_steps=6,
            additional_authorized_imports=[
                "math", "statistics", "datetime", "re", "json", "itertools"
            ],
        )

    def __call__(self, question: str, attachment: str = "") -> str:
        prompt = (
            "You are solving a GAIA benchmark question. Use web search and "
            "calculations if needed. Check evidence and do not invent facts. "
            "Your final_answer must contain ONLY the shortest precise answer "
            "requested by the question. No explanation, citations, markdown, "
            "or the words FINAL ANSWER. Preserve requested units and format.\n\n"
            f"QUESTION:\n{question}\n\n"
            f"{attachment}"
        )
        answer = str(self.agent.run(prompt, reset=True)).strip()
        answer = re.sub(r"^\s*(?:final answer|answer)\s*:\s*", "", answer, flags=re.I)
        answer = re.sub(r"<think>.*?</think>", "", answer, flags=re.I | re.S).strip()
        return answer.strip("` \n")


def answer_task(agent: BasicAgent, item: dict) -> str:
    file_context = read_attachment(item["task_id"], item.get("file_name") or "")
    return agent(item["question"], file_context)


def test_one_question():
    """Try one official sample before running all 20 questions."""
    try:
        response = requests.get(f"{API_URL}/random-question", timeout=20)
        response.raise_for_status()
        item = response.json()
        if not isinstance(item, dict) or not item.get("task_id"):
            return "The sample API returned an unexpected response."
        agent = BasicAgent()
        answer = answer_task(agent, item)
        return f"Question: {item['question']}\n\nAgent answer: {answer}"
    except Exception as exc:
        return f"Test failed: {type(exc).__name__}: {exc}"


def run_and_submit_all(profile: gr.OAuthProfile | None):
    """Run the official GAIA question set and submit the generated answers."""
    if not profile:
        return "Please log in with the Hugging Face button first.", None
    space_id = os.getenv("SPACE_ID")
    if not space_id:
        return "SPACE_ID is missing. Run this code inside your own Hugging Face Space.", None
    username = profile.username

    try:
        agent = BasicAgent()
        response = requests.get(f"{API_URL}/questions", timeout=30)
        response.raise_for_status()
        questions = response.json()
        if not isinstance(questions, list) or not questions:
            return "The question API did not return a question list.", None
    except Exception as exc:
        return f"Initialization failed: {type(exc).__name__}: {exc}", None

    payload = []
    rows = []
    for index, item in enumerate(questions, 1):
        task_id = item.get("task_id")
        question = item.get("question")
        if not task_id or not question:
            continue
        try:
            answer = answer_task(agent, item)
            if not answer:
                raise ValueError("The agent returned an empty answer")
            payload.append({"task_id": task_id, "submitted_answer": answer})
            rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": answer})
            print(f"Task {index}/{len(questions)} completed")
        except Exception as exc:
            error = f"ERROR: {type(exc).__name__}: {exc}"
            print(f"Task {index}/{len(questions)} failed: {error}")
            rows.append({"Task ID": task_id, "Question": question, "Submitted Answer": error})

    table = pd.DataFrame(rows)
    if not payload:
        return "No answers generated. Check the Space logs and HF_TOKEN / model quota.", table

    request_data = {
        "username": username,
        "agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main",
        "answers": payload,
    }
    try:
        response = requests.post(f"{API_URL}/submit", json=request_data, timeout=90)
        response.raise_for_status()
        result = response.json()
        status = (
            "Submission Successful!\n"
            f"User: {result.get('username', username)}\n"
            f"Overall Score: {result.get('score', '?')}% "
            f"({result.get('correct_count', '?')}/{result.get('total_attempted', '?')} correct)\n"
            f"Submitted answers: {len(payload)}/{len(questions)}\n"
            f"Message: {result.get('message', '')}"
        )
        return status, table
    except Exception as exc:
        return f"Submission failed: {type(exc).__name__}: {exc}", table


with gr.Blocks() as demo:
    gr.Markdown("# Hugging Face Agents Course - Unit 4")
    gr.Markdown(
        "First configure the **HF_TOKEN** Secret in your Space Settings. "
        "Log in below for the leaderboard. Test one question before submitting all 20. "
        "This starter handles web search and text-like attachments; image/audio "
        "tasks need additional tools. Model calls may consume paid credits."
    )
    gr.LoginButton()
    test_button = gr.Button("1. Test one question")
    sample_output = gr.Textbox(label="Sample test result", lines=8)
    test_button.click(fn=test_one_question, outputs=sample_output)

    run_button = gr.Button("2. Run Evaluation & Submit All Answers", variant="primary")
    status_output = gr.Textbox(label="Submission result", lines=6)
    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
    run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])

if __name__ == "__main__":
    demo.launch()