File size: 4,678 Bytes
f41b338
c4b2bad
87db952
c4b2bad
0ba6cb0
f41b338
87db952
bb77645
c4b2bad
87db952
 
745fdfb
f41b338
 
290e845
f41b338
 
fb0445d
87db952
c4b2bad
fd72199
8db34db
 
87db952
c4b2bad
 
87db952
f41b338
 
 
 
 
 
 
 
 
 
87db952
c4b2bad
f41b338
 
 
87db952
 
 
 
 
 
 
 
 
 
 
 
f41b338
87db952
c4b2bad
f41b338
87db952
 
 
 
 
 
 
 
 
f41b338
87db952
 
 
f41b338
87db952
 
 
 
 
 
 
 
c4b2bad
0ba6cb0
c4b2bad
f41b338
c4b2bad
87db952
c4b2bad
 
87db952
 
c4b2bad
f41b338
 
 
87db952
 
 
 
f41b338
87db952
c4b2bad
87db952
c4b2bad
 
87db952
 
f41b338
87db952
 
 
 
 
 
f41b338
87db952
 
 
318d540
87db952
f41b338
cf3e905
 
f41b338
85218c0
87db952
 
 
 
 
 
 
f41b338
2ef6991
 
87db952
 
2ef6991
87db952
2ef6991
87db952
 
 
 
 
 
f41b338
87db952
 
 
 
 
 
 
 
 
 
 
 
0e7c9aa
2ef6991
 
f41b338
87db952
 
2ef6991
87db952
f41b338
 
 
 
 
 
 
 
 
87db952
 
 
 
 
 
 
 
 
 
 
 
2ef6991
 
 
87db952
2ef6991
 
87db952
2ef6991
318d540
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
import os
import requests
import pandas as pd
import gradio as gr
import spaces

from io import BytesIO
from smolagents import ToolCallingAgent, DuckDuckGoSearchTool, OpenAIServerModel

API_URL = "https://agents-course-unit4-scoring.hf.space"
AGENT_CODE_URL = "https://huggingface.co/spaces/bacchi87/Final_Assignment_Template/tree/main"

# Gemini model setup
model = OpenAIServerModel(
    model_id="gemini-3.5-flash",
    api_base="https://generativelanguage.googleapis.com/v1beta/openai/",
    api_key=os.environ["GEMINI_API_KEY"],
    max_tokens=1500
)

agent = ToolCallingAgent(
    tools=[DuckDuckGoSearchTool()],
    model=model,
    max_steps=6
)


def test_gemini():
    try:
        result = agent.run(
            "Reply with exactly GEMINI_OK. Do not use tools."
        )
        return f"Gemini connection successful!\nResponse: {result}"
    except Exception as e:
        return f"Gemini connection failed:\n{e}"


def get_attachment(task_id):
    try:
        response = requests.get(
            f"{API_URL}/files/{task_id}", timeout=30
        )
        response.raise_for_status()
        data = BytesIO(response.content)

        try:
            df = pd.read_excel(data)
            return df.to_string(index=False)
        except Exception:
            data.seek(0)
            try:
                df = pd.read_csv(data)
                return df.to_string(index=False)
            except Exception:
                return "Attachment format is not readable."

    except Exception as e:
        return f"Attachment download failed: {e}"


def answer_question(question, attachment=""):
    prompt = f"""
Answer this question accurately.

Question:
{question}

Attached file data:
{attachment}

Use web search or calculations when required.
Return ONLY the answer.
Do not write FINAL ANSWER.
"""
    try:
        result = agent.run(prompt)
        return str(result).strip()
    except Exception as e:
        return f"Agent error: {e}"


@spaces.GPU
def run_evaluation(profile: gr.OAuthProfile | None):

    if profile is None:
        return "Please login with Hugging Face.", []

    username = profile.username
    log = f"Logged in as: {username}\n"

    try:
        response = requests.get(
            f"{API_URL}/questions", timeout=30
        )
        response.raise_for_status()
        questions = response.json()

        if not questions:
            return "No questions returned.", []

    except Exception as e:
        return f"Could not fetch questions: {e}", []

    answers = []

    for i, item in enumerate(questions, start=1):

        task_id = str(item["task_id"])
        question = item["question"]

        log += f"\nQuestion {i}/{len(questions)}: {task_id}\n"

        attachment = ""

        if item.get("file_name"):
            attachment = get_attachment(task_id)

        answer = answer_question(question, attachment)

        # Stop if Gemini fails
        if answer.startswith("Agent error:"):
            log += f"Model inference failed: {answer}\n"
            return log + "\nNo answers submitted.", answers

        answers.append({
            "task_id": task_id,
            "submitted_answer": answer
        })

        log += f"Answer: {answer}\n"

    # Official scoring API
    payload = {
        "username": username,
        "agent_code": AGENT_CODE_URL,
        "answers": answers
    }

    try:
        result = requests.post(
            f"{API_URL}/submit",
            json=payload,
            timeout=60
        )
        result.raise_for_status()

        score_data = result.json()

        log += "\n--- EVALUATION RESULT ---\n"
        log += f"Score: {score_data.get('score')}%\n"
        log += f"Correct: {score_data.get('correct_count')}\n"
        log += f"Attempted: {score_data.get('total_attempted')}\n"
        log += f"Message: {score_data.get('message')}\n"

    except Exception as e:
        log += f"\nSubmission failed: {e}\n"

    return log, answers


with gr.Blocks() as demo:

    gr.Markdown("# Hugging Face Unit 4 - GAIA Evaluation")

    gr.LoginButton()

    test_btn = gr.Button("Test Gemini Connection")
    test_output = gr.Textbox(label="Gemini Test Result")

    test_btn.click(
        fn=test_gemini,
        inputs=[],
        outputs=[test_output]
    )

    run_btn = gr.Button(
        "Run Evaluation & Submit Answers",
        variant="primary"
    )

    status_box = gr.Textbox(
        label="Evaluation Status",
        lines=18
    )

    answers_box = gr.JSON(label="Generated Answers")

    run_btn.click(
        fn=run_evaluation,
        inputs=[],
        outputs=[status_box, answers_box]
    )


if __name__ == "__main__":
    demo.launch()