File size: 4,507 Bytes
10e9b7d
eccf8e4
80b5d6f
f0ef77f
75e4e99
 
9055341
 
 
4021bf3
f0ef77f
 
 
 
 
 
 
 
 
6fc204f
f0ef77f
 
 
 
 
 
 
80b5d6f
becfd78
fc40015
9b3a65b
604d1bf
056d2a5
604d1bf
511db96
44a94f6
80b5d6f
 
becfd78
80b5d6f
 
 
 
 
fc40015
3f25ad6
80b5d6f
 
1ccc2ae
80b5d6f
6fc204f
 
80b5d6f
6fc204f
fc40015
6fc204f
fc40015
 
 
80b5d6f
6fc204f
80b5d6f
fc40015
 
 
6fc204f
fc40015
80b5d6f
 
e80aab9
31243f4
fc40015
 
80b5d6f
 
 
31243f4
80b5d6f
 
 
 
 
 
 
 
 
31243f4
80b5d6f
 
 
 
 
 
f0ef77f
1ccc2ae
80b5d6f
 
 
 
 
 
 
fc40015
6fc204f
fc40015
 
 
 
 
 
 
80b5d6f
1ccc2ae
e80aab9
f0ef77f
e80aab9
80b5d6f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e80aab9
 
 
80b5d6f
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
import os
import requests
import gradio as gr
import spaces
from transformers import pipeline
from smolagents import CodeAgent, DuckDuckGoSearchTool, TransformersModel, tool
from smolagents import InferenceClientModel, CodeAgent
from huggingface_hub import InferenceClient


# 1. Custom tool to download files if needed for tasks
@tool
def get_file_content_tool(task_id: str) -> str:
    """
    Downloads files associated with a specific task ID from the course API.
    
    Args:
        task_id: The ID of the task requiring a file.
    """
    url = f"https://agents-course-unit4-scoring.hf.space/files/{task_id}"
    response = requests.get(url)
    if response.status_code == 200:
        filename = f"temp_{task_id}"
        with open(filename, "wb") as f:
            f.write(response.content)
        return f"File downloaded successfully as {filename}."
    return f"Failed to download file for task {task_id}"

# 2. Initialize model and agent
# 2. Initialize model and agent
# 2. Initialize model and agent with a free serverless-compatible model
# 2. Initialize model and agent with a serverless-supported model
model = InferenceClientModel(
    model_id="meta-llama/Llama-3.3-70B-Instruct", 
    token=os.getenv("HF_TOKEN")
)

agent = CodeAgent(
    tools=[DuckDuckGoSearchTool(), get_file_content_tool],
    model=model,
    max_steps=6,
    verbosity_level=1
)

# 3. Core evaluation function wrapped with @spaces.GPU
@spaces.GPU
def run_evaluation(username):
    if not username:
        return "Please log in with your Hugging Face account first."

    # Updated scoring API base URL
    api_base = "https://agents-course-unit4-scoring.hf.space"
    
    headers = {"User-Agent": "Mozilla/5.0"}
    try:
        resp = requests.get(f"{api_base}/questions", headers=headers, timeout=15)
    except Exception as e:
        return f"Connection error when reaching evaluation server: {str(e)}"
        
    if resp.status_code != 200:
        return f"Failed to fetch questions. Status code: {resp.status_code}. Response: {resp.text}"
    
    try:
        questions_data = resp.json()
    except Exception:
        return f"Invalid JSON response from server: {resp.text}"
        
    answers = []
    log_output = ""

    for item in questions_data:
        task_id = item.get("task_id")
        question_text = item.get("question")
        
        log_output += f"\n--- Processing Task ID: {task_id} ---\nQuestion: {question_text}\n"
        
        try:
            prompt = f"{question_text}\nProvide ONLY the direct, final answer string with no conversational padding."
            raw_answer = agent.run(prompt)
            clean_answer = str(raw_answer).strip()
            
            answers.append({
                "task_id": task_id,
                "submitted_answer": clean_answer
            })
            log_output += f"Answer: {clean_answer}\n"
        except Exception as e:
            log_output += f"Error executing task {task_id}: {str(e)}\n"
            answers.append({
                "task_id": task_id,
                "submitted_answer": "Error"
            })

    # 4. Submit results to the course API
    code_link = f"https://huggingface.co/spaces/{username}/Final_Assignment_Template/tree/main"
    
    payload = {
        "username": username,
        "agent_code": code_link,
        "answers": answers
    }
    
    try:
        sub_resp = requests.post(f"{api_base}/submit", json=payload, headers=headers, timeout=20)
        if sub_resp.status_code == 200:
            result_json = sub_resp.json()
            log_output += f"\nSubmission Successful! Response: {result_json}"
        else:
            log_output += f"\nSubmission Failed (Status {sub_resp.status_code}): {sub_resp.text}"
    except Exception as e:
        log_output += f"\nSubmission Exception: {str(e)}"
        
    return log_output

# 5. Build Gradio UI
with gr.Blocks() as demo:
    gr.Markdown("# Custom Agent Evaluation Runner")
    gr.Markdown("Click below to run your custom `smolagents` code against the Unit 4 benchmark questions and submit automatically.")
    
    with gr.Row():
        login_button = gr.LoginButton()
        
    with gr.Row():
        submit_btn = gr.Button("Run Evaluation & Submit All Answers")
        
    output_box = gr.Textbox(label="Run Status / Submission Result", lines=15)

    submit_btn.click(
        fn=run_evaluation,
        inputs=[login_button],
        outputs=[output_box]
    )

if __name__ == "__main__":
    demo.launch()