File size: 4,507 Bytes
10e9b7d eccf8e4 80b5d6f f0ef77f 75e4e99 9055341 4021bf3 f0ef77f 6fc204f f0ef77f 80b5d6f becfd78 fc40015 9b3a65b 604d1bf 056d2a5 604d1bf 511db96 44a94f6 80b5d6f becfd78 80b5d6f fc40015 3f25ad6 80b5d6f 1ccc2ae 80b5d6f 6fc204f 80b5d6f 6fc204f fc40015 6fc204f fc40015 80b5d6f 6fc204f 80b5d6f fc40015 6fc204f fc40015 80b5d6f e80aab9 31243f4 fc40015 80b5d6f 31243f4 80b5d6f 31243f4 80b5d6f f0ef77f 1ccc2ae 80b5d6f fc40015 6fc204f fc40015 80b5d6f 1ccc2ae e80aab9 f0ef77f e80aab9 80b5d6f e80aab9 80b5d6f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | import os
import requests
import gradio as gr
import spaces
from transformers import pipeline
from smolagents import CodeAgent, DuckDuckGoSearchTool, TransformersModel, tool
from smolagents import InferenceClientModel, CodeAgent
from huggingface_hub import InferenceClient
# 1. Custom tool to download files if needed for tasks
@tool
def get_file_content_tool(task_id: str) -> str:
"""
Downloads files associated with a specific task ID from the course API.
Args:
task_id: The ID of the task requiring a file.
"""
url = f"https://agents-course-unit4-scoring.hf.space/files/{task_id}"
response = requests.get(url)
if response.status_code == 200:
filename = f"temp_{task_id}"
with open(filename, "wb") as f:
f.write(response.content)
return f"File downloaded successfully as {filename}."
return f"Failed to download file for task {task_id}"
# 2. Initialize model and agent
# 2. Initialize model and agent
# 2. Initialize model and agent with a free serverless-compatible model
# 2. Initialize model and agent with a serverless-supported model
model = InferenceClientModel(
model_id="meta-llama/Llama-3.3-70B-Instruct",
token=os.getenv("HF_TOKEN")
)
agent = CodeAgent(
tools=[DuckDuckGoSearchTool(), get_file_content_tool],
model=model,
max_steps=6,
verbosity_level=1
)
# 3. Core evaluation function wrapped with @spaces.GPU
@spaces.GPU
def run_evaluation(username):
if not username:
return "Please log in with your Hugging Face account first."
# Updated scoring API base URL
api_base = "https://agents-course-unit4-scoring.hf.space"
headers = {"User-Agent": "Mozilla/5.0"}
try:
resp = requests.get(f"{api_base}/questions", headers=headers, timeout=15)
except Exception as e:
return f"Connection error when reaching evaluation server: {str(e)}"
if resp.status_code != 200:
return f"Failed to fetch questions. Status code: {resp.status_code}. Response: {resp.text}"
try:
questions_data = resp.json()
except Exception:
return f"Invalid JSON response from server: {resp.text}"
answers = []
log_output = ""
for item in questions_data:
task_id = item.get("task_id")
question_text = item.get("question")
log_output += f"\n--- Processing Task ID: {task_id} ---\nQuestion: {question_text}\n"
try:
prompt = f"{question_text}\nProvide ONLY the direct, final answer string with no conversational padding."
raw_answer = agent.run(prompt)
clean_answer = str(raw_answer).strip()
answers.append({
"task_id": task_id,
"submitted_answer": clean_answer
})
log_output += f"Answer: {clean_answer}\n"
except Exception as e:
log_output += f"Error executing task {task_id}: {str(e)}\n"
answers.append({
"task_id": task_id,
"submitted_answer": "Error"
})
# 4. Submit results to the course API
code_link = f"https://huggingface.co/spaces/{username}/Final_Assignment_Template/tree/main"
payload = {
"username": username,
"agent_code": code_link,
"answers": answers
}
try:
sub_resp = requests.post(f"{api_base}/submit", json=payload, headers=headers, timeout=20)
if sub_resp.status_code == 200:
result_json = sub_resp.json()
log_output += f"\nSubmission Successful! Response: {result_json}"
else:
log_output += f"\nSubmission Failed (Status {sub_resp.status_code}): {sub_resp.text}"
except Exception as e:
log_output += f"\nSubmission Exception: {str(e)}"
return log_output
# 5. Build Gradio UI
with gr.Blocks() as demo:
gr.Markdown("# Custom Agent Evaluation Runner")
gr.Markdown("Click below to run your custom `smolagents` code against the Unit 4 benchmark questions and submit automatically.")
with gr.Row():
login_button = gr.LoginButton()
with gr.Row():
submit_btn = gr.Button("Run Evaluation & Submit All Answers")
output_box = gr.Textbox(label="Run Status / Submission Result", lines=15)
submit_btn.click(
fn=run_evaluation,
inputs=[login_button],
outputs=[output_box]
)
if __name__ == "__main__":
demo.launch() |