Wen-ChuangChou commited on
Commit
2a9c2fe
·
1 Parent(s): 4c8ada6

First pass version

Browse files
Files changed (3) hide show
  1. .gitignore +2 -0
  2. app.py +112 -40
  3. blablador.py +29 -0
.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ .env
2
+ *pycache*
app.py CHANGED
@@ -3,6 +3,9 @@ import gradio as gr
3
  import requests
4
  import inspect
5
  import pandas as pd
 
 
 
6
 
7
  # (Keep Constants as is)
8
  # --- Constants ---
@@ -10,25 +13,68 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
10
 
11
  # --- Basic Agent Definition ---
12
  # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
 
 
 
13
  class BasicAgent:
 
14
  def __init__(self):
15
- print("BasicAgent initialized.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  def __call__(self, question: str) -> str:
17
  print(f"Agent received question (first 50 chars): {question[:50]}...")
18
- fixed_answer = "This is a default answer."
19
- print(f"Agent returning fixed answer: {fixed_answer}")
20
- return fixed_answer
21
 
22
- def run_and_submit_all( profile: gr.OAuthProfile | None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  """
24
  Fetches all questions, runs the BasicAgent on them, submits all answers,
25
  and displays the results.
26
  """
27
  # --- Determine HF Space Runtime URL and Repo URL ---
28
- space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
 
29
 
30
  if profile:
31
- username= f"{profile.username}"
32
  print(f"User logged in: {username}")
33
  else:
34
  print("User not logged in.")
@@ -55,16 +101,16 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
55
  response.raise_for_status()
56
  questions_data = response.json()
57
  if not questions_data:
58
- print("Fetched questions list is empty.")
59
- return "Fetched questions list is empty or invalid format.", None
60
  print(f"Fetched {len(questions_data)} questions.")
61
  except requests.exceptions.RequestException as e:
62
  print(f"Error fetching questions: {e}")
63
  return f"Error fetching questions: {e}", None
64
  except requests.exceptions.JSONDecodeError as e:
65
- print(f"Error decoding JSON response from questions endpoint: {e}")
66
- print(f"Response text: {response.text[:500]}")
67
- return f"Error decoding server response for questions: {e}", None
68
  except Exception as e:
69
  print(f"An unexpected error occurred fetching questions: {e}")
70
  return f"An unexpected error occurred fetching questions: {e}", None
@@ -79,20 +125,41 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
79
  if not task_id or question_text is None:
80
  print(f"Skipping item with missing task_id or question: {item}")
81
  continue
 
 
 
 
 
82
  try:
83
  submitted_answer = agent(question_text)
84
- answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
85
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
 
 
 
 
 
 
 
86
  except Exception as e:
87
- print(f"Error running agent on task {task_id}: {e}")
88
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
 
 
 
 
89
 
90
  if not answers_payload:
91
  print("Agent did not produce any answers to submit.")
92
- return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
93
 
94
- # 4. Prepare Submission
95
- submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
 
 
 
 
96
  status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
97
  print(status_update)
98
 
@@ -107,8 +174,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
107
  f"User: {result_data.get('username')}\n"
108
  f"Overall Score: {result_data.get('score', 'N/A')}% "
109
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
110
- f"Message: {result_data.get('message', 'No message received.')}"
111
- )
112
  print("Submission successful.")
113
  results_df = pd.DataFrame(results_log)
114
  return final_status, results_df
@@ -143,8 +209,7 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
143
  # --- Build Gradio Interface using Blocks ---
144
  with gr.Blocks() as demo:
145
  gr.Markdown("# Basic Agent Evaluation Runner")
146
- gr.Markdown(
147
- """
148
  **Instructions:**
149
 
150
  1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
@@ -155,42 +220,49 @@ with gr.Blocks() as demo:
155
  **Disclaimers:**
156
  Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
157
  This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
158
- """
159
- )
160
 
161
  gr.LoginButton()
162
 
163
  run_button = gr.Button("Run Evaluation & Submit All Answers")
164
 
165
- status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
 
 
166
  # Removed max_rows=10 from DataFrame constructor
167
- results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
 
168
 
169
- run_button.click(
170
- fn=run_and_submit_all,
171
- outputs=[status_output, results_table]
172
- )
173
 
174
  if __name__ == "__main__":
175
- print("\n" + "-"*30 + " App Starting " + "-"*30)
176
  # Check for SPACE_HOST and SPACE_ID at startup for information
177
  space_host_startup = os.getenv("SPACE_HOST")
178
- space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
179
 
180
  if space_host_startup:
181
  print(f"✅ SPACE_HOST found: {space_host_startup}")
182
- print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
 
183
  else:
184
- print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
 
 
185
 
186
- if space_id_startup: # Print repo URLs if SPACE_ID is found
187
  print(f"✅ SPACE_ID found: {space_id_startup}")
188
  print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
189
- print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
 
 
190
  else:
191
- print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
 
 
192
 
193
- print("-"*(60 + len(" App Starting ")) + "\n")
194
 
195
  print("Launching Gradio Interface for Basic Agent Evaluation...")
196
- demo.launch(debug=True, share=False)
 
3
  import requests
4
  import inspect
5
  import pandas as pd
6
+ from dotenv import load_dotenv
7
+ from smolagents import DuckDuckGoSearchTool, OpenAIServerModel, CodeAgent, Tool
8
+ from blablador import Models
9
 
10
  # (Keep Constants as is)
11
  # --- Constants ---
 
13
 
14
  # --- Basic Agent Definition ---
15
  # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
16
+ load_dotenv()
17
+
18
+
19
  class BasicAgent:
20
+
21
  def __init__(self):
22
+ models = Models(api_key=os.getenv("Blablador_API_KEY")).get_model_ids()
23
+ model_id_blablador = 4
24
+ model_name = " ".join(
25
+ models[model_id_blablador].split(" - ")[1].split()[:2])
26
+ print("The agent uses the following model:", model_name)
27
+ answer_llm = OpenAIServerModel(
28
+ model_id=models[model_id_blablador],
29
+ api_base="https://helmholtz-blablador.fz-juelich.de:8000/v1",
30
+ api_key=os.getenv("Blablador_API_KEY"),
31
+ flatten_messages_as_text=True,
32
+ temperature=0.2)
33
+ self.agent = CodeAgent(
34
+ tools=[DuckDuckGoSearchTool()],
35
+ model=answer_llm,
36
+ planning_interval=3,
37
+ max_steps=10,
38
+ # verbosity_level=LogLevel.ERROR,
39
+ )
40
+
41
  def __call__(self, question: str) -> str:
42
  print(f"Agent received question (first 50 chars): {question[:50]}...")
 
 
 
43
 
44
+ SYSTEM_PROMPT = "You are a general AI assistant. I will ask you a question. " \
45
+ "Report your thoughts, and finish your answer with the following template: " \
46
+ "FINAL ANSWER: [YOUR FINAL ANSWER]. " \
47
+ "YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. " \
48
+ "If you are asked for a number, don't use comma to write your number neither use units " \
49
+ "such as $ or percent sign unless specified otherwise. " \
50
+ "If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), " \
51
+ "and write the digits in plain text unless specified otherwise. " \
52
+ "If you are asked for a comma separated list, " \
53
+ "apply the above rules depending of whether the element to be put in the list is a number or a string."
54
+
55
+ # Combine system prompt with the user question
56
+ full_prompt = f"{SYSTEM_PROMPT}\n\nQuestion: {question}"
57
+
58
+ try:
59
+ answer = self.agent.run(full_prompt)
60
+ print(f"Agent returning answer: {answer}")
61
+ return answer
62
+ except Exception as e:
63
+ print(f"Error running agent: {e}")
64
+ return f"Error: {e}"
65
+
66
+
67
+ def run_and_submit_all(profile: gr.OAuthProfile | None):
68
  """
69
  Fetches all questions, runs the BasicAgent on them, submits all answers,
70
  and displays the results.
71
  """
72
  # --- Determine HF Space Runtime URL and Repo URL ---
73
+ space_id = os.getenv(
74
+ "SPACE_ID") # Get the SPACE_ID for sending link to the code
75
 
76
  if profile:
77
+ username = f"{profile.username}"
78
  print(f"User logged in: {username}")
79
  else:
80
  print("User not logged in.")
 
101
  response.raise_for_status()
102
  questions_data = response.json()
103
  if not questions_data:
104
+ print("Fetched questions list is empty.")
105
+ return "Fetched questions list is empty or invalid format.", None
106
  print(f"Fetched {len(questions_data)} questions.")
107
  except requests.exceptions.RequestException as e:
108
  print(f"Error fetching questions: {e}")
109
  return f"Error fetching questions: {e}", None
110
  except requests.exceptions.JSONDecodeError as e:
111
+ print(f"Error decoding JSON response from questions endpoint: {e}")
112
+ print(f"Response text: {response.text[:500]}")
113
+ return f"Error decoding server response for questions: {e}", None
114
  except Exception as e:
115
  print(f"An unexpected error occurred fetching questions: {e}")
116
  return f"An unexpected error occurred fetching questions: {e}", None
 
125
  if not task_id or question_text is None:
126
  print(f"Skipping item with missing task_id or question: {item}")
127
  continue
128
+
129
+ file_name = item.get("file_name")
130
+ file_ext = None
131
+ file_url = None
132
+
133
  try:
134
  submitted_answer = agent(question_text)
135
+ answers_payload.append({
136
+ "task_id": task_id,
137
+ "submitted_answer": submitted_answer
138
+ })
139
+ results_log.append({
140
+ "Task ID": task_id,
141
+ "Question": question_text,
142
+ "Submitted Answer": submitted_answer
143
+ })
144
  except Exception as e:
145
+ print(f"Error running agent on task {task_id}: {e}")
146
+ results_log.append({
147
+ "Task ID": task_id,
148
+ "Question": question_text,
149
+ "Submitted Answer": f"AGENT ERROR: {e}"
150
+ })
151
 
152
  if not answers_payload:
153
  print("Agent did not produce any answers to submit.")
154
+ return "Agent did not produce any answers to submit.", pd.DataFrame(
155
+ results_log)
156
 
157
+ # 4. Prepare Submission
158
+ submission_data = {
159
+ "username": username.strip(),
160
+ "agent_code": agent_code,
161
+ "answers": answers_payload
162
+ }
163
  status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
164
  print(status_update)
165
 
 
174
  f"User: {result_data.get('username')}\n"
175
  f"Overall Score: {result_data.get('score', 'N/A')}% "
176
  f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
177
+ f"Message: {result_data.get('message', 'No message received.')}")
 
178
  print("Submission successful.")
179
  results_df = pd.DataFrame(results_log)
180
  return final_status, results_df
 
209
  # --- Build Gradio Interface using Blocks ---
210
  with gr.Blocks() as demo:
211
  gr.Markdown("# Basic Agent Evaluation Runner")
212
+ gr.Markdown("""
 
213
  **Instructions:**
214
 
215
  1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
 
220
  **Disclaimers:**
221
  Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
222
  This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
223
+ """)
 
224
 
225
  gr.LoginButton()
226
 
227
  run_button = gr.Button("Run Evaluation & Submit All Answers")
228
 
229
+ status_output = gr.Textbox(label="Run Status / Submission Result",
230
+ lines=5,
231
+ interactive=False)
232
  # Removed max_rows=10 from DataFrame constructor
233
+ results_table = gr.DataFrame(label="Questions and Agent Answers",
234
+ wrap=True)
235
 
236
+ run_button.click(fn=run_and_submit_all,
237
+ outputs=[status_output, results_table])
 
 
238
 
239
  if __name__ == "__main__":
240
+ print("\n" + "-" * 30 + " App Starting " + "-" * 30)
241
  # Check for SPACE_HOST and SPACE_ID at startup for information
242
  space_host_startup = os.getenv("SPACE_HOST")
243
+ space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
244
 
245
  if space_host_startup:
246
  print(f"✅ SPACE_HOST found: {space_host_startup}")
247
+ print(
248
+ f" Runtime URL should be: https://{space_host_startup}.hf.space")
249
  else:
250
+ print(
251
+ "ℹ️ SPACE_HOST environment variable not found (running locally?)."
252
+ )
253
 
254
+ if space_id_startup: # Print repo URLs if SPACE_ID is found
255
  print(f"✅ SPACE_ID found: {space_id_startup}")
256
  print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
257
+ print(
258
+ f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main"
259
+ )
260
  else:
261
+ print(
262
+ "ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined."
263
+ )
264
 
265
+ print("-" * (60 + len(" App Starting ")) + "\n")
266
 
267
  print("Launching Gradio Interface for Basic Agent Evaluation...")
268
+ demo.launch(debug=True, share=False)
blablador.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ import json
3
+
4
+
5
+ class Models():
6
+
7
+ def __init__(self, api_key):
8
+ self.api_key = api_key
9
+ self.headers = {
10
+ 'accept': 'application/json',
11
+ 'Authorization': f'Bearer {api_key}'
12
+ }
13
+
14
+ url = "https://helmholtz-blablador.fz-juelich.de:8000/v1/models"
15
+
16
+ def get_model_data(self):
17
+ response = requests.get(url=self.url, headers=self.headers)
18
+ response = json.loads(response.text)
19
+ return (response["data"])
20
+
21
+ def get_model_ids(self):
22
+ response = requests.get(url=self.url, headers=self.headers)
23
+ response = json.loads(response.text)
24
+
25
+ ids = []
26
+ for model in response["data"]:
27
+ ids.append(model["id"])
28
+
29
+ return (ids)