Kackle commited on
Commit
21b42b0
·
verified ·
1 Parent(s): d8ea6bf

Update gemini_agent.py

Browse files
Files changed (1) hide show
  1. gemini_agent.py +399 -294
gemini_agent.py CHANGED
@@ -1,327 +1,432 @@
1
  import os
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  import google.generativeai as genai
 
3
  from dotenv import load_dotenv
4
- from excel_parser import ExcelParser
5
- import re
6
- import time
7
- import asyncio
8
- # Add LangChain tools for Wikipedia and DuckDuckGo
9
- from langchain.tools import DuckDuckGoSearchRun, WikipediaQueryRun
10
- from langchain.utilities import WikipediaAPIWrapper
11
 
12
  load_dotenv()
 
 
 
13
 
14
- class GeminiAgent:
 
 
 
 
 
 
 
 
 
15
  def __init__(self):
16
- print("GeminiAgent initialized.")
17
-
18
- # Get Google API key from environment variables
19
- api_key = os.getenv('GOOGLE_API_KEY')
20
- genai.configure(api_key=api_key)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
 
22
- self.model = genai.GenerativeModel('gemini-2.0-flash-exp')
23
- self.last_request_time = 0
24
- self.min_request_interval = 6.0 # 6 seconds between requests (10 per minute limit)
 
 
 
 
 
 
 
 
 
25
 
26
- # Initialize parsers
27
- self.excel_parser = ExcelParser()
28
- # Initialize Wikipedia and DuckDuckGo tools
29
- self.wiki_tool = WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper())
30
- self.ddg_tool = DuckDuckGoSearchRun()
 
 
31
 
32
  async def __call__(self, question: str) -> str:
33
- print(f"GeminiAgent received question (first 50 chars): {question}...")
 
 
34
 
35
- try:
36
- # Check if question involves video analysis
37
- if 'youtube.com' in question or 'video' in question.lower():
38
- return await self._handle_video_question(question)
39
-
40
- # Check if question involves Excel files
41
- if '.xlsx' in question or '.xls' in question or 'excel' in question.lower():
42
- return await self._handle_excel_question(question)
43
-
44
- # Regular text-based question
45
- return await self._handle_text_question(question)
46
-
47
- except Exception as e:
48
- print(f"Error processing question: {e}")
49
- return "Unable to process request."
50
-
51
- async def _handle_video_question(self, question: str) -> str:
52
- """Handle questions that require video analysis"""
53
- # Extract YouTube URL
54
- youtube_url = re.search(r'https://www\.youtube\.com/watch\?v=[\w-]+', question)
55
- if not youtube_url:
56
- return "No valid YouTube URL found in question."
57
 
58
- url = youtube_url.group()
 
 
59
 
60
- # Extract video ID for reference
61
- video_id = re.search(r'v=([\w-]+)', url).group(1)
62
 
63
- # Extract video information from the question to provide relevant answers
64
- # without hardcoding specific IDs
 
 
 
 
 
 
 
 
65
 
66
- # Enhanced video prompt for better accuracy
67
- video_prompt = f"""You need to answer this question about YouTube video {url}:
68
-
69
- {question}
 
 
 
 
 
70
 
71
- Provide only the direct answer. If it's a quote, give just the quoted text. If it's a number, give just the number. If it's about bird species count, analyze carefully and give the exact count. If it's about dialogue, provide the exact words spoken."""
72
-
73
- try:
74
- await self._rate_limit()
75
- response = self.model.generate_content(
76
- video_prompt,
77
- generation_config=genai.types.GenerationConfig(
78
- max_output_tokens=50,
79
- temperature=0.0
80
- )
81
- )
82
- answer = response.text.strip()
83
-
84
- # Clean up video responses to be more concise
85
- if len(answer) > 100:
86
- # Extract key information
87
- if '"' in answer:
88
- # Extract quoted text
89
- quotes = re.findall(r'"([^"]+)"', answer)
90
- if quotes:
91
- return quotes[0]
92
- # Extract numbers if it's a counting question
93
- if 'how many' in question.lower() or 'number' in question.lower():
94
- numbers = re.findall(r'\b\d+\b', answer)
95
- if numbers:
96
- return numbers[0]
97
- # Take first sentence
98
- sentences = answer.split('. ')
99
- answer = sentences[0]
100
-
101
- return answer
102
-
103
- except Exception as e:
104
- print(f"Video analysis failed: {str(e)}")
105
- # Generate answer based on question content
106
- return await self._generate_video_answer_from_question(question, video_id)
107
-
108
- async def _handle_excel_question(self, question: str) -> str:
109
- """Handle questions that require Excel file analysis"""
110
- # Extract file path from question if present
111
- file_patterns = [r'([A-Za-z]:\\[^\s]+\.xlsx?)', r'([^\s]+\.xlsx?)']
112
- file_path = None
113
 
114
- for pattern in file_patterns:
115
- match = re.search(pattern, question)
116
- if match:
117
- file_path = match.group(1)
118
- break
119
 
120
- # If we have a file path, try to process it
121
- if file_path:
122
  try:
123
- if 'sales' in question.lower() and 'food' in question.lower():
124
- results = self.excel_parser.analyze_sales_data(file_path)
125
- return results.get('total_food_sales', 'No sales data found')
126
- else:
127
- df = self.excel_parser.read_excel_file(file_path)
128
- return f"Excel file loaded with {len(df)} rows and {len(df.columns)} columns."
 
 
 
 
 
 
 
 
 
 
 
 
 
129
  except Exception as e:
130
- print(f"Excel analysis failed: {str(e)}")
131
- # Fall through to Nova Pro search
 
 
 
 
 
 
 
 
 
132
 
133
- # Use Nova Pro to search for information about the Excel file
134
- excel_prompt = f"""I need to analyze an Excel file mentioned in this question, but I don't have direct access to it.
135
- Based on your knowledge, provide the most accurate answer possible:
136
-
137
- {question}
138
-
139
- If you don't have specific information about this Excel file, provide a reasonable estimate based on similar data."""
140
 
141
- try:
142
- await self._rate_limit()
143
- response = self.model.generate_content(
144
- excel_prompt,
145
- generation_config=genai.types.GenerationConfig(
146
- max_output_tokens=150,
147
- temperature=0.0
148
- )
149
- )
150
- answer = response.text.strip()
151
 
152
- # Check if the answer contains a dollar amount
153
- dollar_match = re.search(r'\$[\d,]+\.\d{2}', answer)
154
- if dollar_match:
155
- return dollar_match.group(0)
156
- else:
157
- return answer
158
-
159
- except Exception as e:
160
- print(f"Gemini search failed: {str(e)}")
161
- return "Unable to analyze Excel data. Please provide the file directly."
162
 
163
- async def _handle_text_question(self, question: str) -> str:
164
- """Handle regular text-based questions"""
165
- prompt = ""
166
- # Only use retrieval for explicit web/Wikipedia questions
167
- def is_explicit_retrieval_question(question):
168
- q = question.lower()
169
- return (
170
- "according to wikipedia" in q or
171
- "from wikipedia" in q or
172
- "search the web" in q or
173
- "duckduckgo" in q or
174
- "web search" in q
175
- )
176
- wiki_context = ""
177
- ddg_context = ""
178
- if is_explicit_retrieval_question(question):
179
- if "wikipedia" in question.lower():
180
- try:
181
- wiki_context = self.wiki_tool.run(question)
182
- except Exception as e:
183
- print(f"Wikipedia tool failed: {e}")
184
- if "duckduckgo" in question.lower() or "web search" in question.lower():
185
- try:
186
- ddg_context = self.ddg_tool.run(question)
187
- except Exception as e:
188
- print(f"DuckDuckGo tool failed: {e}")
189
- # Handle attached file questions with enhanced prompts
190
- if 'attached' in question.lower():
191
- if 'python code' in question.lower():
192
- prompt = f"""This question refers to attached Python code. Based on typical code execution patterns, provide the most likely numeric output:\n\n{question}\n\nAnswer:"""
193
- elif '.mp3' in question.lower():
194
- prompt = f"""This question refers to an attached audio file. Provide the most likely answer based on the context:\n\n{question}\n\nAnswer:"""
195
- else:
196
- prompt = f"""This question refers to an attached file. Provide the most likely answer:\n\n{question}\n\nAnswer:"""
197
- # Handle chess position question
198
- elif 'chess position' in question.lower() and 'image' in question.lower():
199
- prompt = f"""This is a chess question with an attached image. Provide the best chess move in algebraic notation:\n\n{question}\n\nAnswer:"""
200
- # Handle list extraction and formatting
201
- elif (
202
- 'alphabetize' in question.lower() or
203
- 'comma separated' in question.lower() or
204
- 'list' in question.lower() or
205
- 'ingredients' in question.lower() or
206
- 'page numbers' in question.lower() or
207
- 'vegetables' in question.lower()
208
- ):
209
- # Add domain definition for botanical vegetables
210
- if 'vegetable' in question.lower() and ('botany' in question.lower() or 'botanical' in question.lower()):
211
- definition = ("In botany, a vegetable is any edible part of a plant that is not a fruit or seed. "
212
- "Fruits contain seeds and develop from the ovary of a flower. Use this definition.")
213
- prompt = f"{definition}\n\n{question}\n\nList only the requested items, alphabetized, comma separated, and do not include any explanations or extra words."
214
- else:
215
- prompt = f"{question}\n\nList only the requested items, alphabetized, comma separated, and do not include any explanations or extra words."
216
- # Create enhanced prompt based on question type
217
- elif 'how many' in question.lower() or 'what is the' in question.lower():
218
- prompt = f"""Provide only the exact answer to this question. No explanations, just the specific number, name, or fact requested:\n\n{question}\n\nAnswer:"""
219
- elif 'who' in question.lower():
220
- prompt = f"""Provide only the name requested. No explanations or additional context:\n\n{question}\n\nAnswer:"""
221
- elif 'where' in question.lower():
222
- prompt = f"""Provide only the location requested. No explanations:\n\n{question}\n\nAnswer:"""
223
  else:
224
- prompt = f"""Answer this question with only the essential information requested:\n\n{question}\n\nAnswer:"""
225
-
226
- # Prepend context to the prompt if available and likely relevant
227
- def is_good_context(context):
228
- return context and not any(x in context.lower() for x in ["not found", "no results", "does not contain information"])
229
- if wiki_context and is_good_context(wiki_context):
230
- prompt = f"Use the following Wikipedia context to answer the question:\n{wiki_context}\n\n{prompt}"
231
- elif ddg_context and is_good_context(ddg_context):
232
- prompt = f"Use the following web search context to answer the question:\n{ddg_context}\n\n{prompt}"
233
-
234
- # Use the constructed prompt for all cases
235
- await self._rate_limit()
236
- response = self.model.generate_content(
237
- prompt,
238
- generation_config=genai.types.GenerationConfig(
239
- max_output_tokens=100,
240
- temperature=0.0
241
- )
242
- )
243
- answer = response.text.strip()
244
-
245
- # Extract the core answer
246
- if ':' in answer:
247
- answer = answer.split(':')[-1].strip()
248
-
249
- # Remove common prefixes
250
- prefixes = ['The answer is', 'Based on', 'According to']
251
- for prefix in prefixes:
252
- if answer.lower().startswith(prefix.lower()):
253
- answer = answer[len(prefix):].strip()
254
- if answer.startswith(','):
255
- answer = answer[1:].strip()
256
-
257
- # Limit length
258
- if len(answer) > 200:
259
- sentences = answer.split('. ')
260
- answer = sentences[0] + '.'
261
-
262
- # If the question expects a single value, extract it
263
- if any(kw in question.lower() for kw in ["how many", "what is the", "who", "where", "give only", "provide only"]):
264
- # Extract the first number, word, or phrase (tweak regex as needed)
265
- match = re.search(r'^[A-Za-z0-9 ,+-]+', answer)
266
- if match:
267
- answer = match.group(0).strip()
268
-
269
- # Post-processing for chess move extraction
270
- if 'chess position' in question.lower() and 'image' in question.lower():
271
- move_match = re.search(r'([KQRBN]?[a-h]?[1-8]?x?[a-h][1-8](=[QRBN])?[+#]?)', answer)
272
- if move_match:
273
- answer = move_match.group(1)
274
 
275
- # Post-processing for sorted, deduplicated lists
276
- if 'page numbers' in question.lower() or 'comma-delimited list' in question.lower():
277
- # Extract numbers, deduplicate, sort, and join
278
- nums = re.findall(r'\d+', answer)
279
- nums = sorted(set(int(n) for n in nums))
280
- answer = ', '.join(str(n) for n in nums)
281
- elif 'alphabetize' in question.lower() or 'alphabetized' in question.lower() or 'ingredients' in question.lower() or 'vegetables' in question.lower():
282
- # Extract words/phrases, deduplicate, sort, and join
283
- items = [item.strip() for item in answer.split(',') if item.strip()]
284
- items = sorted(set(items), key=lambda x: x.lower())
285
- answer = ', '.join(items)
286
 
287
- return answer
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
288
 
289
- async def _generate_video_answer_from_question(self, question: str, video_id: str) -> str:
290
- """Generate an answer for a video question based on the question content"""
291
- # Create a prompt that asks Nova Pro to analyze the question and generate a likely answer
292
- prompt = f"""Based on this question about YouTube video ID {video_id},
293
- what would be the most likely accurate answer? The question is:
294
-
295
- {question}
296
-
297
- Provide only the direct answer without explanation."""
298
 
299
- try:
300
- await self._rate_limit()
301
- response = self.model.generate_content(
302
- prompt,
303
- generation_config=genai.types.GenerationConfig(
304
- max_output_tokens=100,
305
- temperature=0.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
306
  )
307
- )
308
- answer = response.text.strip()
309
-
310
- # Clean up the answer to make it concise
311
- if len(answer) > 100:
312
- sentences = answer.split('. ')
313
- answer = sentences[0]
314
-
315
- return answer
316
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  except Exception as e:
318
- print(f"Failed to generate video answer: {str(e)}")
319
- return "Video analysis unavailable."
320
 
321
- async def _rate_limit(self):
322
- """Ensure minimum time between API requests"""
323
- current_time = time.time()
324
- time_since_last = current_time - self.last_request_time
325
- if time_since_last < self.min_request_interval:
326
- await asyncio.sleep(self.min_request_interval - time_since_last)
327
- self.last_request_time = time.time()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
+ import gradio as gr
3
+ import requests
4
+ import inspect
5
+ import pandas as pd
6
+ import asyncio
7
+ import aiohttp
8
+ import time
9
+ import random
10
+ import json
11
+ import boto3
12
+ from smolagents import FinalAnswerTool, Tool, tool, OpenAIServerModel, DuckDuckGoSearchTool, CodeAgent, VisitWebpageTool
13
+ from nova_agent import NovaProAgent
14
+ from gemini_agent import GeminiAgent
15
+
16
  import google.generativeai as genai
17
+
18
  from dotenv import load_dotenv
 
 
 
 
 
 
 
19
 
20
  load_dotenv()
21
+ # (Keep Constants as is)
22
+ # --- Constants ---
23
+ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
24
 
25
+
26
+ OPENAI_TOKEN = os.getenv("OPENAI_API_KEY")
27
+
28
+ # --- Custom Tools ---
29
+ class KnowledgeBaseTool(Tool):
30
+ name = "knowledge_base"
31
+ description = "Access structured knowledge for common topics"
32
+ inputs = {"topic": {"type": "string", "description": "The topic to look up"}}
33
+ output_type = "string"
34
+
35
  def __init__(self):
36
+ super().__init__()
37
+ self.is_initialized = True
38
+ # Common knowledge base
39
+ self.knowledge = {
40
+ "olympics": "Olympic Games data: Countries, athletes, years, sports",
41
+ "countries": "Country codes: ISO, IOC, FIFA codes and country information",
42
+ "sports": "Sports history, rules, famous athletes and events",
43
+ "science": "Scientific facts, formulas, discoveries, and researchers",
44
+ "history": "Historical events, dates, people, and places",
45
+ "geography": "Countries, capitals, populations, and geographical features"
46
+ }
47
+
48
+ def forward(self, topic: str) -> str:
49
+ topic_lower = topic.lower()
50
+ for key, info in self.knowledge.items():
51
+ if key in topic_lower:
52
+ return f"Knowledge base: {info}. Use this context to answer questions about {topic}."
53
+ return f"No specific knowledge base entry for '{topic}'. Use general reasoning."
54
+
55
+ class WikipediaSearchTool(Tool):
56
+ name = "wikipedia_search"
57
+ description = "Search Wikipedia for information"
58
+ inputs = {"query": {"type": "string", "description": "The search query for Wikipedia"}}
59
+ output_type = "string"
60
+
61
+ def __init__(self):
62
+ super().__init__()
63
+ self.is_initialized = True
64
 
65
+ def forward(self, query: str) -> str:
66
+ """Search Wikipedia with simple fallback."""
67
+ try:
68
+ import requests
69
+ wiki_url = "https://en.wikipedia.org/api/rest_v1/page/summary/" + query.replace(" ", "_")
70
+ response = requests.get(wiki_url, timeout=2)
71
+ if response.status_code == 200:
72
+ data = response.json()
73
+ if 'extract' in data and data['extract']:
74
+ return f"Wikipedia: {data['extract'][:500]}" # Limit length
75
+ except Exception as e:
76
+ print(f"Wikipedia search failed: {e}")
77
 
78
+ return f"Wikipedia search unavailable for '{query}'. Use your knowledge to answer."
79
+
80
+ # --- Basic Agent Definition ---
81
+ # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
82
+ class SlpMultiAgent:
83
+ def __init__(self):
84
+ print("BasicAgent initialized.")
85
 
86
  async def __call__(self, question: str) -> str:
87
+ print(f"Agent received question (first 50 chars): {question}...")
88
+ fixed_answer = "This is a default answer."
89
+ print(f"Agent returning fixed answer: {fixed_answer}")
90
 
91
+ # Truncate question to avoid exceeding model context length
92
+ MAX_QUESTION_LENGTH = 1000
93
+ short_question = question # [:MAX_QUESTION_LENGTH]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
 
95
+ # Use cheaper, faster model
96
+ api_key = os.getenv('GOOGLE_API_KEY')
97
+ genai.configure(api_key=api_key)
98
 
99
+ model = genai.GenerativeModel('gemini-2.0-flash-exp')
 
100
 
101
+ # Create only essential agents with reduced complexity
102
+ research_agent = CodeAgent(
103
+ tools=[KnowledgeBaseTool()], # Remove Wikipedia to avoid timeouts
104
+ model=model,
105
+ additional_authorized_imports=["re", "datetime"],
106
+ max_steps=2, # Reduced steps for cost
107
+ name="ResearchAgent",
108
+ verbosity_level=0,
109
+ description="Quick factual research and knowledge lookup."
110
+ )
111
 
112
+ solver_agent = CodeAgent(
113
+ tools=[],
114
+ model=model,
115
+ additional_authorized_imports=["math", "re", "collections", "itertools"],
116
+ max_steps=2, # Reduced steps
117
+ name="SolverAgent",
118
+ verbosity_level=0,
119
+ description="Problem solving, calculations, and logical reasoning."
120
+ )
121
 
122
+ manager_agent = CodeAgent(
123
+ model=OpenAIServerModel(
124
+ model_id="gpt-3.5-turbo",
125
+ temperature=0.0,
126
+ max_tokens=500
127
+ ),
128
+ tools=[KnowledgeBaseTool()], # Remove Wikipedia to avoid timeouts
129
+ managed_agents=[research_agent, solver_agent], # Only 2 agents
130
+ name="ManagerAgent",
131
+ description="Efficient manager for quick problem solving.",
132
+ additional_authorized_imports=["re", "math"],
133
+ planning_interval=1, # Faster planning
134
+ verbosity_level=0, # Reduce verbosity
135
+ max_steps=3, # Further reduced steps to avoid timeouts
136
+ final_answer_checks=[check_reasoning]
137
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
 
139
+ # Create a task for the agent run with retry mechanism for rate limits
140
+ max_retries = 3
141
+ result = None
 
 
142
 
143
+ for attempt in range(max_retries):
 
144
  try:
145
+ loop = asyncio.get_event_loop()
146
+ result = await loop.run_in_executor(
147
+ None,
148
+ lambda: manager_agent.run(f"""
149
+ Question: {short_question}
150
+
151
+ You have knowledge_base() tool and two agents:
152
+ - ResearchAgent: For factual questions
153
+ - SolverAgent: For calculations and logic
154
+
155
+ IMPORTANT: Always end with exactly this format:
156
+ <code>
157
+ final_answer("your direct answer")
158
+ </code>
159
+
160
+ Be concise and direct.
161
+ """)
162
+ )
163
+ break # Success, exit retry loop
164
  except Exception as e:
165
+ print(f"Attempt {attempt+1}/{max_retries} failed: {e}")
166
+ if "rate limit" in str(e).lower() and attempt < max_retries - 1:
167
+ # Add jitter to avoid synchronized retries
168
+ wait_time = (attempt + 1) * 10 + random.uniform(0, 5)
169
+ print(f"Rate limit hit. Waiting {wait_time:.2f} seconds before retry...")
170
+ await asyncio.sleep(wait_time)
171
+ elif attempt < max_retries - 1:
172
+ await asyncio.sleep(5) # Wait before general retry
173
+ else:
174
+ print(f"All attempts failed. Returning default answer.")
175
+ return "I apologize, but I'm currently experiencing technical difficulties. Please try again later."
176
 
177
+ # If we couldn't get a result after all retries
178
+ if result is None:
179
+ return "I apologize, but I'm currently experiencing technical difficulties. Please try again later."
 
 
 
 
180
 
181
+
182
+ # Extract clean answer from result
183
+ if result and isinstance(result, str):
184
+ # Look for final_answer pattern
185
+ import re
186
+ final_answer_match = re.search(r'final_answer\(["\']([^"\']*)["\'\)]', result) # Fixed regex
187
+ if final_answer_match:
188
+ clean_answer = final_answer_match.group(1)
189
+ return clean_answer
 
190
 
191
+ # If no final_answer found, try to extract the last meaningful line
192
+ lines = result.strip().split('\n')
193
+ for line in reversed(lines):
194
+ line = line.strip()
195
+ if line and not line.startswith('#') and not line.startswith('###') and len(line) < 200:
196
+ return line
197
+
198
+ # Return the result from the agent
199
+ return result if result else "Unable to determine answer."
 
200
 
201
+ def check_reasoning(final_answer, agent_memory):
202
+ # Skip expensive validation to save costs
203
+ return True
204
+
205
+
206
+ async def run_and_submit_all(profile):
207
+ """
208
+ Fetches all questions, runs the BasicAgent on them, submits all answers,
209
+ and displays the results asynchronously.
210
+ """
211
+ # --- Determine HF Space Runtime URL and Repo URL ---
212
+ space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
213
+
214
+ # Handle different profile types
215
+ if profile:
216
+ if hasattr(profile, 'username'):
217
+ # It's an OAuthProfile object
218
+ username = profile.username
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
219
  else:
220
+ # It's a string or other type
221
+ username = str(profile)
222
+ print(f"User logged in: {username}")
223
+ else:
224
+ print("User not logged in.")
225
+ return "Please Login to Hugging Face with the button.", None
226
+
227
+ api_url = DEFAULT_API_URL
228
+ questions_url = f"{api_url}/questions"
229
+ submit_url = f"{api_url}/submit"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
230
 
231
+ # 1. Instantiate Agent ( modify this part to create your agent)
232
+ try:
233
+ agent = SlpMultiAgent()
234
+ except Exception as e:
235
+ print(f"Error instantiating agent: {e}")
236
+ return f"Error initializing agent: {e}", None
237
+ # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
238
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
239
+ print(agent_code)
 
 
240
 
241
+ # 2. Fetch Questions
242
+ print(f"Fetching questions from: {questions_url}")
243
+ try:
244
+ async with aiohttp.ClientSession() as session:
245
+ async with session.get(questions_url, timeout=15) as response:
246
+ response.raise_for_status()
247
+ questions_data = await response.json()
248
+ if not questions_data:
249
+ print("Fetched questions list is empty.")
250
+ return "Fetched questions list is empty or invalid format.", None
251
+ print(f"Fetched {len(questions_data)} questions.")
252
+ except aiohttp.ClientError as e:
253
+ print(f"Error fetching questions: {e}")
254
+ return f"Error fetching questions: {e}", None
255
+ except ValueError as e: # JSON decode error
256
+ print(f"Error decoding JSON response from questions endpoint: {e}")
257
+ return f"Error decoding server response for questions: {e}", None
258
+ except Exception as e:
259
+ print(f"An unexpected error occurred fetching questions: {e}")
260
+ return f"An unexpected error occurred fetching questions: {e}", None
261
+
262
+ # 3. Run your Agent
263
+ results_log = []
264
+ answers_payload = []
265
+ print(f"Running agent on {len(questions_data)} questions...")
266
 
267
+ # Process questions one at a time to avoid rate limits
268
+ semaphore = asyncio.Semaphore(1) # Process 1 question at a time
269
+
270
+ async def process_question(item):
271
+ task_id = item.get("task_id")
272
+ question_text = item.get("question")
273
+ if not task_id or question_text is None:
274
+ print(f"Skipping item with missing task_id or question: {item}")
275
+ return None
276
 
277
+ async with semaphore:
278
+ max_retries = 3
279
+ for attempt in range(max_retries):
280
+ try:
281
+ print(f"Processing task {task_id}, attempt {attempt+1}/{max_retries}")
282
+ submitted_answer = await agent(question_text)
283
+ return {"task_id": task_id, "submitted_answer": submitted_answer,
284
+ "log": {"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer}}
285
+ except Exception as e:
286
+ print(f"Error running agent on task {task_id}, attempt {attempt+1}: {e}")
287
+ if "rate limit" in str(e).lower() and attempt < max_retries - 1:
288
+ # Exponential backoff with jitter
289
+ wait_time = (2 ** attempt) * 5 + random.uniform(0, 3)
290
+ print(f"Rate limit hit. Waiting {wait_time:.2f} seconds before retry...")
291
+ await asyncio.sleep(wait_time)
292
+ elif attempt < max_retries - 1:
293
+ await asyncio.sleep(5) # Reduced wait time
294
+ else:
295
+ # All retries failed, return default answer
296
+ default_answer = "This is a default answer."
297
+ return {"task_id": task_id, "submitted_answer": default_answer,
298
+ "log": {"Task ID": task_id, "Question": question_text, "Submitted Answer": default_answer}}
299
+
300
+ # Create tasks for all questions
301
+ tasks = [process_question(item) for item in questions_data]
302
+ results = await asyncio.gather(*tasks)
303
+
304
+ # Process results
305
+ for result in results:
306
+ if result is not None:
307
+ answers_payload.append({"task_id": result["task_id"], "submitted_answer": result["submitted_answer"]})
308
+ results_log.append(result["log"])
309
+
310
+ if not answers_payload:
311
+ print("Agent did not produce any answers to submit.")
312
+ return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
313
+
314
+ # 4. Prepare Submission
315
+ submission_data = {"username": str(username).strip(), "agent_code": agent_code, "answers": answers_payload}
316
+ status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
317
+ print(status_update)
318
+
319
+ # 5. Submit
320
+ print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
321
+ try:
322
+ async with aiohttp.ClientSession() as session:
323
+ async with session.post(submit_url, json=submission_data, timeout=60) as response:
324
+ response.raise_for_status()
325
+ result_data = await response.json()
326
+ final_status = (
327
+ f"Submission Successful!\n"
328
+ f"User: {result_data.get('username')}\n"
329
+ f"Overall Score: {result_data.get('score', 'N/A')}% "
330
+ f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
331
+ f"Message: {result_data.get('message', 'No message received.')}"
332
  )
333
+ print("Submission successful.")
334
+ results_df = pd.DataFrame(results_log)
335
+ return final_status, results_df
336
+ except aiohttp.ClientResponseError as e:
337
+ error_detail = f"Server responded with status {e.status}."
338
+ try:
339
+ error_text = await e.response.text()
340
+ try:
341
+ error_json = await e.response.json()
342
+ error_detail += f" Detail: {error_json.get('detail', error_text)}"
343
+ except ValueError:
344
+ error_detail += f" Response: {error_text[:500]}"
345
+ except:
346
+ pass
347
+ status_message = f"Submission Failed: {error_detail}"
348
+ print(status_message)
349
+ results_df = pd.DataFrame(results_log)
350
+ return status_message, results_df
351
+ except asyncio.TimeoutError:
352
+ status_message = "Submission Failed: The request timed out."
353
+ print(status_message)
354
+ results_df = pd.DataFrame(results_log)
355
+ return status_message, results_df
356
+ except aiohttp.ClientError as e:
357
+ status_message = f"Submission Failed: Network error - {e}"
358
+ print(status_message)
359
+ results_df = pd.DataFrame(results_log)
360
+ return status_message, results_df
361
+ except Exception as e:
362
+ status_message = f"An unexpected error occurred during submission: {e}"
363
+ print(status_message)
364
+ results_df = pd.DataFrame(results_log)
365
+ return status_message, results_df
366
+
367
+
368
+ # --- Build Gradio Interface using Blocks ---
369
+ with gr.Blocks() as demo:
370
+ gr.Markdown("# Basic Agent Evaluation Runner")
371
+ gr.Markdown(
372
+ """
373
+ **Instructions:**
374
+ 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
375
+ 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
376
+ 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
377
+ ---
378
+ **Disclaimers:**
379
+ Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
380
+ This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
381
+ """
382
+ )
383
+
384
+ login_button = gr.LoginButton()
385
+
386
+ run_button = gr.Button("Run Evaluation & Submit All Answers")
387
+
388
+ status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
389
+ # Removed max_rows=10 from DataFrame constructor
390
+ results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
391
+
392
+ def sync_wrapper(profile):
393
+ # This wrapper ensures we have access to the profile
394
+ if not profile:
395
+ print("No profile available in sync_wrapper")
396
+ return "Please Login to Hugging Face with the button.", None
397
+ print(f"Profile type in wrapper: {type(profile)}")
398
+ try:
399
+ return asyncio.run(run_and_submit_all(profile))
400
  except Exception as e:
401
+ print(f"Error in sync_wrapper: {e}")
402
+ return f"Error processing request: {e}", None
403
 
404
+ run_button.click(
405
+ fn=sync_wrapper,
406
+ inputs=login_button,
407
+ outputs=[status_output, results_table]
408
+ )
409
+
410
+ if __name__ == "__main__":
411
+ print("\n" + "-"*30 + " App Starting " + "-"*30)
412
+ # Check for SPACE_HOST and SPACE_ID at startup for information
413
+ space_host_startup = os.getenv("SPACE_HOST")
414
+ space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
415
+
416
+ if space_host_startup:
417
+ print(f"✅ SPACE_HOST found: {space_host_startup}")
418
+ print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
419
+ else:
420
+ print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
421
+
422
+ if space_id_startup: # Print repo URLs if SPACE_ID is found
423
+ print(f"✅ SPACE_ID found: {space_id_startup}")
424
+ print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
425
+ print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
426
+ else:
427
+ print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
428
+
429
+ print("-"*(60 + len(" App Starting ")) + "\n")
430
+
431
+ print("Launching Gradio Interface for Basic Agent Evaluation...")
432
+ demo.launch(debug=True, share=False)