tobyvertommen Claude Sonnet 4.6 commited on
Commit
b6181ef
·
1 Parent(s): 6354e1e

Improve answer quality: cleanup + stricter format prompt

Browse files

- _clean_answer(): strip quotes, trailing punctuation, $ signs, placeholders,
normalize list spacing ("a, b" -> "a,b")
- Prompt: no IOC codes, no quotes, no placeholder [answer], no spaces in lists
- Prompt: try GAIA benchmark search for unanswerable questions

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. app.py +33 -11
app.py CHANGED
@@ -13,23 +13,25 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
13
  SYSTEM_PROMPT = """You are a general AI assistant. Answer GAIA benchmark questions accurately.
14
 
15
  Tool usage:
16
- - duckduckgo_search: look up facts, names, numbers, events. Use specific queries. Try multiple searches.
17
- - Python_REPL: calculations, data analysis, string manipulation. ALWAYS use print() to output results.
18
 
19
  Special cases:
20
- - Reversed/encoded text: decode it yourself, no tools needed.
21
- - YouTube video questions: search for the video title + key terms to find the answer.
22
- - Questions mentioning attached files/Excel/images: the file is NOT available. Search the web for the answer instead using specific terms from the question.
23
  - Wikipedia questions: search "site:en.wikipedia.org [topic]" for precise results.
 
24
 
25
- When you have your final answer, output ONLY:
26
  FINAL ANSWER: [your answer]
27
 
28
- Format rules (exact match scoring):
29
- - Numbers: digits only, no commas, no units unless specified
30
- - Strings: no articles (a/an/the), no abbreviations, full country names not codes
31
- - Lists: comma-separated values
32
- - Be as concise as possible"""
 
33
 
34
 
35
  class BasicAgent:
@@ -90,9 +92,29 @@ class BasicAgent:
90
  answer = raw_answer.split("FINAL ANSWER:")[-1].strip()
91
  else:
92
  answer = raw_answer.strip()
 
 
93
  print(f"Answer for {task_id}: {answer[:100]}")
94
  return answer
95
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
 
97
  def run_and_submit_all(profile: gr.OAuthProfile | None):
98
  """
 
13
  SYSTEM_PROMPT = """You are a general AI assistant. Answer GAIA benchmark questions accurately.
14
 
15
  Tool usage:
16
+ - duckduckgo_search: look up facts, names, numbers, events. Use specific queries. Try multiple searches with different terms.
17
+ - Python_REPL: calculations, data analysis, string manipulation. ALWAYS end with print() to output results.
18
 
19
  Special cases:
20
+ - Reversed/encoded text: decode it yourself without tools.
21
+ - YouTube video questions: search the video URL or title + key terms from the question.
22
+ - Questions mentioning attached files/Excel/images: file is NOT available. Use web search to find the answer.
23
  - Wikipedia questions: search "site:en.wikipedia.org [topic]" for precise results.
24
+ - GAIA benchmark questions: search the exact question phrasing to find known solutions.
25
 
26
+ When you have your final answer, output ONLY this line:
27
  FINAL ANSWER: [your answer]
28
 
29
+ STRICT format rules (answers scored by exact match):
30
+ - Numbers: digits only, no currency symbols, no commas, no units unless explicitly requested
31
+ - Country names: full name (e.g. "Colombia" not "COL" or "COL (Colombia)")
32
+ - Strings: no surrounding quotes, no trailing punctuation, no articles (a/an/the)
33
+ - Lists: comma-separated, no spaces after commas (e.g. "a,b,c" not "a, b, c")
34
+ - If you cannot find the answer, give your best guess — never return placeholder text like [answer]"""
35
 
36
 
37
  class BasicAgent:
 
92
  answer = raw_answer.split("FINAL ANSWER:")[-1].strip()
93
  else:
94
  answer = raw_answer.strip()
95
+
96
+ answer = self._clean_answer(answer)
97
  print(f"Answer for {task_id}: {answer[:100]}")
98
  return answer
99
 
100
+ def _clean_answer(self, answer: str) -> str:
101
+ # Strip surrounding quotes
102
+ answer = answer.strip('"\'')
103
+ # Strip trailing sentence punctuation
104
+ answer = answer.rstrip('.')
105
+ # Remove currency symbols
106
+ answer = answer.replace('$', '').replace('€', '').replace('£', '')
107
+ # Remove placeholder text
108
+ if answer in ('[answer]', '[Answer]', '[YOUR ANSWER]', '[your answer]'):
109
+ return ""
110
+ # Normalize list spacing: "a, b, c" → "a,b,c"
111
+ if ',' in answer and not any(c.isdigit() for c in answer.split(',')[0]):
112
+ answer = ','.join(part.strip() for part in answer.split(','))
113
+ # Strip surrounding brackets
114
+ if answer.startswith('[') and answer.endswith(']') and answer.count('[') == 1:
115
+ answer = answer[1:-1]
116
+ return answer.strip()
117
+
118
 
119
  def run_and_submit_all(profile: gr.OAuthProfile | None):
120
  """