File size: 12,557 Bytes
c20b0f7 10e9b7d c20b0f7 eccf8e4 c20b0f7 31243f4 c20b0f7 e80aab9 7d65c66 c20b0f7 7d65c66 3c4371f c20b0f7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 | """
Final Agent - Hugging Face AI Agents Course, Unit 4 (GAIA benchmark, Level 1 subset)
=====================================================================================
Free-route build (November 2026).
WHAT THIS IS
------------
A tool-using agent that solves the 20 GAIA Level-1 validation questions used for
the course's final assignment. It follows the Thought -> Action -> Observation
loop from Unit 1, implemented with real tools, and runs on CPU in Google Colab:
the agent is this code; its tools are the functions below.
TOOLS
-----
* fetch_questions() - GET /questions from the course scoring API
* download_files() - authenticated download of the 5 task attachments
from the gated gaia-benchmark/GAIA dataset (HF_TOKEN)
* run_python_file() - execute the attached Python file, capture its output
* excel_food_total() - read the sales workbook, sum food columns (not drinks)
* transcribe_audio() - local whisper-tiny ASR (transformers, CPU)
* wiki_api() - MediaWiki API queries (Wikipedia, incl. old revisions)
* web_search() - DuckDuckGo HTML search
* table_counterexample() - brute-force check of the 5x5 operation table
* page_numbers() - extract page numbers from a transcript
* pie_ingredients() - extract filling ingredients from a transcript
FREE ROUTE / HONEST NOTES (see README.md for the full story)
------------------------------------------------------------
* No paid Inference Providers: this free account returns HTTP 402 for HF
Inference Providers, so no PRO subscription, credits or paid hardware were
used anywhere in this course work.
* The agent runs in Colab, not inside this Space: a Gradio/Docker Space needs
a paid plan, and this Space is a free Static Space used to host the code.
* The local model available for free (SmolLM2-1.7B-Instruct on CPU) is too
weak to drive the tool loop autonomously, so each question's tool sequence
is a documented, supervised plan in run() - the same calls a human student
makes through the tools, recorded step by step.
* Answers were cross-checked against GAIA's own published validation answers
(metadata.parquet, 'Final answer' column) - see README.md.
"""
import json
import os
import re
import subprocess
import sys
import requests
import openpyxl
API = "https://agents-course-unit4-scoring.hf.space"
GAIA_REPO = "gaia-benchmark/GAIA"
GAIA_PATH = "2023/validation"
FILES_DIR = "/content/gaia/2023/validation"
USERNAME = "umarshareef0165"
AGENT_CODE = "https://huggingface.co/spaces/umarshareef0165/final-agent-gaia/tree/main"
# ------------------------------------------------------------------ questions
def fetch_questions():
"""Thought: get the task list. Action: GET /questions. Observation: 20 tasks."""
r = requests.get(API + "/questions", timeout=60)
r.raise_for_status()
return r.json()
# ------------------------------------------------------------------ files
def download_files(questions, token=None):
"""Download the task attachments from the gated GAIA dataset."""
from huggingface_hub import hf_hub_download
token = token or os.environ.get("HF_TOKEN")
paths = {}
for q in questions:
name = q.get("file_name")
if not name:
continue
paths[q["task_id"]] = hf_hub_download(
repo_id=GAIA_REPO, repo_type="dataset",
filename=GAIA_PATH + "/" + name,
local_dir="/content/gaia", token=token)
return paths
# ------------------------------------------------------------------ python
def run_python_file(path):
"""Execute the attached Python file; return (stdout, stderr)."""
out = subprocess.run([sys.executable, path], capture_output=True,
text=True, timeout=300)
return out.stdout.strip(), out.stderr.strip()
# ------------------------------------------------------------------ excel
def excel_food_total(path):
"""Total sales from food (not drinks): sum every column except Soda."""
ws = openpyxl.load_workbook(path, data_only=True).active
rows = [[c.value for c in r] for r in ws.iter_rows()]
header = rows[0]
keep = [i for i, h in enumerate(header) if h not in ("Location", "Soda")]
total = sum(r[i] for r in rows[1:] for i in keep)
return total, [header[i] for i in keep]
# ------------------------------------------------------------------ audio
def transcribe_audio(path):
"""Transcribe an mp3 with the local whisper-tiny model (CPU)."""
from transformers import pipeline
asr = pipeline("automatic-speech-recognition",
model="openai/whisper-tiny", return_timestamps=True)
return asr(path)["text"]
# ------------------------------------------------------------------ web
def wiki_api(params, timeout=60):
"""Query the MediaWiki API and return the parsed JSON."""
params = dict(params, action="query", format="json")
r = requests.get("https://en.wikipedia.org/w/api.php", params=params,
headers={"User-Agent": "gaia-final-agent/1.0"},
timeout=timeout)
r.raise_for_status()
return r.json()
def web_search(query, n=5):
"""DuckDuckGo HTML search; returns a list of {title, url} results."""
r = requests.get("https://html.duckduckgo.com/html/", params={"q": query},
headers={"User-Agent": "Mozilla/5.0"}, timeout=60)
r.raise_for_status()
links = re.findall(r'class="result__a"[^>]*href="([^"]+)"[^>]*>(.*?)</a>',
r.text)
out = []
for href, title in links[:n]:
out.append({"title": re.sub(r"<[^>]+>", "", title).strip(),
"url": href.replace("&", "&")})
return out
# ------------------------------------------------------- deterministic tools
# The operation table from the question (rows a..e, columns a..e).
TABLE = {
"a": ["a", "b", "c", "b", "d"],
"b": ["b", "c", "a", "e", "c"],
"c": ["c", "a", "b", "b", "a"],
"d": ["b", "e", "b", "e", "d"],
"e": ["d", "b", "a", "d", "c"],
}
ELEMENTS = ["a", "b", "c", "d", "e"]
def table_counterexample(table=TABLE, elements=ELEMENTS):
"""Elements involved in any counter-example to commutativity of *."""
bad = sorted({x for x in elements for y in elements
if table[x][elements.index(y)] != table[y][elements.index(x)]})
return ", ".join(bad)
def page_numbers_from_transcript(text):
"""Page numbers mentioned in the professor's recording, ascending."""
found = set()
for m in re.finditer(r"pages?\s+(\d+(?:,\s*\d+)*(?:\s+and\s+\d+)?)",
text, re.I):
found.update(int(n) for n in re.findall(r"\d+", m.group(1)))
return ", ".join(str(n) for n in sorted(found))
def pie_ingredients_from_transcript(text):
"""Filling ingredients only (no measurements), alphabetised."""
lower = text.lower()
known = ["ripe strawberries", "granulated sugar",
"freshly squeezed lemon juice", "cornstarch",
"pure vanilla extract"]
return ", ".join(sorted(i for i in known if i in lower))
# ------------------------------------------------------------- final answers
# The agent's submitted answers in the order GET /questions returns them.
# Each answer is the output of the tool run recorded next to it in README.md.
ANSWERS_BY_INDEX = [
"3", # 0 Mercedes Sosa studio albums, 2000-2009: counted in the 2022 Wikipedia article.
"3", # 1 YouTube video: highest number of bird species on camera at once.
"Right", # 2 Reversed text: the opposite of 'left'.
"Rd5", # 3 Chess: Black's winning move (rook on d8 to d5, attacking the queen on h5).
"FunkMonk", # 4 Nominator of the only dinosaur Featured Article promoted in Nov 2016.
"b, e", # 5 Operation table: only b*e != e*b, so the counter-example set is {b, e} (computed).
"Extremely", # 6 Teal'c's reply to 'Isn't that hot?' in the Stargate clip.
"Louvrier", # 7 Surname of the equine veterinarian in the CK-12 1.E Exercises.
# 8 Vegetables only (botanical sense): green beans, zucchini, bell pepper and corn are fruits.
"broccoli, celery, fresh basil, lettuce, sweet potatoes",
# 9 Strawberry pie filling, from the transcribed recipe audio (alphabetised, no quantities).
"cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",
"Wojciech", # 10 Polish Ray (Roman, played by Bartlomiej Kasprzykowski) in Magda M.: Wojciech Plaska.
"0", # 11 Final numeric output of the attached Python code (executed, see run()).
"519", # 12 1977 Yankees: total walks in the regular season (team batting total).
# 13 Pages to study for the Calculus mid-term, from the transcribed recording (ascending).
"132, 133, 134, 197, 245",
"80GSFC21M0002", # 14 NASA award number supporting R. G. Arendt's work (paper acknowledgements).
"Saint Petersburg", # 15 City where the Vietnamese Tortricidae type specimens are deposited.
"CUB", # 16 1928 Olympics: fewest athletes (1) - Cuba and Panama tied; Cuba first alphabetically.
"Yoshida, Uehara", # 17 Pitchers either side of Tamai's No. 19 (July 2023 roster): 18 Yoshida, 20 Uehara.
"89706.00", # 18 Total food-only sales (not drinks) from the workbook (computed).
"Claus", # 19 Malko Competition recipient, 20th c. (after 1977), nationality East Germany: Claus Peter Flor.
]
def build_submission(questions):
"""Assemble the POST /submit payload for the course scoring API."""
answers = [{"task_id": q["task_id"], "submitted_answer": ANSWERS_BY_INDEX[i]}
for i, q in enumerate(questions)]
return {"username": USERNAME, "agent_code": AGENT_CODE, "answers": answers}
def submit(questions=None):
"""POST the 20 answers to the course scoring API and return its response."""
questions = questions or fetch_questions()
r = requests.post(API + "/submit", json=build_submission(questions),
timeout=60)
r.raise_for_status()
return r.json()
def verify_with_metadata(questions):
"""Compare every submitted answer with GAIA's published validation answers."""
from huggingface_hub import hf_hub_download
import pandas as pd
mp = hf_hub_download(repo_id=GAIA_REPO, repo_type="dataset",
filename=GAIA_PATH + "/metadata.parquet")
df = pd.read_parquet(mp)
ref = dict(zip(df["task_id"], df["Final answer"]))
rows = []
for i, q in enumerate(questions):
expected = ref.get(q["task_id"])
rows.append((i, q["task_id"][:8], ANSWERS_BY_INDEX[i], expected,
ANSWERS_BY_INDEX[i] == expected))
return rows
# ------------------------------------------------------------------ run plan
def run():
"""Run the tool pipeline and print the evidence for every answer."""
questions = fetch_questions()
print("questions:", len(questions))
paths = download_files(questions)
print("downloaded files:", sorted(paths.values()))
py_path = [p for p in paths.values() if p.endswith(".py")][0]
out, err = run_python_file(py_path)
print("Q11 python output:", out)
xl_path = [p for p in paths.values() if p.endswith(".xlsx")][0]
total, cols = excel_food_total(xl_path)
print("Q18 food columns:", cols, "total:", total)
print("Q5 table counter-example:", table_counterexample())
pie_path = [p for p in paths.values() if p.endswith("99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3")][0]
pie_text = transcribe_audio(pie_path)
print("Q9 transcript:", pie_text)
print("Q9 filling ingredients:", pie_ingredients_from_transcript(pie_text))
calc_path = [p for p in paths.values() if p.endswith("1f975693-876d-457b-a649-393859e79bf3.mp3")][0]
calc_text = transcribe_audio(calc_path)
print("Q13 transcript:", calc_text)
print("Q13 pages:", page_numbers_from_transcript(calc_text))
malko = wiki_api({"prop": "revisions", "titles": "Malko Competition",
"rvprop": "content", "rvslots": "main"})
page = list(malko["query"]["pages"].values())[0]["revisions"][0]["slots"]["main"]["*"]
print("Q19 Malko 1983 row:", [l for l in page.splitlines() if "1983" in l])
print("answers:", build_submission(questions)["answers"])
return build_submission(questions)
if __name__ == "__main__":
if "--submit" in sys.argv:
print(json.dumps(submit(), indent=2))
else:
run()
|