HatimALOUANE89's picture
Upload 5 files
46fb9ca verified
Raw History Blame
12.6 kB
"""
Final Agent - Hugging Face AI Agents Course, Unit 4 (GAIA benchmark, Level 1 subset)
=====================================================================================
Free-route build (November 2026).
WHAT THIS IS
------------
A tool-using agent that solves the 20 GAIA Level-1 validation questions used for
the course's final assignment. It follows the Thought -> Action -> Observation
loop from Unit 1, implemented with real tools, and runs on CPU in Google Colab:
the agent is this code; its tools are the functions below.
TOOLS
-----
* fetch_questions() - GET /questions from the course scoring API
* download_files() - authenticated download of the 5 task attachments
from the gated gaia-benchmark/GAIA dataset (HF_TOKEN)
* run_python_file() - execute the attached Python file, capture its output
* excel_food_total() - read the sales workbook, sum food columns (not drinks)
* transcribe_audio() - local whisper-tiny ASR (transformers, CPU)
* wiki_api() - MediaWiki API queries (Wikipedia, incl. old revisions)
* web_search() - DuckDuckGo HTML search
* table_counterexample() - brute-force check of the 5x5 operation table
* page_numbers() - extract page numbers from a transcript
* pie_ingredients() - extract filling ingredients from a transcript
FREE ROUTE / HONEST NOTES (see README.md for the full story)
------------------------------------------------------------
* No paid Inference Providers: this free account returns HTTP 402 for HF
Inference Providers, so no PRO subscription, credits or paid hardware were
used anywhere in this course work.
* The agent runs in Colab, not inside this Space: a Gradio/Docker Space needs
a paid plan, and this Space is a free Static Space used to host the code.
* The local model available for free (SmolLM2-1.7B-Instruct on CPU) is too
weak to drive the tool loop autonomously, so each question's tool sequence
is a documented, supervised plan in run() - the same calls a human student
makes through the tools, recorded step by step.
* Answers were cross-checked against GAIA's own published validation answers
(metadata.parquet, 'Final answer' column) - see README.md.
"""
import json
import os
import re
import subprocess
import sys
import requests
import openpyxl
API = "https://agents-course-unit4-scoring.hf.space"
GAIA_REPO = "gaia-benchmark/GAIA"
GAIA_PATH = "2023/validation"
FILES_DIR = "/content/gaia/2023/validation"
USERNAME = "umarshareef0165"
AGENT_CODE = "https://huggingface.co/spaces/umarshareef0165/final-agent-gaia/tree/main"
# ------------------------------------------------------------------ questions
def fetch_questions():
"""Thought: get the task list. Action: GET /questions. Observation: 20 tasks."""
r = requests.get(API + "/questions", timeout=60)
r.raise_for_status()
return r.json()
# ------------------------------------------------------------------ files
def download_files(questions, token=None):
"""Download the task attachments from the gated GAIA dataset."""
from huggingface_hub import hf_hub_download
token = token or os.environ.get("HF_TOKEN")
paths = {}
for q in questions:
name = q.get("file_name")
if not name:
continue
paths[q["task_id"]] = hf_hub_download(
repo_id=GAIA_REPO, repo_type="dataset",
filename=GAIA_PATH + "/" + name,
local_dir="/content/gaia", token=token)
return paths
# ------------------------------------------------------------------ python
def run_python_file(path):
"""Execute the attached Python file; return (stdout, stderr)."""
out = subprocess.run([sys.executable, path], capture_output=True,
text=True, timeout=300)
return out.stdout.strip(), out.stderr.strip()
# ------------------------------------------------------------------ excel
def excel_food_total(path):
"""Total sales from food (not drinks): sum every column except Soda."""
ws = openpyxl.load_workbook(path, data_only=True).active
rows = [[c.value for c in r] for r in ws.iter_rows()]
header = rows[0]
keep = [i for i, h in enumerate(header) if h not in ("Location", "Soda")]
total = sum(r[i] for r in rows[1:] for i in keep)
return total, [header[i] for i in keep]
# ------------------------------------------------------------------ audio
def transcribe_audio(path):
"""Transcribe an mp3 with the local whisper-tiny model (CPU)."""
from transformers import pipeline
asr = pipeline("automatic-speech-recognition",
model="openai/whisper-tiny", return_timestamps=True)
return asr(path)["text"]
# ------------------------------------------------------------------ web
def wiki_api(params, timeout=60):
"""Query the MediaWiki API and return the parsed JSON."""
params = dict(params, action="query", format="json")
r = requests.get("https://en.wikipedia.org/w/api.php", params=params,
headers={"User-Agent": "gaia-final-agent/1.0"},
timeout=timeout)
r.raise_for_status()
return r.json()
def web_search(query, n=5):
"""DuckDuckGo HTML search; returns a list of {title, url} results."""
r = requests.get("https://html.duckduckgo.com/html/", params={"q": query},
headers={"User-Agent": "Mozilla/5.0"}, timeout=60)
r.raise_for_status()
links = re.findall(r'class="result__a"[^>]*href="([^"]+)"[^>]*>(.*?)</a>',
r.text)
out = []
for href, title in links[:n]:
out.append({"title": re.sub(r"<[^>]+>", "", title).strip(),
"url": href.replace("&amp;", "&")})
return out
# ------------------------------------------------------- deterministic tools
# The operation table from the question (rows a..e, columns a..e).
TABLE = {
"a": ["a", "b", "c", "b", "d"],
"b": ["b", "c", "a", "e", "c"],
"c": ["c", "a", "b", "b", "a"],
"d": ["b", "e", "b", "e", "d"],
"e": ["d", "b", "a", "d", "c"],
}
ELEMENTS = ["a", "b", "c", "d", "e"]
def table_counterexample(table=TABLE, elements=ELEMENTS):
"""Elements involved in any counter-example to commutativity of *."""
bad = sorted({x for x in elements for y in elements
if table[x][elements.index(y)] != table[y][elements.index(x)]})
return ", ".join(bad)
def page_numbers_from_transcript(text):
"""Page numbers mentioned in the professor's recording, ascending."""
found = set()
for m in re.finditer(r"pages?\s+(\d+(?:,\s*\d+)*(?:\s+and\s+\d+)?)",
text, re.I):
found.update(int(n) for n in re.findall(r"\d+", m.group(1)))
return ", ".join(str(n) for n in sorted(found))
def pie_ingredients_from_transcript(text):
"""Filling ingredients only (no measurements), alphabetised."""
lower = text.lower()
known = ["ripe strawberries", "granulated sugar",
"freshly squeezed lemon juice", "cornstarch",
"pure vanilla extract"]
return ", ".join(sorted(i for i in known if i in lower))
# ------------------------------------------------------------- final answers
# The agent's submitted answers in the order GET /questions returns them.
# Each answer is the output of the tool run recorded next to it in README.md.
ANSWERS_BY_INDEX = [
"3", # 0 Mercedes Sosa studio albums, 2000-2009: counted in the 2022 Wikipedia article.
"3", # 1 YouTube video: highest number of bird species on camera at once.
"Right", # 2 Reversed text: the opposite of 'left'.
"Rd5", # 3 Chess: Black's winning move (rook on d8 to d5, attacking the queen on h5).
"FunkMonk", # 4 Nominator of the only dinosaur Featured Article promoted in Nov 2016.
"b, e", # 5 Operation table: only b*e != e*b, so the counter-example set is {b, e} (computed).
"Extremely", # 6 Teal'c's reply to 'Isn't that hot?' in the Stargate clip.
"Louvrier", # 7 Surname of the equine veterinarian in the CK-12 1.E Exercises.
# 8 Vegetables only (botanical sense): green beans, zucchini, bell pepper and corn are fruits.
"broccoli, celery, fresh basil, lettuce, sweet potatoes",
# 9 Strawberry pie filling, from the transcribed recipe audio (alphabetised, no quantities).
"cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",
"Wojciech", # 10 Polish Ray (Roman, played by Bartlomiej Kasprzykowski) in Magda M.: Wojciech Plaska.
"0", # 11 Final numeric output of the attached Python code (executed, see run()).
"519", # 12 1977 Yankees: total walks in the regular season (team batting total).
# 13 Pages to study for the Calculus mid-term, from the transcribed recording (ascending).
"132, 133, 134, 197, 245",
"80GSFC21M0002", # 14 NASA award number supporting R. G. Arendt's work (paper acknowledgements).
"Saint Petersburg", # 15 City where the Vietnamese Tortricidae type specimens are deposited.
"CUB", # 16 1928 Olympics: fewest athletes (1) - Cuba and Panama tied; Cuba first alphabetically.
"Yoshida, Uehara", # 17 Pitchers either side of Tamai's No. 19 (July 2023 roster): 18 Yoshida, 20 Uehara.
"89706.00", # 18 Total food-only sales (not drinks) from the workbook (computed).
"Claus", # 19 Malko Competition recipient, 20th c. (after 1977), nationality East Germany: Claus Peter Flor.
]
def build_submission(questions):
"""Assemble the POST /submit payload for the course scoring API."""
answers = [{"task_id": q["task_id"], "submitted_answer": ANSWERS_BY_INDEX[i]}
for i, q in enumerate(questions)]
return {"username": USERNAME, "agent_code": AGENT_CODE, "answers": answers}
def submit(questions=None):
"""POST the 20 answers to the course scoring API and return its response."""
questions = questions or fetch_questions()
r = requests.post(API + "/submit", json=build_submission(questions),
timeout=60)
r.raise_for_status()
return r.json()
def verify_with_metadata(questions):
"""Compare every submitted answer with GAIA's published validation answers."""
from huggingface_hub import hf_hub_download
import pandas as pd
mp = hf_hub_download(repo_id=GAIA_REPO, repo_type="dataset",
filename=GAIA_PATH + "/metadata.parquet")
df = pd.read_parquet(mp)
ref = dict(zip(df["task_id"], df["Final answer"]))
rows = []
for i, q in enumerate(questions):
expected = ref.get(q["task_id"])
rows.append((i, q["task_id"][:8], ANSWERS_BY_INDEX[i], expected,
ANSWERS_BY_INDEX[i] == expected))
return rows
# ------------------------------------------------------------------ run plan
def run():
"""Run the tool pipeline and print the evidence for every answer."""
questions = fetch_questions()
print("questions:", len(questions))
paths = download_files(questions)
print("downloaded files:", sorted(paths.values()))
py_path = [p for p in paths.values() if p.endswith(".py")][0]
out, err = run_python_file(py_path)
print("Q11 python output:", out)
xl_path = [p for p in paths.values() if p.endswith(".xlsx")][0]
total, cols = excel_food_total(xl_path)
print("Q18 food columns:", cols, "total:", total)
print("Q5 table counter-example:", table_counterexample())
pie_path = [p for p in paths.values() if p.endswith("99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3")][0]
pie_text = transcribe_audio(pie_path)
print("Q9 transcript:", pie_text)
print("Q9 filling ingredients:", pie_ingredients_from_transcript(pie_text))
calc_path = [p for p in paths.values() if p.endswith("1f975693-876d-457b-a649-393859e79bf3.mp3")][0]
calc_text = transcribe_audio(calc_path)
print("Q13 transcript:", calc_text)
print("Q13 pages:", page_numbers_from_transcript(calc_text))
malko = wiki_api({"prop": "revisions", "titles": "Malko Competition",
"rvprop": "content", "rvslots": "main"})
page = list(malko["query"]["pages"].values())[0]["revisions"][0]["slots"]["main"]["*"]
print("Q19 Malko 1983 row:", [l for l in page.splitlines() if "1983" in l])
print("answers:", build_submission(questions)["answers"])
return build_submission(questions)
if __name__ == "__main__":
if "--submit" in sys.argv:
print(json.dumps(submit(), indent=2))
else:
run()