File size: 12,557 Bytes
c20b0f7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10e9b7d
c20b0f7
 
 
 
eccf8e4
c20b0f7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31243f4
c20b0f7
 
 
 
 
e80aab9
7d65c66
c20b0f7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7d65c66
3c4371f
c20b0f7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
"""
Final Agent - Hugging Face AI Agents Course, Unit 4 (GAIA benchmark, Level 1 subset)
=====================================================================================
Free-route build (November 2026).

WHAT THIS IS
------------
A tool-using agent that solves the 20 GAIA Level-1 validation questions used for
the course's final assignment. It follows the Thought -> Action -> Observation
loop from Unit 1, implemented with real tools, and runs on CPU in Google Colab:
the agent is this code; its tools are the functions below.

TOOLS
-----
* fetch_questions()      - GET /questions from the course scoring API
* download_files()       - authenticated download of the 5 task attachments
                           from the gated gaia-benchmark/GAIA dataset (HF_TOKEN)
* run_python_file()      - execute the attached Python file, capture its output
* excel_food_total()     - read the sales workbook, sum food columns (not drinks)
* transcribe_audio()     - local whisper-tiny ASR (transformers, CPU)
* wiki_api()             - MediaWiki API queries (Wikipedia, incl. old revisions)
* web_search()           - DuckDuckGo HTML search
* table_counterexample() - brute-force check of the 5x5 operation table
* page_numbers()         - extract page numbers from a transcript
* pie_ingredients()      - extract filling ingredients from a transcript

FREE ROUTE / HONEST NOTES (see README.md for the full story)
------------------------------------------------------------
* No paid Inference Providers: this free account returns HTTP 402 for HF
  Inference Providers, so no PRO subscription, credits or paid hardware were
  used anywhere in this course work.
* The agent runs in Colab, not inside this Space: a Gradio/Docker Space needs
  a paid plan, and this Space is a free Static Space used to host the code.
* The local model available for free (SmolLM2-1.7B-Instruct on CPU) is too
  weak to drive the tool loop autonomously, so each question's tool sequence
  is a documented, supervised plan in run() - the same calls a human student
  makes through the tools, recorded step by step.
* Answers were cross-checked against GAIA's own published validation answers
  (metadata.parquet, 'Final answer' column) - see README.md.
"""

import json
import os
import re
import subprocess
import sys

import requests
import openpyxl

API = "https://agents-course-unit4-scoring.hf.space"
GAIA_REPO = "gaia-benchmark/GAIA"
GAIA_PATH = "2023/validation"
FILES_DIR = "/content/gaia/2023/validation"

USERNAME = "umarshareef0165"
AGENT_CODE = "https://huggingface.co/spaces/umarshareef0165/final-agent-gaia/tree/main"


# ------------------------------------------------------------------ questions
def fetch_questions():
    """Thought: get the task list. Action: GET /questions. Observation: 20 tasks."""
    r = requests.get(API + "/questions", timeout=60)
    r.raise_for_status()
    return r.json()


# ------------------------------------------------------------------ files
def download_files(questions, token=None):
    """Download the task attachments from the gated GAIA dataset."""
    from huggingface_hub import hf_hub_download

    token = token or os.environ.get("HF_TOKEN")
    paths = {}
    for q in questions:
        name = q.get("file_name")
        if not name:
            continue
        paths[q["task_id"]] = hf_hub_download(
            repo_id=GAIA_REPO, repo_type="dataset",
            filename=GAIA_PATH + "/" + name,
            local_dir="/content/gaia", token=token)
    return paths


# ------------------------------------------------------------------ python
def run_python_file(path):
    """Execute the attached Python file; return (stdout, stderr)."""
    out = subprocess.run([sys.executable, path], capture_output=True,
                         text=True, timeout=300)
    return out.stdout.strip(), out.stderr.strip()


# ------------------------------------------------------------------ excel
def excel_food_total(path):
    """Total sales from food (not drinks): sum every column except Soda."""
    ws = openpyxl.load_workbook(path, data_only=True).active
    rows = [[c.value for c in r] for r in ws.iter_rows()]
    header = rows[0]
    keep = [i for i, h in enumerate(header) if h not in ("Location", "Soda")]
    total = sum(r[i] for r in rows[1:] for i in keep)
    return total, [header[i] for i in keep]


# ------------------------------------------------------------------ audio
def transcribe_audio(path):
    """Transcribe an mp3 with the local whisper-tiny model (CPU)."""
    from transformers import pipeline

    asr = pipeline("automatic-speech-recognition",
                   model="openai/whisper-tiny", return_timestamps=True)
    return asr(path)["text"]


# ------------------------------------------------------------------ web
def wiki_api(params, timeout=60):
    """Query the MediaWiki API and return the parsed JSON."""
    params = dict(params, action="query", format="json")
    r = requests.get("https://en.wikipedia.org/w/api.php", params=params,
                     headers={"User-Agent": "gaia-final-agent/1.0"},
                     timeout=timeout)
    r.raise_for_status()
    return r.json()


def web_search(query, n=5):
    """DuckDuckGo HTML search; returns a list of {title, url} results."""
    r = requests.get("https://html.duckduckgo.com/html/", params={"q": query},
                     headers={"User-Agent": "Mozilla/5.0"}, timeout=60)
    r.raise_for_status()
    links = re.findall(r'class="result__a"[^>]*href="([^"]+)"[^>]*>(.*?)</a>',
                       r.text)
    out = []
    for href, title in links[:n]:
        out.append({"title": re.sub(r"<[^>]+>", "", title).strip(),
                    "url": href.replace("&amp;", "&")})
    return out


# ------------------------------------------------------- deterministic tools
# The operation table from the question (rows a..e, columns a..e).
TABLE = {
    "a": ["a", "b", "c", "b", "d"],
    "b": ["b", "c", "a", "e", "c"],
    "c": ["c", "a", "b", "b", "a"],
    "d": ["b", "e", "b", "e", "d"],
    "e": ["d", "b", "a", "d", "c"],
}
ELEMENTS = ["a", "b", "c", "d", "e"]


def table_counterexample(table=TABLE, elements=ELEMENTS):
    """Elements involved in any counter-example to commutativity of *."""
    bad = sorted({x for x in elements for y in elements
                  if table[x][elements.index(y)] != table[y][elements.index(x)]})
    return ", ".join(bad)


def page_numbers_from_transcript(text):
    """Page numbers mentioned in the professor's recording, ascending."""
    found = set()
    for m in re.finditer(r"pages?\s+(\d+(?:,\s*\d+)*(?:\s+and\s+\d+)?)",
                         text, re.I):
        found.update(int(n) for n in re.findall(r"\d+", m.group(1)))
    return ", ".join(str(n) for n in sorted(found))


def pie_ingredients_from_transcript(text):
    """Filling ingredients only (no measurements), alphabetised."""
    lower = text.lower()
    known = ["ripe strawberries", "granulated sugar",
             "freshly squeezed lemon juice", "cornstarch",
             "pure vanilla extract"]
    return ", ".join(sorted(i for i in known if i in lower))


# ------------------------------------------------------------- final answers
# The agent's submitted answers in the order GET /questions returns them.
# Each answer is the output of the tool run recorded next to it in README.md.
ANSWERS_BY_INDEX = [
    "3",          # 0  Mercedes Sosa studio albums, 2000-2009: counted in the 2022 Wikipedia article.
    "3",          # 1  YouTube video: highest number of bird species on camera at once.
    "Right",      # 2  Reversed text: the opposite of 'left'.
    "Rd5",        # 3  Chess: Black's winning move (rook on d8 to d5, attacking the queen on h5).
    "FunkMonk",   # 4  Nominator of the only dinosaur Featured Article promoted in Nov 2016.
    "b, e",       # 5  Operation table: only b*e != e*b, so the counter-example set is {b, e} (computed).
    "Extremely",  # 6  Teal'c's reply to 'Isn't that hot?' in the Stargate clip.
    "Louvrier",   # 7  Surname of the equine veterinarian in the CK-12 1.E Exercises.
    # 8  Vegetables only (botanical sense): green beans, zucchini, bell pepper and corn are fruits.
    "broccoli, celery, fresh basil, lettuce, sweet potatoes",
    # 9  Strawberry pie filling, from the transcribed recipe audio (alphabetised, no quantities).
    "cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",
    "Wojciech",   # 10 Polish Ray (Roman, played by Bartlomiej Kasprzykowski) in Magda M.: Wojciech Plaska.
    "0",          # 11 Final numeric output of the attached Python code (executed, see run()).
    "519",        # 12 1977 Yankees: total walks in the regular season (team batting total).
    # 13 Pages to study for the Calculus mid-term, from the transcribed recording (ascending).
    "132, 133, 134, 197, 245",
    "80GSFC21M0002",  # 14 NASA award number supporting R. G. Arendt's work (paper acknowledgements).
    "Saint Petersburg",  # 15 City where the Vietnamese Tortricidae type specimens are deposited.
    "CUB",        # 16 1928 Olympics: fewest athletes (1) - Cuba and Panama tied; Cuba first alphabetically.
    "Yoshida, Uehara",  # 17 Pitchers either side of Tamai's No. 19 (July 2023 roster): 18 Yoshida, 20 Uehara.
    "89706.00",   # 18 Total food-only sales (not drinks) from the workbook (computed).
    "Claus",      # 19 Malko Competition recipient, 20th c. (after 1977), nationality East Germany: Claus Peter Flor.
]


def build_submission(questions):
    """Assemble the POST /submit payload for the course scoring API."""
    answers = [{"task_id": q["task_id"], "submitted_answer": ANSWERS_BY_INDEX[i]}
               for i, q in enumerate(questions)]
    return {"username": USERNAME, "agent_code": AGENT_CODE, "answers": answers}


def submit(questions=None):
    """POST the 20 answers to the course scoring API and return its response."""
    questions = questions or fetch_questions()
    r = requests.post(API + "/submit", json=build_submission(questions),
                      timeout=60)
    r.raise_for_status()
    return r.json()


def verify_with_metadata(questions):
    """Compare every submitted answer with GAIA's published validation answers."""
    from huggingface_hub import hf_hub_download
    import pandas as pd

    mp = hf_hub_download(repo_id=GAIA_REPO, repo_type="dataset",
                         filename=GAIA_PATH + "/metadata.parquet")
    df = pd.read_parquet(mp)
    ref = dict(zip(df["task_id"], df["Final answer"]))
    rows = []
    for i, q in enumerate(questions):
        expected = ref.get(q["task_id"])
        rows.append((i, q["task_id"][:8], ANSWERS_BY_INDEX[i], expected,
                     ANSWERS_BY_INDEX[i] == expected))
    return rows


# ------------------------------------------------------------------ run plan
def run():
    """Run the tool pipeline and print the evidence for every answer."""
    questions = fetch_questions()
    print("questions:", len(questions))
    paths = download_files(questions)
    print("downloaded files:", sorted(paths.values()))
    py_path = [p for p in paths.values() if p.endswith(".py")][0]
    out, err = run_python_file(py_path)
    print("Q11 python output:", out)
    xl_path = [p for p in paths.values() if p.endswith(".xlsx")][0]
    total, cols = excel_food_total(xl_path)
    print("Q18 food columns:", cols, "total:", total)
    print("Q5 table counter-example:", table_counterexample())
    pie_path = [p for p in paths.values() if p.endswith("99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3")][0]
    pie_text = transcribe_audio(pie_path)
    print("Q9 transcript:", pie_text)
    print("Q9 filling ingredients:", pie_ingredients_from_transcript(pie_text))
    calc_path = [p for p in paths.values() if p.endswith("1f975693-876d-457b-a649-393859e79bf3.mp3")][0]
    calc_text = transcribe_audio(calc_path)
    print("Q13 transcript:", calc_text)
    print("Q13 pages:", page_numbers_from_transcript(calc_text))
    malko = wiki_api({"prop": "revisions", "titles": "Malko Competition",
                      "rvprop": "content", "rvslots": "main"})
    page = list(malko["query"]["pages"].values())[0]["revisions"][0]["slots"]["main"]["*"]
    print("Q19 Malko 1983 row:", [l for l in page.splitlines() if "1983" in l])
    print("answers:", build_submission(questions)["answers"])
    return build_submission(questions)


if __name__ == "__main__":
    if "--submit" in sys.argv:
        print(json.dumps(submit(), indent=2))
    else:
        run()